feat: add measured detection review loop

backend/app/services/detection_review_service.py

@@ -0,0 +1,252 @@
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any
+from uuid import UUID
+
+from sqlalchemy.orm import Session
+
+from app.core.errors import AppError
+from app.models import Detection, DetectionReview, QualityCheck, VectorFeature
+from app.schemas.detection_review import (
+    DetectionReviewList,
+    DetectionReviewRead,
+    DetectionReviewSummary,
+    DetectionReviewUpsert,
+)
+
+
+class DetectionReviewService:
+    ALLOWED_DECISIONS = {
+        "false_positive": {
+            "confirmed_model_false_positive",
+            "reference_gap_or_change",
+            "qa_alignment_mismatch",
+            "uncertain",
+            "unreviewed",
+        },
+        "false_negative": {
+            "confirmed_model_false_negative",
+            "reference_gap_or_change",
+            "qa_alignment_mismatch",
+            "imagery_obscured_or_uncertain",
+            "uncertain",
+            "unreviewed",
+        },
+    }
+
+    @staticmethod
+    def _quality_check(db: Session, project_id: UUID, quality_check_id: UUID) -> QualityCheck:
+        quality_check = db.get(QualityCheck, quality_check_id)
+        if not quality_check or quality_check.project_id != project_id:
+            raise AppError(code="QUALITY_CHECK_NOT_FOUND", message="Quality check not found", status_code=404)
+        if quality_check.check_type != "detections_vs_reference":
+            raise AppError(
+                code="DETECTION_REVIEW_UNSUPPORTED",
+                message="Only persisted detection-versus-reference quality checks can be reviewed",
+                status_code=422,
+            )
+        return quality_check
+
+    @staticmethod
+    def _evidence_items(quality_check: QualityCheck) -> list[dict[str, str]]:
+        findings = quality_check.findings_json or {}
+        items: list[dict[str, str]] = []
+        for role, key, id_key in (
+            ("false_positive", "false_positive_evidence", "candidate_feature_id"),
+            ("false_negative", "false_negative_evidence", "reference_feature_id"),
+        ):
+            evidence_rows = findings.get(key)
+            if not isinstance(evidence_rows, list):
+                continue
+            for evidence in evidence_rows:
+                if not isinstance(evidence, dict):
+                    continue
+                value = str(evidence.get(id_key) or "").strip()
+                if value:
+                    items.append({"evidence_role": role, "evidence_feature_id": value})
+        return items
+
+    @staticmethod
+    def _uuid(value: str) -> UUID | None:
+        try:
+            return UUID(value)
+        except (TypeError, ValueError):
+            return None
+
+    @staticmethod
+    def _review_index(db: Session, quality_check_id: UUID) -> dict[tuple[str, str], DetectionReview]:
+        rows = db.query(DetectionReview).filter(DetectionReview.quality_check_id == quality_check_id).all()
+        return {(row.evidence_role, row.evidence_feature_id): row for row in rows}
+
+    @staticmethod
+    def _summary(evidence: list[dict[str, str]], reviews: dict[tuple[str, str], DetectionReview]) -> DetectionReviewSummary:
+        evidence_keys = {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence}
+        decisions = Counter(
+            reviews[key].decision if key in reviews else "unreviewed"
+            for key in evidence_keys
+        )
+        reviewed = sum(count for decision, count in decisions.items() if decision != "unreviewed")
+        false_positive_total = sum(1 for item in evidence if item["evidence_role"] == "false_positive")
+        false_negative_total = sum(1 for item in evidence if item["evidence_role"] == "false_negative")
+        return DetectionReviewSummary(
+            total=len(evidence),
+            reviewed=reviewed,
+            remaining=max(len(evidence) - reviewed, 0),
+            false_positive_total=false_positive_total,
+            false_negative_total=false_negative_total,
+            decision_counts=dict(sorted(decisions.items())),
+        )
+
+    @staticmethod
+    def _read_item(
+        db: Session,
+        quality_check: QualityCheck,
+        evidence: dict[str, str],
+        review: DetectionReview | None,
+    ) -> DetectionReviewRead:
+        role = evidence["evidence_role"]
+        feature_id = evidence["evidence_feature_id"]
+        feature_uuid = DetectionReviewService._uuid(feature_id)
+        detection = db.get(Detection, feature_uuid) if role == "false_positive" and feature_uuid else None
+        reference = db.get(VectorFeature, feature_uuid) if role == "false_negative" and feature_uuid else None
+        return DetectionReviewRead(
+            id=review.id if review else None,
+            project_id=quality_check.project_id,
+            quality_check_id=quality_check.id,
+            analysis_run_id=quality_check.analysis_run_id,
+            evidence_role=role,
+            evidence_feature_id=feature_id,
+            detection_id=detection.id if detection else review.detection_id if review else None,
+            reference_feature_id=reference.id if reference else review.reference_feature_id if review else None,
+            decision=review.decision if review else "unreviewed",
+            notes=review.notes if review else None,
+            reviewed_by=review.reviewed_by if review else None,
+            confidence=detection.confidence if detection else None,
+            class_name=(detection.class_name if detection else reference.feature_class if reference else None),
+            source_tile_path=detection.source_tile_path if detection else None,
+            created_at=review.created_at if review else None,
+            updated_at=review.updated_at if review else None,
+        )
+
+    @staticmethod
+    def list_reviews(
+        db: Session,
+        *,
+        project_id: UUID,
+        quality_check_id: UUID,
+        evidence_role: str | None = None,
+        decision: str | None = None,
+        reviewed: bool | None = None,
+        limit: int = 50,
+        offset: int = 0,
+    ) -> DetectionReviewList:
+        quality_check = DetectionReviewService._quality_check(db, project_id, quality_check_id)
+        evidence = DetectionReviewService._evidence_items(quality_check)
+        reviews = DetectionReviewService._review_index(db, quality_check_id)
+        filtered = [item for item in evidence if evidence_role is None or item["evidence_role"] == evidence_role]
+        if decision is not None:
+            filtered = [
+                item
+                for item in filtered
+                if (reviews.get((item["evidence_role"], item["evidence_feature_id"])).decision
+                    if reviews.get((item["evidence_role"], item["evidence_feature_id"]))
+                    else "unreviewed")
+                == decision
+            ]
+        if reviewed is not None:
+            filtered = [
+                item
+                for item in filtered
+                if (
+                    (reviews.get((item["evidence_role"], item["evidence_feature_id"])).decision
+                     if reviews.get((item["evidence_role"], item["evidence_feature_id"]))
+                     else "unreviewed")
+                    != "unreviewed"
+                )
+                == reviewed
+            ]
+        page = filtered[offset : offset + limit]
+        return DetectionReviewList(
+            items=[
+                DetectionReviewService._read_item(
+                    db,
+                    quality_check,
+                    item,
+                    reviews.get((item["evidence_role"], item["evidence_feature_id"])),
+                )
+                for item in page
+            ],
+            total=len(filtered),
+            limit=limit,
+            offset=offset,
+            summary=DetectionReviewService._summary(evidence, reviews),
+        )
+
+    @staticmethod
+    def upsert_review(
+        db: Session,
+        *,
+        project_id: UUID,
+        quality_check_id: UUID,
+        payload: DetectionReviewUpsert,
+    ) -> DetectionReviewRead:
+        quality_check = DetectionReviewService._quality_check(db, project_id, quality_check_id)
+        if payload.decision not in DetectionReviewService.ALLOWED_DECISIONS[payload.evidence_role]:
+            raise AppError(
+                code="INVALID_DETECTION_REVIEW_DECISION",
+                message="The review decision is not valid for this evidence role",
+                details={"evidence_role": payload.evidence_role, "decision": payload.decision},
+                status_code=422,
+            )
+        evidence = DetectionReviewService._evidence_items(quality_check)
+        evidence_key = (payload.evidence_role, payload.evidence_feature_id)
+        if evidence_key not in {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence}:
+            raise AppError(
+                code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND",
+                message="The evidence feature does not belong to this quality check",
+                status_code=404,
+            )
+        feature_uuid = DetectionReviewService._uuid(payload.evidence_feature_id)
+        detection = db.get(Detection, feature_uuid) if payload.evidence_role == "false_positive" and feature_uuid else None
+        reference = db.get(VectorFeature, feature_uuid) if payload.evidence_role == "false_negative" and feature_uuid else None
+        if payload.evidence_role == "false_positive" and (not detection or detection.analysis_run_id != quality_check.analysis_run_id):
+            raise AppError(code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND", message="Persisted detection evidence was not found", status_code=404)
+        if payload.evidence_role == "false_negative" and (not reference or reference.dataset_id != quality_check.reference_dataset_id):
+            raise AppError(code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND", message="Persisted reference evidence was not found", status_code=404)
+
+        review = (
+            db.query(DetectionReview)
+            .filter(
+                DetectionReview.quality_check_id == quality_check_id,
+                DetectionReview.evidence_role == payload.evidence_role,
+                DetectionReview.evidence_feature_id == payload.evidence_feature_id,
+            )
+            .first()
+        )
+        if review is None:
+            review = DetectionReview(
+                project_id=project_id,
+                quality_check_id=quality_check_id,
+                analysis_run_id=quality_check.analysis_run_id,
+                evidence_role=payload.evidence_role,
+                evidence_feature_id=payload.evidence_feature_id,
+                detection_id=detection.id if detection else None,
+                reference_feature_id=reference.id if reference else None,
+                decision=payload.decision,
+                notes=payload.notes.strip() if payload.notes and payload.notes.strip() else None,
+                reviewed_by=payload.reviewed_by.strip(),
+            )
+        else:
+            review.decision = payload.decision
+            review.notes = payload.notes.strip() if payload.notes and payload.notes.strip() else None
+            review.reviewed_by = payload.reviewed_by.strip()
+        db.add(review)
+        db.commit()
+        db.refresh(review)
+        return DetectionReviewService._read_item(
+            db,
+            quality_check,
+            {"evidence_role": payload.evidence_role, "evidence_feature_id": payload.evidence_feature_id},
+            review,
+        )

backend/app/services/quality_evidence_service.py

@@ -5,10 +5,11 @@ from uuid import UUID
 
 from geoalchemy2.shape import to_shape
 from shapely.geometry import mapping
+from sqlalchemy import or_
 from sqlalchemy.orm import Session
 
 from app.core.errors import AppError
-from app.models import Detection, QualityCheck, Segmentation, VectorFeature
+from app.models import Detection, DetectionReview, QualityCheck, Segmentation, VectorFeature
 
 
 class QualityEvidenceService:
@@ -21,8 +22,9 @@ class QualityEvidenceService:
         findings = quality_check.findings_json or {}
         features: list[dict[str, Any]] = []
         warnings: list[str] = []
-        candidate_index = QualityEvidenceService._candidate_feature_index(db, quality_check)
-        reference_index = QualityEvidenceService._reference_feature_index(db, quality_check)
+        candidate_ids, reference_ids = QualityEvidenceService._evidence_identifiers(findings)
+        candidate_index = QualityEvidenceService._candidate_feature_index(db, quality_check, candidate_ids)
+        reference_index = QualityEvidenceService._reference_feature_index(db, quality_check, reference_ids)
 
         for evidence in QualityEvidenceService._evidence_items(findings.get("match_evidence")):
             candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id"))
@@ -89,6 +91,8 @@ class QualityEvidenceService:
             else:
                 warnings.append(f"False-negative evidence feature not found: {reference_id}")
 
+        QualityEvidenceService._annotate_reviews(db, quality_check, features)
+
         return {
             "quality_check_id": str(quality_check.id),
             "project_id": str(quality_check.project_id),
@@ -117,34 +121,129 @@ class QualityEvidenceService:
         return text or None
 
     @staticmethod
-    def _candidate_feature_index(db: Session, quality_check: QualityCheck) -> dict[str, Any]:
+    def _evidence_identifiers(findings: dict[str, Any]) -> tuple[set[str], set[str]]:
+        candidate_ids: set[str] = set()
+        reference_ids: set[str] = set()
+        for evidence in QualityEvidenceService._evidence_items(findings.get("match_evidence")):
+            candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id"))
+            reference_id = QualityEvidenceService._string_value(evidence.get("reference_feature_id"))
+            if candidate_id:
+                candidate_ids.add(candidate_id)
+            if reference_id:
+                reference_ids.add(reference_id)
+        for evidence in QualityEvidenceService._evidence_items(findings.get("false_positive_evidence")):
+            candidate_id = QualityEvidenceService._string_value(evidence.get("candidate_feature_id"))
+            if candidate_id:
+                candidate_ids.add(candidate_id)
+        for evidence in QualityEvidenceService._evidence_items(findings.get("false_negative_evidence")):
+            reference_id = QualityEvidenceService._string_value(evidence.get("reference_feature_id"))
+            if reference_id:
+                reference_ids.add(reference_id)
+        return candidate_ids, reference_ids
+
+    @staticmethod
+    def _uuid_identifiers(identifiers: set[str]) -> list[UUID]:
+        values: list[UUID] = []
+        for identifier in identifiers:
+            try:
+                values.append(UUID(identifier))
+            except (TypeError, ValueError):
+                continue
+        return values
+
+    @staticmethod
+    def _vector_feature_rows(db: Session, dataset_id: UUID, identifiers: set[str]) -> list[VectorFeature]:
+        if not identifiers:
+            return []
+        conditions = [VectorFeature.source_feature_id.in_(identifiers)]
+        uuid_identifiers = QualityEvidenceService._uuid_identifiers(identifiers)
+        if uuid_identifiers:
+            conditions.append(VectorFeature.id.in_(uuid_identifiers))
+        return (
+            db.query(VectorFeature)
+            .filter(VectorFeature.dataset_id == dataset_id, or_(*conditions))
+            .all()
+        )
+
+    @staticmethod
+    def _candidate_feature_index(
+        db: Session,
+        quality_check: QualityCheck,
+        identifiers: set[str],
+    ) -> dict[str, Any]:
         index: dict[str, Any] = {}
+        if not identifiers:
+            return index
+        uuid_identifiers = QualityEvidenceService._uuid_identifiers(identifiers)
         if quality_check.candidate_dataset_id:
-            for row in db.query(VectorFeature).filter(VectorFeature.dataset_id == quality_check.candidate_dataset_id).all():
+            for row in QualityEvidenceService._vector_feature_rows(db, quality_check.candidate_dataset_id, identifiers):
                 QualityEvidenceService._add_index_keys(index, row)
-            for row in db.query(Detection).filter(Detection.dataset_id == quality_check.candidate_dataset_id).all():
-                QualityEvidenceService._add_index_keys(index, row)
-            for row in db.query(Segmentation).filter(Segmentation.dataset_id == quality_check.candidate_dataset_id).all():
-                QualityEvidenceService._add_index_keys(index, row)
-            if quality_check.analysis_run_id:
-                for row in db.query(Detection).filter(Detection.analysis_run_id == quality_check.analysis_run_id).all():
+            if uuid_identifiers:
+                for row in db.query(Detection).filter(
+                    Detection.dataset_id == quality_check.candidate_dataset_id,
+                    Detection.id.in_(uuid_identifiers),
+                ).all():
                     QualityEvidenceService._add_index_keys(index, row)
-                for row in db.query(Segmentation).filter(Segmentation.analysis_run_id == quality_check.analysis_run_id).all():
+                for row in db.query(Segmentation).filter(
+                    Segmentation.dataset_id == quality_check.candidate_dataset_id,
+                    Segmentation.id.in_(uuid_identifiers),
+                ).all():
                     QualityEvidenceService._add_index_keys(index, row)
-        elif quality_check.analysis_run_id:
-            for row in db.query(Detection).filter(Detection.analysis_run_id == quality_check.analysis_run_id).all():
+        if quality_check.analysis_run_id and uuid_identifiers:
+            for row in db.query(Detection).filter(
+                Detection.analysis_run_id == quality_check.analysis_run_id,
+                Detection.id.in_(uuid_identifiers),
+            ).all():
                 QualityEvidenceService._add_index_keys(index, row)
-            for row in db.query(Segmentation).filter(Segmentation.analysis_run_id == quality_check.analysis_run_id).all():
+            for row in db.query(Segmentation).filter(
+                Segmentation.analysis_run_id == quality_check.analysis_run_id,
+                Segmentation.id.in_(uuid_identifiers),
+            ).all():
                 QualityEvidenceService._add_index_keys(index, row)
         return index
 
     @staticmethod
-    def _reference_feature_index(db: Session, quality_check: QualityCheck) -> dict[str, Any]:
+    def _reference_feature_index(
+        db: Session,
+        quality_check: QualityCheck,
+        identifiers: set[str],
+    ) -> dict[str, Any]:
         index: dict[str, Any] = {}
-        for row in db.query(VectorFeature).filter(VectorFeature.dataset_id == quality_check.reference_dataset_id).all():
+        for row in QualityEvidenceService._vector_feature_rows(db, quality_check.reference_dataset_id, identifiers):
             QualityEvidenceService._add_index_keys(index, row)
         return index
 
+    @staticmethod
+    def _annotate_reviews(
+        db: Session,
+        quality_check: QualityCheck,
+        features: list[dict[str, Any]],
+    ) -> None:
+        if quality_check.check_type != "detections_vs_reference":
+            return
+        reviews = db.query(DetectionReview).filter(DetectionReview.quality_check_id == quality_check.id).all()
+        review_index = {(row.evidence_role, row.evidence_feature_id): row for row in reviews}
+        for feature in features:
+            properties = feature.get("properties")
+            if not isinstance(properties, dict):
+                continue
+            role = QualityEvidenceService._string_value(properties.get("qa_evidence_role"))
+            if role == "false_positive":
+                evidence_id = QualityEvidenceService._string_value(properties.get("candidate_feature_id"))
+            elif role == "false_negative":
+                evidence_id = QualityEvidenceService._string_value(properties.get("reference_feature_id"))
+            else:
+                continue
+            review = review_index.get((role, evidence_id or ""))
+            properties.update(
+                {
+                    "review_decision": review.decision if review else "unreviewed",
+                    "review_notes": review.notes if review else None,
+                    "reviewed_by": review.reviewed_by if review else None,
+                    "reviewed_at": review.updated_at.isoformat() if review and review.updated_at else None,
+                }
+            )
+
     @staticmethod
     def _add_index_keys(index: dict[str, Any], row: Any) -> None:
         for key in QualityEvidenceService._row_identifiers(row):