geointel/backend/app/services/detection_review_service.py

from __future__ import annotations

from collections import Counter
from typing import Any
from uuid import UUID

from sqlalchemy.orm import Session

from app.core.errors import AppError
from app.services.reviewed_metrics_service import ReviewedMetricsService
from app.models import Detection, DetectionReview, QualityCheck, VectorFeature
from app.schemas.detection_review import (
    DetectionReviewList,
    DetectionReviewRead,
    DetectionReviewSummary,
    DetectionReviewUpsert,
)


class DetectionReviewService:
    ALLOWED_DECISIONS = {
        "false_positive": {
            "confirmed_model_false_positive",
            "reference_gap_or_change",
            "qa_alignment_mismatch",
            "uncertain",
            "unreviewed",
        },
        "false_negative": {
            "confirmed_model_false_negative",
            "reference_gap_or_change",
            "qa_alignment_mismatch",
            "imagery_obscured_or_uncertain",
            "uncertain",
            "unreviewed",
        },
    }

    @staticmethod
    def _quality_check(db: Session, project_id: UUID, quality_check_id: UUID) -> QualityCheck:
        quality_check = db.get(QualityCheck, quality_check_id)
        if not quality_check or quality_check.project_id != project_id:
            raise AppError(code="QUALITY_CHECK_NOT_FOUND", message="Quality check not found", status_code=404)
        if quality_check.check_type != "detections_vs_reference":
            raise AppError(
                code="DETECTION_REVIEW_UNSUPPORTED",
                message="Only persisted detection-versus-reference quality checks can be reviewed",
                status_code=422,
            )
        return quality_check

    @staticmethod
    def _evidence_items(quality_check: QualityCheck) -> list[dict[str, str]]:
        findings = quality_check.findings_json or {}
        items: list[dict[str, str]] = []
        for role, key, id_key in (
            ("false_positive", "false_positive_evidence", "candidate_feature_id"),
            ("false_negative", "false_negative_evidence", "reference_feature_id"),
        ):
            evidence_rows = findings.get(key)
            if not isinstance(evidence_rows, list):
                continue
            for evidence in evidence_rows:
                if not isinstance(evidence, dict):
                    continue
                value = str(evidence.get(id_key) or "").strip()
                if value:
                    items.append({"evidence_role": role, "evidence_feature_id": value})
        return items

    @staticmethod
    def _uuid(value: str) -> UUID | None:
        try:
            return UUID(value)
        except (TypeError, ValueError):
            return None

    @staticmethod
    def _review_index(db: Session, quality_check_id: UUID) -> dict[tuple[str, str], DetectionReview]:
        rows = db.query(DetectionReview).filter(DetectionReview.quality_check_id == quality_check_id).all()
        return {(row.evidence_role, row.evidence_feature_id): row for row in rows}

    @staticmethod
    def _summary(
        evidence: list[dict[str, str]],
        reviews: dict[tuple[str, str], DetectionReview],
        quality_check: QualityCheck | None = None,
    ) -> DetectionReviewSummary:
        evidence_keys = {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence}
        decisions = Counter(
            reviews[key].decision if key in reviews else "unreviewed"
            for key in evidence_keys
        )
        reviewed = sum(count for decision, count in decisions.items() if decision != "unreviewed")
        false_positive_total = sum(1 for item in evidence if item["evidence_role"] == "false_positive")
        false_negative_total = sum(1 for item in evidence if item["evidence_role"] == "false_negative")
        return DetectionReviewSummary(
            total=len(evidence),
            reviewed=reviewed,
            remaining=max(len(evidence) - reviewed, 0),
            false_positive_total=false_positive_total,
            false_negative_total=false_negative_total,
            decision_counts=dict(sorted(decisions.items())),
            reviewed_metrics=DetectionReviewService._reviewed_metrics(evidence_keys, reviews, quality_check),
        )

    @staticmethod
    def _reviewed_metrics(
        evidence_keys: set[tuple[str, str]],
        reviews: dict[tuple[str, str], DetectionReview],
        quality_check: QualityCheck | None,
    ) -> dict | None:
        """The score with the operator's verdicts applied.

        Without this the panel shows a precision the operator has already
        disproved: a false positive adjudicated as a reference gap is not the
        model's error, and the raw number keeps counting it as one.
        """

        if quality_check is None:
            return None
        findings = quality_check.findings_json if isinstance(quality_check.findings_json, dict) else {}
        matches = findings.get("matches")
        false_positives = findings.get("false_positives")
        false_negatives = findings.get("false_negatives")
        if not all(isinstance(value, int) for value in (matches, false_positives, false_negatives)):
            return None

        per_role: dict[str, Counter] = {"false_positive": Counter(), "false_negative": Counter()}
        for role, feature_id in evidence_keys:
            review = reviews.get((role, feature_id))
            if review is not None and role in per_role:
                per_role[role][review.decision] += 1

        return ReviewedMetricsService.adjudicate(
            matches=int(matches),
            false_positives=int(false_positives),
            false_negatives=int(false_negatives),
            false_positive_decisions=dict(per_role["false_positive"]),
            false_negative_decisions=dict(per_role["false_negative"]),
        )

    @staticmethod
    def _read_item(
        db: Session,
        quality_check: QualityCheck,
        evidence: dict[str, str],
        review: DetectionReview | None,
    ) -> DetectionReviewRead:
        role = evidence["evidence_role"]
        feature_id = evidence["evidence_feature_id"]
        feature_uuid = DetectionReviewService._uuid(feature_id)
        detection = db.get(Detection, feature_uuid) if role == "false_positive" and feature_uuid else None
        reference = db.get(VectorFeature, feature_uuid) if role == "false_negative" and feature_uuid else None
        return DetectionReviewRead(
            id=review.id if review else None,
            project_id=quality_check.project_id,
            quality_check_id=quality_check.id,
            analysis_run_id=quality_check.analysis_run_id,
            evidence_role=role,
            evidence_feature_id=feature_id,
            detection_id=detection.id if detection else review.detection_id if review else None,
            reference_feature_id=reference.id if reference else review.reference_feature_id if review else None,
            decision=review.decision if review else "unreviewed",
            notes=review.notes if review else None,
            reviewed_by=review.reviewed_by if review else None,
            confidence=detection.confidence if detection else None,
            class_name=(detection.class_name if detection else reference.feature_class if reference else None),
            source_tile_path=detection.source_tile_path if detection else None,
            created_at=review.created_at if review else None,
            updated_at=review.updated_at if review else None,
        )

    @staticmethod
    def list_reviews(
        db: Session,
        *,
        project_id: UUID,
        quality_check_id: UUID,
        evidence_role: str | None = None,
        decision: str | None = None,
        reviewed: bool | None = None,
        limit: int = 50,
        offset: int = 0,
    ) -> DetectionReviewList:
        quality_check = DetectionReviewService._quality_check(db, project_id, quality_check_id)
        evidence = DetectionReviewService._evidence_items(quality_check)
        reviews = DetectionReviewService._review_index(db, quality_check_id)
        filtered = [item for item in evidence if evidence_role is None or item["evidence_role"] == evidence_role]
        if decision is not None:
            filtered = [
                item
                for item in filtered
                if (reviews.get((item["evidence_role"], item["evidence_feature_id"])).decision
                    if reviews.get((item["evidence_role"], item["evidence_feature_id"]))
                    else "unreviewed")
                == decision
            ]
        if reviewed is not None:
            filtered = [
                item
                for item in filtered
                if (
                    (reviews.get((item["evidence_role"], item["evidence_feature_id"])).decision
                     if reviews.get((item["evidence_role"], item["evidence_feature_id"]))
                     else "unreviewed")
                    != "unreviewed"
                )
                == reviewed
            ]
        page = filtered[offset : offset + limit]
        return DetectionReviewList(
            items=[
                DetectionReviewService._read_item(
                    db,
                    quality_check,
                    item,
                    reviews.get((item["evidence_role"], item["evidence_feature_id"])),
                )
                for item in page
            ],
            total=len(filtered),
            limit=limit,
            offset=offset,
            summary=DetectionReviewService._summary(evidence, reviews, quality_check),
        )

    @staticmethod
    def upsert_review(
        db: Session,
        *,
        project_id: UUID,
        quality_check_id: UUID,
        payload: DetectionReviewUpsert,
    ) -> DetectionReviewRead:
        quality_check = DetectionReviewService._quality_check(db, project_id, quality_check_id)
        if payload.decision not in DetectionReviewService.ALLOWED_DECISIONS[payload.evidence_role]:
            raise AppError(
                code="INVALID_DETECTION_REVIEW_DECISION",
                message="The review decision is not valid for this evidence role",
                details={"evidence_role": payload.evidence_role, "decision": payload.decision},
                status_code=422,
            )
        evidence = DetectionReviewService._evidence_items(quality_check)
        evidence_key = (payload.evidence_role, payload.evidence_feature_id)
        if evidence_key not in {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence}:
            raise AppError(
                code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND",
                message="The evidence feature does not belong to this quality check",
                status_code=404,
            )
        feature_uuid = DetectionReviewService._uuid(payload.evidence_feature_id)
        detection = db.get(Detection, feature_uuid) if payload.evidence_role == "false_positive" and feature_uuid else None
        reference = db.get(VectorFeature, feature_uuid) if payload.evidence_role == "false_negative" and feature_uuid else None
        if payload.evidence_role == "false_positive" and (not detection or detection.analysis_run_id != quality_check.analysis_run_id):
            raise AppError(code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND", message="Persisted detection evidence was not found", status_code=404)
        if payload.evidence_role == "false_negative" and (not reference or reference.dataset_id != quality_check.reference_dataset_id):
            raise AppError(code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND", message="Persisted reference evidence was not found", status_code=404)

        review = (
            db.query(DetectionReview)
            .filter(
                DetectionReview.quality_check_id == quality_check_id,
                DetectionReview.evidence_role == payload.evidence_role,
                DetectionReview.evidence_feature_id == payload.evidence_feature_id,
            )
            .first()
        )
        if review is None:
            review = DetectionReview(
                project_id=project_id,
                quality_check_id=quality_check_id,
                analysis_run_id=quality_check.analysis_run_id,
                evidence_role=payload.evidence_role,
                evidence_feature_id=payload.evidence_feature_id,
                detection_id=detection.id if detection else None,
                reference_feature_id=reference.id if reference else None,
                decision=payload.decision,
                notes=payload.notes.strip() if payload.notes and payload.notes.strip() else None,
                reviewed_by=payload.reviewed_by.strip(),
            )
        else:
            review.decision = payload.decision
            review.notes = payload.notes.strip() if payload.notes and payload.notes.strip() else None
            review.reviewed_by = payload.reviewed_by.strip()
        db.add(review)
        db.commit()
        db.refresh(review)
        return DetectionReviewService._read_item(
            db,
            quality_check,
            {"evidence_role": payload.evidence_role, "evidence_feature_id": payload.evidence_feature_id},
            review,
        )