294 lines
13 KiB
Python
294 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
from collections import Counter
|
|
from uuid import UUID
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.core.errors import AppError
|
|
from app.services.reviewed_metrics_service import ReviewedMetricsService
|
|
from app.models import Detection, DetectionReview, QualityCheck, VectorFeature
|
|
from app.schemas.detection_review import (
|
|
DetectionReviewList,
|
|
DetectionReviewRead,
|
|
DetectionReviewSummary,
|
|
DetectionReviewUpsert,
|
|
)
|
|
|
|
|
|
class DetectionReviewService:
|
|
ALLOWED_DECISIONS = {
|
|
"false_positive": {
|
|
"confirmed_model_false_positive",
|
|
"reference_gap_or_change",
|
|
"qa_alignment_mismatch",
|
|
"uncertain",
|
|
"unreviewed",
|
|
},
|
|
"false_negative": {
|
|
"confirmed_model_false_negative",
|
|
"reference_gap_or_change",
|
|
"qa_alignment_mismatch",
|
|
"imagery_obscured_or_uncertain",
|
|
"uncertain",
|
|
"unreviewed",
|
|
},
|
|
}
|
|
|
|
@staticmethod
|
|
def _quality_check(db: Session, project_id: UUID, quality_check_id: UUID) -> QualityCheck:
|
|
quality_check = db.get(QualityCheck, quality_check_id)
|
|
if not quality_check or quality_check.project_id != project_id:
|
|
raise AppError(code="QUALITY_CHECK_NOT_FOUND", message="Quality check not found", status_code=404)
|
|
if quality_check.check_type != "detections_vs_reference":
|
|
raise AppError(
|
|
code="DETECTION_REVIEW_UNSUPPORTED",
|
|
message="Only persisted detection-versus-reference quality checks can be reviewed",
|
|
status_code=422,
|
|
)
|
|
return quality_check
|
|
|
|
@staticmethod
|
|
def _evidence_items(quality_check: QualityCheck) -> list[dict[str, str]]:
|
|
findings = quality_check.findings_json or {}
|
|
items: list[dict[str, str]] = []
|
|
for role, key, id_key in (
|
|
("false_positive", "false_positive_evidence", "candidate_feature_id"),
|
|
("false_negative", "false_negative_evidence", "reference_feature_id"),
|
|
):
|
|
evidence_rows = findings.get(key)
|
|
if not isinstance(evidence_rows, list):
|
|
continue
|
|
for evidence in evidence_rows:
|
|
if not isinstance(evidence, dict):
|
|
continue
|
|
value = str(evidence.get(id_key) or "").strip()
|
|
if value:
|
|
items.append({"evidence_role": role, "evidence_feature_id": value})
|
|
return items
|
|
|
|
@staticmethod
|
|
def _uuid(value: str) -> UUID | None:
|
|
try:
|
|
return UUID(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
|
|
@staticmethod
|
|
def _review_index(db: Session, quality_check_id: UUID) -> dict[tuple[str, str], DetectionReview]:
|
|
rows = db.query(DetectionReview).filter(DetectionReview.quality_check_id == quality_check_id).all()
|
|
return {(row.evidence_role, row.evidence_feature_id): row for row in rows}
|
|
|
|
@staticmethod
|
|
def _summary(
|
|
evidence: list[dict[str, str]],
|
|
reviews: dict[tuple[str, str], DetectionReview],
|
|
quality_check: QualityCheck | None = None,
|
|
) -> DetectionReviewSummary:
|
|
evidence_keys = {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence}
|
|
decisions = Counter(
|
|
reviews[key].decision if key in reviews else "unreviewed"
|
|
for key in evidence_keys
|
|
)
|
|
reviewed = sum(count for decision, count in decisions.items() if decision != "unreviewed")
|
|
false_positive_total = sum(1 for item in evidence if item["evidence_role"] == "false_positive")
|
|
false_negative_total = sum(1 for item in evidence if item["evidence_role"] == "false_negative")
|
|
return DetectionReviewSummary(
|
|
total=len(evidence),
|
|
reviewed=reviewed,
|
|
remaining=max(len(evidence) - reviewed, 0),
|
|
false_positive_total=false_positive_total,
|
|
false_negative_total=false_negative_total,
|
|
decision_counts=dict(sorted(decisions.items())),
|
|
reviewed_metrics=DetectionReviewService._reviewed_metrics(evidence_keys, reviews, quality_check),
|
|
)
|
|
|
|
@staticmethod
|
|
def _reviewed_metrics(
|
|
evidence_keys: set[tuple[str, str]],
|
|
reviews: dict[tuple[str, str], DetectionReview],
|
|
quality_check: QualityCheck | None,
|
|
) -> dict | None:
|
|
"""The score with the operator's verdicts applied.
|
|
|
|
Without this the panel shows a precision the operator has already
|
|
disproved: a false positive adjudicated as a reference gap is not the
|
|
model's error, and the raw number keeps counting it as one.
|
|
"""
|
|
|
|
if quality_check is None:
|
|
return None
|
|
findings = quality_check.findings_json if isinstance(quality_check.findings_json, dict) else {}
|
|
matches = findings.get("matches")
|
|
false_positives = findings.get("false_positives")
|
|
false_negatives = findings.get("false_negatives")
|
|
if not all(isinstance(value, int) for value in (matches, false_positives, false_negatives)):
|
|
return None
|
|
|
|
per_role: dict[str, Counter] = {"false_positive": Counter(), "false_negative": Counter()}
|
|
for role, feature_id in evidence_keys:
|
|
review = reviews.get((role, feature_id))
|
|
if review is not None and role in per_role:
|
|
per_role[role][review.decision] += 1
|
|
|
|
return ReviewedMetricsService.adjudicate(
|
|
matches=int(matches),
|
|
false_positives=int(false_positives),
|
|
false_negatives=int(false_negatives),
|
|
false_positive_decisions=dict(per_role["false_positive"]),
|
|
false_negative_decisions=dict(per_role["false_negative"]),
|
|
)
|
|
|
|
@staticmethod
|
|
def _read_item(
|
|
db: Session,
|
|
quality_check: QualityCheck,
|
|
evidence: dict[str, str],
|
|
review: DetectionReview | None,
|
|
) -> DetectionReviewRead:
|
|
role = evidence["evidence_role"]
|
|
feature_id = evidence["evidence_feature_id"]
|
|
feature_uuid = DetectionReviewService._uuid(feature_id)
|
|
detection = db.get(Detection, feature_uuid) if role == "false_positive" and feature_uuid else None
|
|
reference = db.get(VectorFeature, feature_uuid) if role == "false_negative" and feature_uuid else None
|
|
return DetectionReviewRead(
|
|
id=review.id if review else None,
|
|
project_id=quality_check.project_id,
|
|
quality_check_id=quality_check.id,
|
|
analysis_run_id=quality_check.analysis_run_id,
|
|
evidence_role=role,
|
|
evidence_feature_id=feature_id,
|
|
detection_id=detection.id if detection else review.detection_id if review else None,
|
|
reference_feature_id=reference.id if reference else review.reference_feature_id if review else None,
|
|
decision=review.decision if review else "unreviewed",
|
|
notes=review.notes if review else None,
|
|
reviewed_by=review.reviewed_by if review else None,
|
|
confidence=detection.confidence if detection else None,
|
|
class_name=(detection.class_name if detection else reference.feature_class if reference else None),
|
|
source_tile_path=detection.source_tile_path if detection else None,
|
|
created_at=review.created_at if review else None,
|
|
updated_at=review.updated_at if review else None,
|
|
)
|
|
|
|
@staticmethod
|
|
def list_reviews(
|
|
db: Session,
|
|
*,
|
|
project_id: UUID,
|
|
quality_check_id: UUID,
|
|
evidence_role: str | None = None,
|
|
decision: str | None = None,
|
|
reviewed: bool | None = None,
|
|
limit: int = 50,
|
|
offset: int = 0,
|
|
) -> DetectionReviewList:
|
|
quality_check = DetectionReviewService._quality_check(db, project_id, quality_check_id)
|
|
evidence = DetectionReviewService._evidence_items(quality_check)
|
|
reviews = DetectionReviewService._review_index(db, quality_check_id)
|
|
filtered = [item for item in evidence if evidence_role is None or item["evidence_role"] == evidence_role]
|
|
if decision is not None:
|
|
filtered = [
|
|
item
|
|
for item in filtered
|
|
if (reviews.get((item["evidence_role"], item["evidence_feature_id"])).decision
|
|
if reviews.get((item["evidence_role"], item["evidence_feature_id"]))
|
|
else "unreviewed")
|
|
== decision
|
|
]
|
|
if reviewed is not None:
|
|
filtered = [
|
|
item
|
|
for item in filtered
|
|
if (
|
|
(reviews.get((item["evidence_role"], item["evidence_feature_id"])).decision
|
|
if reviews.get((item["evidence_role"], item["evidence_feature_id"]))
|
|
else "unreviewed")
|
|
!= "unreviewed"
|
|
)
|
|
== reviewed
|
|
]
|
|
page = filtered[offset : offset + limit]
|
|
return DetectionReviewList(
|
|
items=[
|
|
DetectionReviewService._read_item(
|
|
db,
|
|
quality_check,
|
|
item,
|
|
reviews.get((item["evidence_role"], item["evidence_feature_id"])),
|
|
)
|
|
for item in page
|
|
],
|
|
total=len(filtered),
|
|
limit=limit,
|
|
offset=offset,
|
|
summary=DetectionReviewService._summary(evidence, reviews, quality_check),
|
|
)
|
|
|
|
@staticmethod
|
|
def upsert_review(
|
|
db: Session,
|
|
*,
|
|
project_id: UUID,
|
|
quality_check_id: UUID,
|
|
payload: DetectionReviewUpsert,
|
|
) -> DetectionReviewRead:
|
|
quality_check = DetectionReviewService._quality_check(db, project_id, quality_check_id)
|
|
if payload.decision not in DetectionReviewService.ALLOWED_DECISIONS[payload.evidence_role]:
|
|
raise AppError(
|
|
code="INVALID_DETECTION_REVIEW_DECISION",
|
|
message="The review decision is not valid for this evidence role",
|
|
details={"evidence_role": payload.evidence_role, "decision": payload.decision},
|
|
status_code=422,
|
|
)
|
|
evidence = DetectionReviewService._evidence_items(quality_check)
|
|
evidence_key = (payload.evidence_role, payload.evidence_feature_id)
|
|
if evidence_key not in {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence}:
|
|
raise AppError(
|
|
code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND",
|
|
message="The evidence feature does not belong to this quality check",
|
|
status_code=404,
|
|
)
|
|
feature_uuid = DetectionReviewService._uuid(payload.evidence_feature_id)
|
|
detection = db.get(Detection, feature_uuid) if payload.evidence_role == "false_positive" and feature_uuid else None
|
|
reference = db.get(VectorFeature, feature_uuid) if payload.evidence_role == "false_negative" and feature_uuid else None
|
|
if payload.evidence_role == "false_positive" and (not detection or detection.analysis_run_id != quality_check.analysis_run_id):
|
|
raise AppError(code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND", message="Persisted detection evidence was not found", status_code=404)
|
|
if payload.evidence_role == "false_negative" and (not reference or reference.dataset_id != quality_check.reference_dataset_id):
|
|
raise AppError(code="DETECTION_REVIEW_EVIDENCE_NOT_FOUND", message="Persisted reference evidence was not found", status_code=404)
|
|
|
|
review = (
|
|
db.query(DetectionReview)
|
|
.filter(
|
|
DetectionReview.quality_check_id == quality_check_id,
|
|
DetectionReview.evidence_role == payload.evidence_role,
|
|
DetectionReview.evidence_feature_id == payload.evidence_feature_id,
|
|
)
|
|
.first()
|
|
)
|
|
if review is None:
|
|
review = DetectionReview(
|
|
project_id=project_id,
|
|
quality_check_id=quality_check_id,
|
|
analysis_run_id=quality_check.analysis_run_id,
|
|
evidence_role=payload.evidence_role,
|
|
evidence_feature_id=payload.evidence_feature_id,
|
|
detection_id=detection.id if detection else None,
|
|
reference_feature_id=reference.id if reference else None,
|
|
decision=payload.decision,
|
|
notes=payload.notes.strip() if payload.notes and payload.notes.strip() else None,
|
|
reviewed_by=payload.reviewed_by.strip(),
|
|
)
|
|
else:
|
|
review.decision = payload.decision
|
|
review.notes = payload.notes.strip() if payload.notes and payload.notes.strip() else None
|
|
review.reviewed_by = payload.reviewed_by.strip()
|
|
db.add(review)
|
|
db.commit()
|
|
db.refresh(review)
|
|
return DetectionReviewService._read_item(
|
|
db,
|
|
quality_check,
|
|
{"evidence_role": payload.evidence_role, "evidence_feature_id": payload.evidence_feature_id},
|
|
review,
|
|
)
|