diff --git a/backend/app/schemas/detection_review.py b/backend/app/schemas/detection_review.py index 82e4ee7f..06016b4f 100644 --- a/backend/app/schemas/detection_review.py +++ b/backend/app/schemas/detection_review.py @@ -53,6 +53,10 @@ class DetectionReviewSummary(BaseModel): false_positive_total: int false_negative_total: int decision_counts: dict[str, int] + # The score with the operator's verdicts applied, next to the raw one. A + # finding adjudicated as a reference gap is not the model's error, and an + # interval covers what the unreviewed remainder could still turn out to be. + reviewed_metrics: dict | None = None class DetectionReviewList(BaseModel): diff --git a/backend/app/services/detection_review_service.py b/backend/app/services/detection_review_service.py index 8c836323..a3b9bcc5 100644 --- a/backend/app/services/detection_review_service.py +++ b/backend/app/services/detection_review_service.py @@ -7,6 +7,7 @@ from uuid import UUID from sqlalchemy.orm import Session from app.core.errors import AppError +from app.services.reviewed_metrics_service import ReviewedMetricsService from app.models import Detection, DetectionReview, QualityCheck, VectorFeature from app.schemas.detection_review import ( DetectionReviewList, @@ -80,7 +81,11 @@ class DetectionReviewService: return {(row.evidence_role, row.evidence_feature_id): row for row in rows} @staticmethod - def _summary(evidence: list[dict[str, str]], reviews: dict[tuple[str, str], DetectionReview]) -> DetectionReviewSummary: + def _summary( + evidence: list[dict[str, str]], + reviews: dict[tuple[str, str], DetectionReview], + quality_check: QualityCheck | None = None, + ) -> DetectionReviewSummary: evidence_keys = {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence} decisions = Counter( reviews[key].decision if key in reviews else "unreviewed" @@ -96,6 +101,43 @@ class DetectionReviewService: false_positive_total=false_positive_total, false_negative_total=false_negative_total, decision_counts=dict(sorted(decisions.items())), + reviewed_metrics=DetectionReviewService._reviewed_metrics(evidence_keys, reviews, quality_check), + ) + + @staticmethod + def _reviewed_metrics( + evidence_keys: set[tuple[str, str]], + reviews: dict[tuple[str, str], DetectionReview], + quality_check: QualityCheck | None, + ) -> dict | None: + """The score with the operator's verdicts applied. + + Without this the panel shows a precision the operator has already + disproved: a false positive adjudicated as a reference gap is not the + model's error, and the raw number keeps counting it as one. + """ + + if quality_check is None: + return None + findings = quality_check.findings_json if isinstance(quality_check.findings_json, dict) else {} + matches = findings.get("matches") + false_positives = findings.get("false_positives") + false_negatives = findings.get("false_negatives") + if not all(isinstance(value, int) for value in (matches, false_positives, false_negatives)): + return None + + per_role: dict[str, Counter] = {"false_positive": Counter(), "false_negative": Counter()} + for role, feature_id in evidence_keys: + review = reviews.get((role, feature_id)) + if review is not None and role in per_role: + per_role[role][review.decision] += 1 + + return ReviewedMetricsService.adjudicate( + matches=int(matches), + false_positives=int(false_positives), + false_negatives=int(false_negatives), + false_positive_decisions=dict(per_role["false_positive"]), + false_negative_decisions=dict(per_role["false_negative"]), ) @staticmethod @@ -180,7 +222,7 @@ class DetectionReviewService: total=len(filtered), limit=limit, offset=offset, - summary=DetectionReviewService._summary(evidence, reviews), + summary=DetectionReviewService._summary(evidence, reviews, quality_check), ) @staticmethod diff --git a/backend/app/services/reviewed_metrics_service.py b/backend/app/services/reviewed_metrics_service.py new file mode 100644 index 00000000..09bd05be --- /dev/null +++ b/backend/app/services/reviewed_metrics_service.py @@ -0,0 +1,112 @@ +"""Detection metrics after an operator has adjudicated the evidence. + +The review vocabulary already separates a model error from a reference gap, +because the product's position is that official footprints are not +automatically perfect ground truth. Until now those verdicts were only counted: +an operator who established that twelve of forty false positives are buildings +the reference simply lacks still saw a precision counting all forty against the +model — a number they had personally disproved. + +Applying the verdicts gives an adjudicated score. Because part of the evidence +is usually still unreviewed, the honest form is an interval: pessimistic +assumes every unreviewed item is a model error, optimistic assumes none is. The +raw score stays reported alongside, so nothing is quietly improved. +""" + +from __future__ import annotations + +from typing import Any + +# A verdict that the finding was not the model's fault. The detection (or the +# missing detection) was right; the reference or the matching rule was not. +EXONERATING_DECISIONS = frozenset({"reference_gap_or_change", "qa_alignment_mismatch"}) + +# Verdicts that confirm the finding, and verdicts that reach no conclusion. +# Both keep counting: being unable to judge is not evidence in the model's +# favour, and treating it as such is how a score drifts upward unearned. +CONFIRMING_DECISIONS = frozenset( + {"confirmed_model_false_positive", "confirmed_model_false_negative"} +) +INCONCLUSIVE_DECISIONS = frozenset({"uncertain", "imagery_obscured_or_uncertain"}) + + +def _score(matches: int, false_positives: int, false_negatives: int) -> dict[str, Any]: + precision = matches / (matches + false_positives) if matches + false_positives > 0 else None + recall = matches / (matches + false_negatives) if matches + false_negatives > 0 else None + f1_score = None + if precision is not None and recall is not None: + f1_score = (2 * precision * recall / (precision + recall)) if precision + recall > 0 else 0.0 + return { + "matches": matches, + "false_positives": false_positives, + "false_negatives": false_negatives, + "precision": precision, + "recall": recall, + "f1_score": f1_score, + } + + +class ReviewedMetricsService: + @staticmethod + def _adjudicate_role(total: int, decisions: dict[str, int]) -> tuple[int, int, int]: + """Split a finding count into exonerated, confirmed and unreviewed. + + A decision the runtime does not recognise — one from a later release — + counts as no judgement rather than as an exoneration. + """ + + exonerated = sum(count for name, count in decisions.items() if name in EXONERATING_DECISIONS) + judged = sum( + count + for name, count in decisions.items() + if name in EXONERATING_DECISIONS | CONFIRMING_DECISIONS | INCONCLUSIVE_DECISIONS + ) + exonerated = min(exonerated, total) + judged = min(judged, total) + return exonerated, judged - exonerated, max(0, total - judged) + + @staticmethod + def adjudicate( + *, + matches: int, + false_positives: int, + false_negatives: int, + false_positive_decisions: dict[str, int], + false_negative_decisions: dict[str, int], + ) -> dict[str, Any]: + """Apply operator verdicts to a quality check's counts.""" + + fp_exonerated, _fp_confirmed, fp_unreviewed = ReviewedMetricsService._adjudicate_role( + false_positives, false_positive_decisions + ) + fn_exonerated, _fn_confirmed, fn_unreviewed = ReviewedMetricsService._adjudicate_role( + false_negatives, false_negative_decisions + ) + + adjudicated_fp = false_positives - fp_exonerated + adjudicated_fn = false_negatives - fn_exonerated + + # The interval covers what the unreviewed remainder could still turn + # out to be, so a partly reviewed check never reads as a settled one. + pessimistic = _score(matches, adjudicated_fp, adjudicated_fn) + optimistic = _score(matches, adjudicated_fp - fp_unreviewed, adjudicated_fn - fn_unreviewed) + + return { + "raw": _score(matches, false_positives, false_negatives), + "adjudicated": pessimistic, + "pessimistic": pessimistic, + "optimistic": optimistic, + "review_complete": fp_unreviewed == 0 and fn_unreviewed == 0, + "false_positive_breakdown": { + "total": false_positives, + "exonerated": fp_exonerated, + "confirmed_or_inconclusive": false_positives - fp_exonerated - fp_unreviewed, + "unreviewed": fp_unreviewed, + }, + "false_negative_breakdown": { + "total": false_negatives, + "exonerated": fn_exonerated, + "confirmed_or_inconclusive": false_negatives - fn_exonerated - fn_unreviewed, + "unreviewed": fn_unreviewed, + }, + } diff --git a/backend/tests/test_reviewed_detection_metrics.py b/backend/tests/test_reviewed_detection_metrics.py new file mode 100644 index 00000000..2ff122ab --- /dev/null +++ b/backend/tests/test_reviewed_detection_metrics.py @@ -0,0 +1,254 @@ +"""An operator's adjudication must reach the score. + +The review vocabulary already distinguishes a model error from a reference gap +— the product's own position is that official footprints are not automatically +perfect ground truth. But the reviews were only counted. An operator who +inspects forty false positives and establishes that twelve are buildings the +reference simply lacks still sees a precision that counts all forty against the +model, and that they have personally disproved. + +Because part of the evidence is usually still unreviewed, the honest answer is +an interval, not a single corrected number: pessimistic assumes every +unreviewed item is a model error, optimistic assumes none is. +""" + +from __future__ import annotations + +import pytest + +from app.services.reviewed_metrics_service import ReviewedMetricsService + + +def _counts(**decisions: int) -> dict[str, int]: + return decisions + + +class TestAdjudication: + def test_a_reference_gap_stops_counting_against_precision(self) -> None: + result = ReviewedMetricsService.adjudicate( + matches=80, + false_positives=20, + false_negatives=10, + false_positive_decisions=_counts(reference_gap_or_change=20), + false_negative_decisions={}, + ) + + # Every false positive was the reference missing a real building. + assert result["adjudicated"]["false_positives"] == 0 + assert result["adjudicated"]["precision"] == pytest.approx(1.0) + + def test_a_confirmed_model_error_keeps_counting(self) -> None: + result = ReviewedMetricsService.adjudicate( + matches=80, + false_positives=20, + false_negatives=0, + false_positive_decisions=_counts(confirmed_model_false_positive=20), + false_negative_decisions={}, + ) + + assert result["adjudicated"]["false_positives"] == 20 + assert result["adjudicated"]["precision"] == pytest.approx(0.8) + + def test_an_alignment_mismatch_is_not_a_model_error(self) -> None: + """Both the detection and the footprint were right; the matching failed.""" + + result = ReviewedMetricsService.adjudicate( + matches=80, + false_positives=20, + false_negatives=0, + false_positive_decisions=_counts(qa_alignment_mismatch=20), + false_negative_decisions={}, + ) + + assert result["adjudicated"]["false_positives"] == 0 + + def test_a_reference_gap_on_a_miss_stops_counting_against_recall(self) -> None: + result = ReviewedMetricsService.adjudicate( + matches=80, + false_positives=0, + false_negatives=20, + false_positive_decisions={}, + false_negative_decisions=_counts(reference_gap_or_change=20), + ) + + # The reference held twenty footprints that are not there. + assert result["adjudicated"]["false_negatives"] == 0 + assert result["adjudicated"]["recall"] == pytest.approx(1.0) + + def test_an_uncertain_verdict_keeps_counting_against_the_model(self) -> None: + """Not being able to judge is not evidence in the model's favour.""" + + result = ReviewedMetricsService.adjudicate( + matches=80, + false_positives=20, + false_negatives=0, + false_positive_decisions=_counts(uncertain=10, imagery_obscured_or_uncertain=10), + false_negative_decisions={}, + ) + + assert result["adjudicated"]["false_positives"] == 20 + + +class TestBounds: + def test_a_partly_reviewed_check_reports_an_interval(self) -> None: + result = ReviewedMetricsService.adjudicate( + matches=80, + false_positives=20, + false_negatives=0, + false_positive_decisions=_counts(reference_gap_or_change=10), + false_negative_decisions={}, + ) + + # Ten unreviewed: pessimistically all model errors, optimistically none. + assert result["pessimistic"]["precision"] == pytest.approx(80 / 90) + assert result["optimistic"]["precision"] == pytest.approx(1.0) + assert result["review_complete"] is False + + def test_a_fully_reviewed_check_collapses_the_interval(self) -> None: + result = ReviewedMetricsService.adjudicate( + matches=80, + false_positives=20, + false_negatives=5, + false_positive_decisions=_counts(reference_gap_or_change=12, confirmed_model_false_positive=8), + false_negative_decisions=_counts(confirmed_model_false_negative=5), + ) + + assert result["review_complete"] is True + assert result["pessimistic"]["precision"] == pytest.approx(result["optimistic"]["precision"]) + assert result["adjudicated"]["precision"] == pytest.approx(80 / 88) + + def test_an_unreviewed_check_reports_the_raw_numbers_unchanged(self) -> None: + result = ReviewedMetricsService.adjudicate( + matches=80, + false_positives=20, + false_negatives=10, + false_positive_decisions={}, + false_negative_decisions={}, + ) + + assert result["review_complete"] is False + assert result["adjudicated"]["precision"] == pytest.approx(result["raw"]["precision"]) + assert result["adjudicated"]["recall"] == pytest.approx(result["raw"]["recall"]) + + def test_the_raw_score_is_always_reported_alongside(self) -> None: + result = ReviewedMetricsService.adjudicate( + matches=80, + false_positives=20, + false_negatives=0, + false_positive_decisions=_counts(reference_gap_or_change=20), + false_negative_decisions={}, + ) + + assert result["raw"]["precision"] == pytest.approx(0.8) + assert result["adjudicated"]["precision"] == pytest.approx(1.0) + + +class TestEdges: + def test_a_check_without_findings_makes_no_claim(self) -> None: + result = ReviewedMetricsService.adjudicate( + matches=0, + false_positives=0, + false_negatives=0, + false_positive_decisions={}, + false_negative_decisions={}, + ) + + assert result["adjudicated"]["precision"] is None + assert result["adjudicated"]["recall"] is None + assert result["review_complete"] is True + + def test_more_decisions_than_findings_cannot_invent_a_negative_count(self) -> None: + result = ReviewedMetricsService.adjudicate( + matches=10, + false_positives=2, + false_negatives=0, + false_positive_decisions=_counts(reference_gap_or_change=99), + false_negative_decisions={}, + ) + + assert result["adjudicated"]["false_positives"] == 0 + + def test_an_unknown_decision_is_treated_as_no_judgement(self) -> None: + result = ReviewedMetricsService.adjudicate( + matches=10, + false_positives=5, + false_negatives=0, + false_positive_decisions=_counts(something_new_from_a_later_release=5), + false_negative_decisions={}, + ) + + assert result["adjudicated"]["false_positives"] == 5 + assert result["review_complete"] is False + + +class TestThroughTheReviewPanel: + """The score the panel shows, not just the arithmetic behind it.""" + + def _quality_check(self, quality_check_id, project_id): + from app.models import QualityCheck + + return QualityCheck( + id=quality_check_id, + project_id=project_id, + reference_dataset_id=__import__("uuid").uuid4(), + check_type="detections_vs_reference", + status="ok", + findings_json={ + "matches": 80, + "false_positives": 20, + "false_negatives": 0, + "false_positive_evidence": [ + {"candidate_feature_id": f"detection-{index}"} for index in range(20) + ], + "false_negative_evidence": [], + }, + ) + + def test_adjudicated_reference_gaps_raise_the_reported_precision(self) -> None: + import uuid + + from app.models import DetectionReview, QualityCheck + from app.services.detection_review_service import DetectionReviewService + + quality_check_id, project_id = uuid.uuid4(), uuid.uuid4() + quality_check = self._quality_check(quality_check_id, project_id) + reviews = [ + DetectionReview( + id=uuid.uuid4(), + quality_check_id=quality_check_id, + evidence_role="false_positive", + evidence_feature_id=f"detection-{index}", + decision="reference_gap_or_change", + ) + for index in range(12) + ] + + class _Query: + def __init__(self, rows): + self.rows = rows + + def filter(self, *_args): + return self + + def all(self): + return self.rows + + class _Session: + def get(self, model, item_id): + return quality_check if model is QualityCheck and item_id == quality_check_id else None + + def query(self, _model): + return _Query(reviews) + + result = DetectionReviewService.list_reviews( + _Session(), project_id=project_id, quality_check_id=quality_check_id + ) + metrics = result.summary.reviewed_metrics + + assert metrics is not None + assert metrics["raw"]["precision"] == pytest.approx(0.8) + # Twelve of the twenty were the reference missing a building. + assert metrics["adjudicated"]["precision"] == pytest.approx(80 / 88) + assert metrics["review_complete"] is False + assert metrics["false_positive_breakdown"]["exonerated"] == 12 + assert metrics["false_positive_breakdown"]["unreviewed"] == 8 diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index 2748a634..7e5b5acd 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -1850,6 +1850,32 @@ Each feature includes: Returns persisted detections for a dataset as a GeoJSON FeatureCollection. Optional filters match the detection list endpoint. +### GET `/api/v1/projects/{project_id}/quality-checks/{id}/reviews` + +Returns the evidence queue plus a `summary`, which now carries +`reviewed_metrics`: the score with the operator's verdicts applied, next to the +raw one. + +The review vocabulary already separates a model error from a reference gap, +because official footprints are not automatically perfect ground truth. Those +verdicts were only counted, so an operator who established that twelve of forty +false positives are buildings the reference simply lacks still saw a precision +counting all forty against the model — a number they had personally disproved. + +- `reference_gap_or_change` and `qa_alignment_mismatch` exonerate a finding: the + detection, or the missing detection, was not the model's error. +- `confirmed_model_false_positive` / `confirmed_model_false_negative` keep it. +- `uncertain` and `imagery_obscured_or_uncertain` also keep it. Being unable to + judge is not evidence in the model's favour, and treating it as such is how a + score drifts upward unearned. A decision from a later release the runtime does + not recognise is likewise treated as no judgement. + +Because part of the evidence is usually unreviewed, the result is an interval: +`pessimistic` assumes every unreviewed finding is a model error, `optimistic` +assumes none is, and `adjudicated` equals the pessimistic reading so a partly +reviewed check never presents as a settled one. `review_complete` says whether +the interval has collapsed. + ### POST `/api/v1/detection/runs/compare` Scores several persisted runs against one reference and ranks them on average diff --git a/frontend/src/components/quality/DetectionReviewPanel.tsx b/frontend/src/components/quality/DetectionReviewPanel.tsx index f2bf9f4c..a2bbe6b9 100644 --- a/frontend/src/components/quality/DetectionReviewPanel.tsx +++ b/frontend/src/components/quality/DetectionReviewPanel.tsx @@ -51,6 +51,10 @@ function shortId(value: string): string { return value.length > 18 ? `${value.slice(0, 8)}...${value.slice(-6)}` : value } +function formatScore(value: number | null | undefined): string { + return typeof value === 'number' && Number.isFinite(value) ? value.toFixed(3) : 'n.v.t.' +} + export function DetectionReviewPanel({ projectId, qualityCheckId, @@ -132,6 +136,30 @@ export function DetectionReviewPanel({ ) : null} + {queue?.summary.reviewed_metrics ? ( +
+
+
+ Precisie zoals gemeten + {formatScore(queue.summary.reviewed_metrics.raw.precision)} +
+
+ Precisie na uw beoordeling + {formatScore(queue.summary.reviewed_metrics.adjudicated.precision)} +
+
+ Herkenningsgraad na beoordeling + {formatScore(queue.summary.reviewed_metrics.adjudicated.recall)} +
+
+

+ {queue.summary.reviewed_metrics.review_complete + ? `Alle bevindingen zijn beoordeeld. ${queue.summary.reviewed_metrics.false_positive_breakdown.exonerated} onterecht gevonden objecten en ${queue.summary.reviewed_metrics.false_negative_breakdown.exonerated} gemiste objecten bleken een hiaat in de referentielaag, niet een modelfout.` + : `Nog ${queue.summary.reviewed_metrics.false_positive_breakdown.unreviewed + queue.summary.reviewed_metrics.false_negative_breakdown.unreviewed} bevindingen onbeoordeeld. De precisie ligt tussen ${formatScore(queue.summary.reviewed_metrics.pessimistic.precision)} en ${formatScore(queue.summary.reviewed_metrics.optimistic.precision)}; de getoonde waarde rekent elke onbeoordeelde bevinding nog als modelfout.`} +

+
+ ) : null} +