let an operator's adjudication reach the score

The review vocabulary already separates a model error from a reference gap,
because the product's position is that official footprints are not
automatically perfect ground truth. Those verdicts were only counted. An
operator who inspected forty false positives and established that twelve are
buildings the reference simply lacks still saw a precision counting all forty
against the model — a number they had personally disproved, on the panel where
they disproved it.

Applying the verdicts gives an adjudicated score reported next to the raw one,
so nothing is quietly improved. Not being able to judge is not evidence in the
model's favour, so uncertain and obscured verdicts keep counting, as does a
decision from a later release that this runtime does not recognise.

Because part of the evidence is usually still unreviewed, the honest form is an
interval rather than a single corrected number: pessimistic assumes every
unreviewed finding is a model error, optimistic assumes none is, and the
headline equals the pessimistic reading so a partly reviewed check never
presents as a settled one.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

backend/app/schemas/detection_review.py

@@ -53,6 +53,10 @@ class DetectionReviewSummary(BaseModel):
     false_positive_total: int
     false_negative_total: int
     decision_counts: dict[str, int]
+    # The score with the operator's verdicts applied, next to the raw one. A
+    # finding adjudicated as a reference gap is not the model's error, and an
+    # interval covers what the unreviewed remainder could still turn out to be.
+    reviewed_metrics: dict | None = None
 
 
 class DetectionReviewList(BaseModel):

backend/app/services/detection_review_service.py

@@ -7,6 +7,7 @@ from uuid import UUID
 from sqlalchemy.orm import Session
 
 from app.core.errors import AppError
+from app.services.reviewed_metrics_service import ReviewedMetricsService
 from app.models import Detection, DetectionReview, QualityCheck, VectorFeature
 from app.schemas.detection_review import (
     DetectionReviewList,
@@ -80,7 +81,11 @@ class DetectionReviewService:
         return {(row.evidence_role, row.evidence_feature_id): row for row in rows}
 
     @staticmethod
-    def _summary(evidence: list[dict[str, str]], reviews: dict[tuple[str, str], DetectionReview]) -> DetectionReviewSummary:
+    def _summary(
+        evidence: list[dict[str, str]],
+        reviews: dict[tuple[str, str], DetectionReview],
+        quality_check: QualityCheck | None = None,
+    ) -> DetectionReviewSummary:
         evidence_keys = {(item["evidence_role"], item["evidence_feature_id"]) for item in evidence}
         decisions = Counter(
             reviews[key].decision if key in reviews else "unreviewed"
@@ -96,6 +101,43 @@ class DetectionReviewService:
             false_positive_total=false_positive_total,
             false_negative_total=false_negative_total,
             decision_counts=dict(sorted(decisions.items())),
+            reviewed_metrics=DetectionReviewService._reviewed_metrics(evidence_keys, reviews, quality_check),
+        )
+
+    @staticmethod
+    def _reviewed_metrics(
+        evidence_keys: set[tuple[str, str]],
+        reviews: dict[tuple[str, str], DetectionReview],
+        quality_check: QualityCheck | None,
+    ) -> dict | None:
+        """The score with the operator's verdicts applied.
+
+        Without this the panel shows a precision the operator has already
+        disproved: a false positive adjudicated as a reference gap is not the
+        model's error, and the raw number keeps counting it as one.
+        """
+
+        if quality_check is None:
+            return None
+        findings = quality_check.findings_json if isinstance(quality_check.findings_json, dict) else {}
+        matches = findings.get("matches")
+        false_positives = findings.get("false_positives")
+        false_negatives = findings.get("false_negatives")
+        if not all(isinstance(value, int) for value in (matches, false_positives, false_negatives)):
+            return None
+
+        per_role: dict[str, Counter] = {"false_positive": Counter(), "false_negative": Counter()}
+        for role, feature_id in evidence_keys:
+            review = reviews.get((role, feature_id))
+            if review is not None and role in per_role:
+                per_role[role][review.decision] += 1
+
+        return ReviewedMetricsService.adjudicate(
+            matches=int(matches),
+            false_positives=int(false_positives),
+            false_negatives=int(false_negatives),
+            false_positive_decisions=dict(per_role["false_positive"]),
+            false_negative_decisions=dict(per_role["false_negative"]),
         )
 
     @staticmethod
@@ -180,7 +222,7 @@ class DetectionReviewService:
             total=len(filtered),
             limit=limit,
             offset=offset,
-            summary=DetectionReviewService._summary(evidence, reviews),
+            summary=DetectionReviewService._summary(evidence, reviews, quality_check),
         )
 
     @staticmethod

backend/app/services/reviewed_metrics_service.py

@@ -0,0 +1,112 @@
+"""Detection metrics after an operator has adjudicated the evidence.
+
+The review vocabulary already separates a model error from a reference gap,
+because the product's position is that official footprints are not
+automatically perfect ground truth. Until now those verdicts were only counted:
+an operator who established that twelve of forty false positives are buildings
+the reference simply lacks still saw a precision counting all forty against the
+model — a number they had personally disproved.
+
+Applying the verdicts gives an adjudicated score. Because part of the evidence
+is usually still unreviewed, the honest form is an interval: pessimistic
+assumes every unreviewed item is a model error, optimistic assumes none is. The
+raw score stays reported alongside, so nothing is quietly improved.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+# A verdict that the finding was not the model's fault. The detection (or the
+# missing detection) was right; the reference or the matching rule was not.
+EXONERATING_DECISIONS = frozenset({"reference_gap_or_change", "qa_alignment_mismatch"})
+
+# Verdicts that confirm the finding, and verdicts that reach no conclusion.
+# Both keep counting: being unable to judge is not evidence in the model's
+# favour, and treating it as such is how a score drifts upward unearned.
+CONFIRMING_DECISIONS = frozenset(
+    {"confirmed_model_false_positive", "confirmed_model_false_negative"}
+)
+INCONCLUSIVE_DECISIONS = frozenset({"uncertain", "imagery_obscured_or_uncertain"})
+
+
+def _score(matches: int, false_positives: int, false_negatives: int) -> dict[str, Any]:
+    precision = matches / (matches + false_positives) if matches + false_positives > 0 else None
+    recall = matches / (matches + false_negatives) if matches + false_negatives > 0 else None
+    f1_score = None
+    if precision is not None and recall is not None:
+        f1_score = (2 * precision * recall / (precision + recall)) if precision + recall > 0 else 0.0
+    return {
+        "matches": matches,
+        "false_positives": false_positives,
+        "false_negatives": false_negatives,
+        "precision": precision,
+        "recall": recall,
+        "f1_score": f1_score,
+    }
+
+
+class ReviewedMetricsService:
+    @staticmethod
+    def _adjudicate_role(total: int, decisions: dict[str, int]) -> tuple[int, int, int]:
+        """Split a finding count into exonerated, confirmed and unreviewed.
+
+        A decision the runtime does not recognise — one from a later release —
+        counts as no judgement rather than as an exoneration.
+        """
+
+        exonerated = sum(count for name, count in decisions.items() if name in EXONERATING_DECISIONS)
+        judged = sum(
+            count
+            for name, count in decisions.items()
+            if name in EXONERATING_DECISIONS | CONFIRMING_DECISIONS | INCONCLUSIVE_DECISIONS
+        )
+        exonerated = min(exonerated, total)
+        judged = min(judged, total)
+        return exonerated, judged - exonerated, max(0, total - judged)
+
+    @staticmethod
+    def adjudicate(
+        *,
+        matches: int,
+        false_positives: int,
+        false_negatives: int,
+        false_positive_decisions: dict[str, int],
+        false_negative_decisions: dict[str, int],
+    ) -> dict[str, Any]:
+        """Apply operator verdicts to a quality check's counts."""
+
+        fp_exonerated, _fp_confirmed, fp_unreviewed = ReviewedMetricsService._adjudicate_role(
+            false_positives, false_positive_decisions
+        )
+        fn_exonerated, _fn_confirmed, fn_unreviewed = ReviewedMetricsService._adjudicate_role(
+            false_negatives, false_negative_decisions
+        )
+
+        adjudicated_fp = false_positives - fp_exonerated
+        adjudicated_fn = false_negatives - fn_exonerated
+
+        # The interval covers what the unreviewed remainder could still turn
+        # out to be, so a partly reviewed check never reads as a settled one.
+        pessimistic = _score(matches, adjudicated_fp, adjudicated_fn)
+        optimistic = _score(matches, adjudicated_fp - fp_unreviewed, adjudicated_fn - fn_unreviewed)
+
+        return {
+            "raw": _score(matches, false_positives, false_negatives),
+            "adjudicated": pessimistic,
+            "pessimistic": pessimistic,
+            "optimistic": optimistic,
+            "review_complete": fp_unreviewed == 0 and fn_unreviewed == 0,
+            "false_positive_breakdown": {
+                "total": false_positives,
+                "exonerated": fp_exonerated,
+                "confirmed_or_inconclusive": false_positives - fp_exonerated - fp_unreviewed,
+                "unreviewed": fp_unreviewed,
+            },
+            "false_negative_breakdown": {
+                "total": false_negatives,
+                "exonerated": fn_exonerated,
+                "confirmed_or_inconclusive": false_negatives - fn_exonerated - fn_unreviewed,
+                "unreviewed": fn_unreviewed,
+            },
+        }

backend/tests/test_reviewed_detection_metrics.py

@@ -0,0 +1,254 @@
+"""An operator's adjudication must reach the score.
+
+The review vocabulary already distinguishes a model error from a reference gap
+— the product's own position is that official footprints are not automatically
+perfect ground truth. But the reviews were only counted. An operator who
+inspects forty false positives and establishes that twelve are buildings the
+reference simply lacks still sees a precision that counts all forty against the
+model, and that they have personally disproved.
+
+Because part of the evidence is usually still unreviewed, the honest answer is
+an interval, not a single corrected number: pessimistic assumes every
+unreviewed item is a model error, optimistic assumes none is.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from app.services.reviewed_metrics_service import ReviewedMetricsService
+
+
+def _counts(**decisions: int) -> dict[str, int]:
+    return decisions
+
+
+class TestAdjudication:
+    def test_a_reference_gap_stops_counting_against_precision(self) -> None:
+        result = ReviewedMetricsService.adjudicate(
+            matches=80,
+            false_positives=20,
+            false_negatives=10,
+            false_positive_decisions=_counts(reference_gap_or_change=20),
+            false_negative_decisions={},
+        )
+
+        # Every false positive was the reference missing a real building.
+        assert result["adjudicated"]["false_positives"] == 0
+        assert result["adjudicated"]["precision"] == pytest.approx(1.0)
+
+    def test_a_confirmed_model_error_keeps_counting(self) -> None:
+        result = ReviewedMetricsService.adjudicate(
+            matches=80,
+            false_positives=20,
+            false_negatives=0,
+            false_positive_decisions=_counts(confirmed_model_false_positive=20),
+            false_negative_decisions={},
+        )
+
+        assert result["adjudicated"]["false_positives"] == 20
+        assert result["adjudicated"]["precision"] == pytest.approx(0.8)
+
+    def test_an_alignment_mismatch_is_not_a_model_error(self) -> None:
+        """Both the detection and the footprint were right; the matching failed."""
+
+        result = ReviewedMetricsService.adjudicate(
+            matches=80,
+            false_positives=20,
+            false_negatives=0,
+            false_positive_decisions=_counts(qa_alignment_mismatch=20),
+            false_negative_decisions={},
+        )
+
+        assert result["adjudicated"]["false_positives"] == 0
+
+    def test_a_reference_gap_on_a_miss_stops_counting_against_recall(self) -> None:
+        result = ReviewedMetricsService.adjudicate(
+            matches=80,
+            false_positives=0,
+            false_negatives=20,
+            false_positive_decisions={},
+            false_negative_decisions=_counts(reference_gap_or_change=20),
+        )
+
+        # The reference held twenty footprints that are not there.
+        assert result["adjudicated"]["false_negatives"] == 0
+        assert result["adjudicated"]["recall"] == pytest.approx(1.0)
+
+    def test_an_uncertain_verdict_keeps_counting_against_the_model(self) -> None:
+        """Not being able to judge is not evidence in the model's favour."""
+
+        result = ReviewedMetricsService.adjudicate(
+            matches=80,
+            false_positives=20,
+            false_negatives=0,
+            false_positive_decisions=_counts(uncertain=10, imagery_obscured_or_uncertain=10),
+            false_negative_decisions={},
+        )
+
+        assert result["adjudicated"]["false_positives"] == 20
+
+
+class TestBounds:
+    def test_a_partly_reviewed_check_reports_an_interval(self) -> None:
+        result = ReviewedMetricsService.adjudicate(
+            matches=80,
+            false_positives=20,
+            false_negatives=0,
+            false_positive_decisions=_counts(reference_gap_or_change=10),
+            false_negative_decisions={},
+        )
+
+        # Ten unreviewed: pessimistically all model errors, optimistically none.
+        assert result["pessimistic"]["precision"] == pytest.approx(80 / 90)
+        assert result["optimistic"]["precision"] == pytest.approx(1.0)
+        assert result["review_complete"] is False
+
+    def test_a_fully_reviewed_check_collapses_the_interval(self) -> None:
+        result = ReviewedMetricsService.adjudicate(
+            matches=80,
+            false_positives=20,
+            false_negatives=5,
+            false_positive_decisions=_counts(reference_gap_or_change=12, confirmed_model_false_positive=8),
+            false_negative_decisions=_counts(confirmed_model_false_negative=5),
+        )
+
+        assert result["review_complete"] is True
+        assert result["pessimistic"]["precision"] == pytest.approx(result["optimistic"]["precision"])
+        assert result["adjudicated"]["precision"] == pytest.approx(80 / 88)
+
+    def test_an_unreviewed_check_reports_the_raw_numbers_unchanged(self) -> None:
+        result = ReviewedMetricsService.adjudicate(
+            matches=80,
+            false_positives=20,
+            false_negatives=10,
+            false_positive_decisions={},
+            false_negative_decisions={},
+        )
+
+        assert result["review_complete"] is False
+        assert result["adjudicated"]["precision"] == pytest.approx(result["raw"]["precision"])
+        assert result["adjudicated"]["recall"] == pytest.approx(result["raw"]["recall"])
+
+    def test_the_raw_score_is_always_reported_alongside(self) -> None:
+        result = ReviewedMetricsService.adjudicate(
+            matches=80,
+            false_positives=20,
+            false_negatives=0,
+            false_positive_decisions=_counts(reference_gap_or_change=20),
+            false_negative_decisions={},
+        )
+
+        assert result["raw"]["precision"] == pytest.approx(0.8)
+        assert result["adjudicated"]["precision"] == pytest.approx(1.0)
+
+
+class TestEdges:
+    def test_a_check_without_findings_makes_no_claim(self) -> None:
+        result = ReviewedMetricsService.adjudicate(
+            matches=0,
+            false_positives=0,
+            false_negatives=0,
+            false_positive_decisions={},
+            false_negative_decisions={},
+        )
+
+        assert result["adjudicated"]["precision"] is None
+        assert result["adjudicated"]["recall"] is None
+        assert result["review_complete"] is True
+
+    def test_more_decisions_than_findings_cannot_invent_a_negative_count(self) -> None:
+        result = ReviewedMetricsService.adjudicate(
+            matches=10,
+            false_positives=2,
+            false_negatives=0,
+            false_positive_decisions=_counts(reference_gap_or_change=99),
+            false_negative_decisions={},
+        )
+
+        assert result["adjudicated"]["false_positives"] == 0
+
+    def test_an_unknown_decision_is_treated_as_no_judgement(self) -> None:
+        result = ReviewedMetricsService.adjudicate(
+            matches=10,
+            false_positives=5,
+            false_negatives=0,
+            false_positive_decisions=_counts(something_new_from_a_later_release=5),
+            false_negative_decisions={},
+        )
+
+        assert result["adjudicated"]["false_positives"] == 5
+        assert result["review_complete"] is False
+
+
+class TestThroughTheReviewPanel:
+    """The score the panel shows, not just the arithmetic behind it."""
+
+    def _quality_check(self, quality_check_id, project_id):
+        from app.models import QualityCheck
+
+        return QualityCheck(
+            id=quality_check_id,
+            project_id=project_id,
+            reference_dataset_id=__import__("uuid").uuid4(),
+            check_type="detections_vs_reference",
+            status="ok",
+            findings_json={
+                "matches": 80,
+                "false_positives": 20,
+                "false_negatives": 0,
+                "false_positive_evidence": [
+                    {"candidate_feature_id": f"detection-{index}"} for index in range(20)
+                ],
+                "false_negative_evidence": [],
+            },
+        )
+
+    def test_adjudicated_reference_gaps_raise_the_reported_precision(self) -> None:
+        import uuid
+
+        from app.models import DetectionReview, QualityCheck
+        from app.services.detection_review_service import DetectionReviewService
+
+        quality_check_id, project_id = uuid.uuid4(), uuid.uuid4()
+        quality_check = self._quality_check(quality_check_id, project_id)
+        reviews = [
+            DetectionReview(
+                id=uuid.uuid4(),
+                quality_check_id=quality_check_id,
+                evidence_role="false_positive",
+                evidence_feature_id=f"detection-{index}",
+                decision="reference_gap_or_change",
+            )
+            for index in range(12)
+        ]
+
+        class _Query:
+            def __init__(self, rows):
+                self.rows = rows
+
+            def filter(self, *_args):
+                return self
+
+            def all(self):
+                return self.rows
+
+        class _Session:
+            def get(self, model, item_id):
+                return quality_check if model is QualityCheck and item_id == quality_check_id else None
+
+            def query(self, _model):
+                return _Query(reviews)
+
+        result = DetectionReviewService.list_reviews(
+            _Session(), project_id=project_id, quality_check_id=quality_check_id
+        )
+        metrics = result.summary.reviewed_metrics
+
+        assert metrics is not None
+        assert metrics["raw"]["precision"] == pytest.approx(0.8)
+        # Twelve of the twenty were the reference missing a building.
+        assert metrics["adjudicated"]["precision"] == pytest.approx(80 / 88)
+        assert metrics["review_complete"] is False
+        assert metrics["false_positive_breakdown"]["exonerated"] == 12
+        assert metrics["false_positive_breakdown"]["unreviewed"] == 8

docs/API_CONTRACTS.md

@@ -1850,6 +1850,32 @@ Each feature includes:
 
 Returns persisted detections for a dataset as a GeoJSON FeatureCollection. Optional filters match the detection list endpoint.
 
+### GET `/api/v1/projects/{project_id}/quality-checks/{id}/reviews`
+
+Returns the evidence queue plus a `summary`, which now carries
+`reviewed_metrics`: the score with the operator's verdicts applied, next to the
+raw one.
+
+The review vocabulary already separates a model error from a reference gap,
+because official footprints are not automatically perfect ground truth. Those
+verdicts were only counted, so an operator who established that twelve of forty
+false positives are buildings the reference simply lacks still saw a precision
+counting all forty against the model — a number they had personally disproved.
+
+- `reference_gap_or_change` and `qa_alignment_mismatch` exonerate a finding: the
+  detection, or the missing detection, was not the model's error.
+- `confirmed_model_false_positive` / `confirmed_model_false_negative` keep it.
+- `uncertain` and `imagery_obscured_or_uncertain` also keep it. Being unable to
+  judge is not evidence in the model's favour, and treating it as such is how a
+  score drifts upward unearned. A decision from a later release the runtime does
+  not recognise is likewise treated as no judgement.
+
+Because part of the evidence is usually unreviewed, the result is an interval:
+`pessimistic` assumes every unreviewed finding is a model error, `optimistic`
+assumes none is, and `adjudicated` equals the pessimistic reading so a partly
+reviewed check never presents as a settled one. `review_complete` says whether
+the interval has collapsed.
+
 ### POST `/api/v1/detection/runs/compare`
 
 Scores several persisted runs against one reference and ranks them on average

frontend/src/components/quality/DetectionReviewPanel.tsx

@@ -51,6 +51,10 @@ function shortId(value: string): string {
   return value.length > 18 ? `${value.slice(0, 8)}...${value.slice(-6)}` : value
 }
 
+function formatScore(value: number | null | undefined): string {
+  return typeof value === 'number' && Number.isFinite(value) ? value.toFixed(3) : 'n.v.t.'
+}
+
 export function DetectionReviewPanel({
   projectId,
   qualityCheckId,
@@ -132,6 +136,30 @@ export function DetectionReviewPanel({
         </div>
       ) : null}
 
+      {queue?.summary.reviewed_metrics ? (
+        <div className="detection-review-adjudicated" aria-label="Score na beoordeling">
+          <div className="summary-grid">
+            <div>
+              <span>Precisie zoals gemeten</span>
+              <strong>{formatScore(queue.summary.reviewed_metrics.raw.precision)}</strong>
+            </div>
+            <div>
+              <span>Precisie na uw beoordeling</span>
+              <strong>{formatScore(queue.summary.reviewed_metrics.adjudicated.precision)}</strong>
+            </div>
+            <div>
+              <span>Herkenningsgraad na beoordeling</span>
+              <strong>{formatScore(queue.summary.reviewed_metrics.adjudicated.recall)}</strong>
+            </div>
+          </div>
+          <p className="geo-data-notice">
+            {queue.summary.reviewed_metrics.review_complete
+              ? `Alle bevindingen zijn beoordeeld. ${queue.summary.reviewed_metrics.false_positive_breakdown.exonerated} onterecht gevonden objecten en ${queue.summary.reviewed_metrics.false_negative_breakdown.exonerated} gemiste objecten bleken een hiaat in de referentielaag, niet een modelfout.`
+              : `Nog ${queue.summary.reviewed_metrics.false_positive_breakdown.unreviewed + queue.summary.reviewed_metrics.false_negative_breakdown.unreviewed} bevindingen onbeoordeeld. De precisie ligt tussen ${formatScore(queue.summary.reviewed_metrics.pessimistic.precision)} en ${formatScore(queue.summary.reviewed_metrics.optimistic.precision)}; de getoonde waarde rekent elke onbeoordeelde bevinding nog als modelfout.`}
+          </p>
+        </div>
+      ) : null}
+
       <div className="detection-review-filters">
         <label>
           Soort

frontend/src/types.ts

@@ -1747,6 +1747,38 @@ export interface DetectionReviewRead {
   updated_at?: string | null
 }
 
+export interface ReviewedScore {
+  matches: number
+  false_positives: number
+  false_negatives: number
+  precision: number | null
+  recall: number | null
+  f1_score: number | null
+}
+
+export interface ReviewedFindingBreakdown {
+  total: number
+  /** Findings the operator established were not the model's error. */
+  exonerated: number
+  confirmed_or_inconclusive: number
+  unreviewed: number
+}
+
+/**
+ * The score with operator verdicts applied, next to the raw one. A finding
+ * adjudicated as a reference gap is not the model's error; the interval covers
+ * what the unreviewed remainder could still turn out to be.
+ */
+export interface ReviewedMetrics {
+  raw: ReviewedScore
+  adjudicated: ReviewedScore
+  pessimistic: ReviewedScore
+  optimistic: ReviewedScore
+  review_complete: boolean
+  false_positive_breakdown: ReviewedFindingBreakdown
+  false_negative_breakdown: ReviewedFindingBreakdown
+}
+
 export interface DetectionReviewSummary {
   total: number
   reviewed: number
@@ -1754,6 +1786,7 @@ export interface DetectionReviewSummary {
   false_positive_total: number
   false_negative_total: number
   decision_counts: Record<string, number>
+  reviewed_metrics?: ReviewedMetrics | null
 }
 
 export interface DetectionReviewList {