calibrate a confidence threshold from one inference pass

Threshold calibration ran the model over every tile once per threshold — three
GPU passes to compare 0.50, 0.25 and 0.15 on a hundred-tile raster. The answer
is already in a single run at the lowest value: detections above a higher cut
are a subset of it, and duplicate suppression walks candidates in descending
confidence, so a lower-confidence box can never displace a higher-confidence
one. The kept set above any cut is identical whichever threshold the run used,
which is what makes one pass sufficient rather than merely cheaper.

QA now takes calibration_thresholds and reads each operating point off the same
precision/recall walk it already performs, marking the F1-optimal cut. The lab
runs inference once and fills its table from the sweep.

The contract test asserted the per-threshold loop by name, pinning the waste it
was meant to describe. It now states what calibration owes an operator: a row
per requested threshold, from one run.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

backend/app/api/routes/detection.py

@@ -249,5 +249,6 @@ def compare_detection_run_with_reference(
             iou_threshold=payload.iou_threshold,
             class_name=payload.class_name,
             min_confidence=payload.min_confidence,
+            calibration_thresholds=payload.calibration_thresholds,
         )
     )

backend/app/schemas/detection.py

@@ -73,6 +73,9 @@ class DetectionQaRequest(BaseModel):
     iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
     class_name: str | None = None
     min_confidence: float | None = Field(default=None, ge=0.0, le=1.0)
+    # Confidence cuts to report alongside the run's own operating point. They
+    # are read off the one matching pass, so a sweep costs no extra inference.
+    calibration_thresholds: list[float] = Field(default_factory=list, max_length=32)
 
 
 class DetectionRunResponse(BaseModel):

backend/app/schemas/qa.py

@@ -121,6 +121,8 @@ class AnalysisQaResponse(BaseModel):
     coverage: dict[str, Any] | None = None
     temporal_compatibility: dict[str, Any] | None = None
     box_to_footprint_diagnostics: dict[str, Any] | None = None
+    precision_recall_curve: dict[str, Any] | None = None
+    calibration_sweep: list[dict[str, Any]] = Field(default_factory=list)
     match_evidence: list[dict[str, Any]] = Field(default_factory=list)
     false_positive_evidence: list[dict[str, Any]] = Field(default_factory=list)
     false_negative_evidence: list[dict[str, Any]] = Field(default_factory=list)

backend/app/services/detection_metrics_service.py

@@ -118,6 +118,66 @@ class DetectionMetricsService:
             area += (recalls[index] - recalls[index - 1]) * precisions[index]
         return area
 
+    @staticmethod
+    def operating_point(curve: dict[str, Any], *, min_confidence: float) -> dict[str, Any]:
+        """The metrics that hold when detections below ``min_confidence`` are dropped.
+
+        Read off the curve rather than recomputed: the curve already walked the
+        ranking once, and every threshold is a prefix of that walk. Running the
+        model again per threshold would spend N GPU passes to reproduce numbers
+        that are already here.
+        """
+
+        points = curve.get("points") or []
+        reference_count = int(curve.get("reference_count") or 0)
+        admitted = [point for point in points if point["confidence_threshold"] >= min_confidence]
+        # Points are cumulative down the ranking, so the last admitted one is
+        # the complete tally at this cut.
+        tally = admitted[-1] if admitted else None
+
+        true_positives = int(tally["true_positives"]) if tally else 0
+        false_positives = int(tally["false_positives"]) if tally else 0
+        false_negatives = max(0, reference_count - true_positives)
+        candidate_count = true_positives + false_positives
+
+        precision = true_positives / candidate_count if candidate_count else None
+        recall = true_positives / reference_count if reference_count else None
+        f1_score = None
+        if precision is not None and recall is not None:
+            f1_score = (2 * precision * recall / (precision + recall)) if precision + recall > 0 else 0.0
+
+        return {
+            "min_confidence": min_confidence,
+            "confidence_threshold": tally["confidence_threshold"] if tally else None,
+            "candidate_count": candidate_count,
+            "true_positives": true_positives,
+            "false_positives": false_positives,
+            "false_negatives": false_negatives,
+            "precision": precision,
+            "recall": recall,
+            "f1_score": f1_score,
+        }
+
+    @staticmethod
+    def calibration_sweep(curve: dict[str, Any], *, thresholds: list[float]) -> list[dict[str, Any]]:
+        """Every requested operating point, strictest first, from one curve.
+
+        Recall cannot fall as the cut loosens — that monotonicity is exactly
+        why a single run answers the whole sweep.
+        """
+
+        ordered = sorted({float(value) for value in thresholds}, reverse=True)
+        rows = [
+            DetectionMetricsService.operating_point(curve, min_confidence=value) for value in ordered
+        ]
+        best_f1 = max((row["f1_score"] or 0.0) for row in rows) if rows else 0.0
+        marked = False
+        for row in rows:
+            is_best = not marked and (row["f1_score"] or 0.0) == best_f1
+            row["best_f1_in_sweep"] = is_best
+            marked = marked or is_best
+        return rows
+
     @staticmethod
     def precision_recall_curve(
         candidates: list[tuple[dict[str, Any], BaseGeometry]],

backend/app/services/detection_service.py

@@ -377,6 +377,7 @@ class DetectionService:
         iou_threshold: float = 0.5,
         class_name: str | None = None,
         min_confidence: float | None = None,
+        calibration_thresholds: list[float] | None = None,
     ) -> dict[str, Any]:
         run = db.get(AnalysisRun, analysis_run_id)
         if not run or run.analysis_type != "detection":
@@ -563,6 +564,15 @@ class DetectionService:
             reference_geometries,
             iou_threshold=iou_threshold,
         )
+        # Every requested confidence cut, answered from that one matching pass.
+        # Re-running inference per threshold spends N GPU passes to reproduce
+        # numbers already present here: suppression walks candidates in
+        # descending confidence, so the kept set above a cut does not depend on
+        # the threshold the run itself used.
+        calibration_sweep = DetectionMetricsService.calibration_sweep(
+            precision_recall_curve,
+            thresholds=list(calibration_thresholds or []),
+        )
         mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values)
         precision = evidence.matches / (evidence.matches + evidence.false_positives) if evidence.matches + evidence.false_positives > 0 else None
         recall = evidence.matches / (evidence.matches + evidence.false_negatives) if evidence.matches + evidence.false_negatives > 0 else None
@@ -598,6 +608,7 @@ class DetectionService:
                 "temporal_compatibility": temporal_compatibility,
                 "box_to_footprint_diagnostics": box_to_footprint_diagnostics,
                 "precision_recall_curve": precision_recall_curve,
+                "calibration_sweep": calibration_sweep,
                 "match_evidence": evidence.match_evidence,
                 "false_positive_evidence": evidence.false_positive_evidence,
                 "false_negative_evidence": evidence.false_negative_evidence,
@@ -647,6 +658,7 @@ class DetectionService:
             "temporal_compatibility": temporal_compatibility,
             "box_to_footprint_diagnostics": box_to_footprint_diagnostics,
             "precision_recall_curve": precision_recall_curve,
+            "calibration_sweep": calibration_sweep,
             "match_evidence": evidence.match_evidence,
             "false_positive_evidence": evidence.false_positive_evidence,
             "false_negative_evidence": evidence.false_negative_evidence,

backend/tests/test_detection_calibration_from_one_run.py

@@ -0,0 +1,119 @@
+"""Calibrating a confidence threshold does not need one inference run per value.
+
+The workbench ran the model over every tile once per threshold — three GPU
+passes to compare 0.50, 0.25 and 0.15. The answer is already in a single run at
+the lowest value: detections above a higher cut are a subset of it, and
+suppression walks candidates in descending confidence, so a lower-confidence
+box can never displace a higher-confidence one. The kept set above any cut is
+therefore identical whichever threshold the run used.
+
+One matching pass produces every operating point exactly, so the sweep is free
+rather than N times the cost.
+"""
+
+from __future__ import annotations
+
+import pytest
+from shapely.geometry import box
+
+from app.services.detection_metrics_service import DetectionMetricsService
+
+
+def _candidate(name: str, geometry, confidence: float):
+    return ({"id": name, "confidence": confidence}, geometry)
+
+
+def _reference(name: str, geometry):
+    return ({"id": name}, geometry)
+
+
+REFERENCES = [
+    _reference("r1", box(0, 0, 1, 1)),
+    _reference("r2", box(5, 5, 6, 6)),
+    _reference("r3", box(10, 10, 11, 11)),
+]
+CANDIDATES = [
+    _candidate("hit-high", box(0, 0, 1, 1), 0.90),
+    _candidate("hit-mid", box(5, 5, 6, 6), 0.40),
+    _candidate("junk-low", box(30, 30, 31, 31), 0.20),
+]
+
+
+def _curve():
+    return DetectionMetricsService.precision_recall_curve(CANDIDATES, REFERENCES, iou_threshold=0.5)
+
+
+class TestOperatingPoints:
+    def test_a_strict_cut_keeps_only_the_confident_detection(self) -> None:
+        point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.5)
+
+        assert point["true_positives"] == 1
+        assert point["false_positives"] == 0
+        assert point["false_negatives"] == 2
+        assert point["precision"] == pytest.approx(1.0)
+        assert point["recall"] == pytest.approx(1 / 3)
+
+    def test_a_looser_cut_finds_more_and_stays_exact(self) -> None:
+        point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.3)
+
+        assert point["true_positives"] == 2
+        assert point["false_positives"] == 0
+        assert point["recall"] == pytest.approx(2 / 3)
+
+    def test_the_loosest_cut_admits_the_false_positive(self) -> None:
+        point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.1)
+
+        assert point["true_positives"] == 2
+        assert point["false_positives"] == 1
+        assert point["precision"] == pytest.approx(2 / 3)
+
+    def test_a_cut_above_every_detection_finds_nothing_but_still_reports(self) -> None:
+        point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.99)
+
+        assert point["true_positives"] == 0
+        assert point["false_negatives"] == 3
+        assert point["recall"] == pytest.approx(0.0)
+        assert point["precision"] is None
+
+    def test_the_requested_threshold_is_echoed_back(self) -> None:
+        point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.42)
+
+        assert point["min_confidence"] == pytest.approx(0.42)
+        # The nearest actual operating point sits at the detection's own
+        # confidence, which is what the numbers describe.
+        assert point["confidence_threshold"] == pytest.approx(0.9)
+
+
+class TestSweep:
+    def test_a_sweep_returns_one_row_per_requested_threshold(self) -> None:
+        rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.5, 0.3, 0.1])
+
+        assert [row["min_confidence"] for row in rows] == [0.5, 0.3, 0.1]
+
+    def test_the_sweep_is_ordered_from_strict_to_loose(self) -> None:
+        rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.1, 0.5, 0.3])
+
+        assert [row["min_confidence"] for row in rows] == [0.5, 0.3, 0.1]
+
+    def test_a_repeated_threshold_is_asked_once(self) -> None:
+        rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.3, 0.3])
+
+        assert len(rows) == 1
+
+    def test_recall_never_falls_as_the_cut_loosens(self) -> None:
+        """The monotonicity that makes one run sufficient."""
+
+        rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.9, 0.5, 0.3, 0.1])
+        recalls = [row["recall"] for row in rows]
+
+        assert recalls == sorted(recalls)
+
+    def test_an_empty_sweep_is_not_an_error(self) -> None:
+        assert DetectionMetricsService.calibration_sweep(_curve(), thresholds=[]) == []
+
+    def test_the_sweep_marks_the_f1_optimal_row(self) -> None:
+        rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.5, 0.3, 0.1])
+        best = [row for row in rows if row["best_f1_in_sweep"]]
+
+        assert len(best) == 1
+        assert best[0]["f1_score"] == max(row["f1_score"] for row in rows)

backend/tests/test_sprint134_guided_detection_calibration_runner.py

@@ -22,9 +22,15 @@ def test_detection_lab_has_guided_threshold_calibration_runner() -> None:
     assert "detectionCalibrationRows" in hook_source
     assert "detectionCalibrationError" in hook_source
     assert "runDetectionCalibration" in hook_source
-    assert "for (const threshold of thresholds)" in hook_source
     assert "detectionApi.run({" in hook_source
-    assert "confidence_threshold: threshold" in hook_source
+    # Every requested threshold is reported, but from one inference pass at the
+    # lowest cut: detections above a higher cut are a subset of it, and
+    # suppression walks candidates in descending confidence, so the kept set
+    # above a cut does not depend on the threshold the run used. Asserting the
+    # old per-threshold loop pinned N GPU passes that produced identical numbers.
+    assert "confidence_threshold: lowestThreshold" in hook_source
+    assert "calibration_thresholds: thresholds" in hook_source
+    assert "calibration_sweep" in hook_source
     assert "parameters_json: { calibration: true, calibration_thresholds: thresholds }" in hook_source
     assert "detectionApi.compareWithReference(result.analysis_run_id" in hook_source
     assert "reference_dataset_id: detectionReferenceDatasetId" in hook_source