JensandClaude Opus 5
2cd2c49389
Threshold calibration ran the model over every tile once per threshold — three GPU passes to compare 0.50, 0.25 and 0.15 on a hundred-tile raster. The answer is already in a single run at the lowest value: detections above a higher cut are a subset of it, and duplicate suppression walks candidates in descending confidence, so a lower-confidence box can never displace a higher-confidence one. The kept set above any cut is identical whichever threshold the run used, which is what makes one pass sufficient rather than merely cheaper. QA now takes calibration_thresholds and reads each operating point off the same precision/recall walk it already performs, marking the F1-optimal cut. The lab runs inference once and fills its table from the sweep. The contract test asserted the per-threshold loop by name, pinning the waste it was meant to describe. It now states what calibration owes an operator: a row per requested threshold, from one run. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
120 lines
4.6 KiB
Python
120 lines
4.6 KiB
Python
"""Calibrating a confidence threshold does not need one inference run per value.
|
|
|
|
The workbench ran the model over every tile once per threshold — three GPU
|
|
passes to compare 0.50, 0.25 and 0.15. The answer is already in a single run at
|
|
the lowest value: detections above a higher cut are a subset of it, and
|
|
suppression walks candidates in descending confidence, so a lower-confidence
|
|
box can never displace a higher-confidence one. The kept set above any cut is
|
|
therefore identical whichever threshold the run used.
|
|
|
|
One matching pass produces every operating point exactly, so the sweep is free
|
|
rather than N times the cost.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
from shapely.geometry import box
|
|
|
|
from app.services.detection_metrics_service import DetectionMetricsService
|
|
|
|
|
|
def _candidate(name: str, geometry, confidence: float):
|
|
return ({"id": name, "confidence": confidence}, geometry)
|
|
|
|
|
|
def _reference(name: str, geometry):
|
|
return ({"id": name}, geometry)
|
|
|
|
|
|
REFERENCES = [
|
|
_reference("r1", box(0, 0, 1, 1)),
|
|
_reference("r2", box(5, 5, 6, 6)),
|
|
_reference("r3", box(10, 10, 11, 11)),
|
|
]
|
|
CANDIDATES = [
|
|
_candidate("hit-high", box(0, 0, 1, 1), 0.90),
|
|
_candidate("hit-mid", box(5, 5, 6, 6), 0.40),
|
|
_candidate("junk-low", box(30, 30, 31, 31), 0.20),
|
|
]
|
|
|
|
|
|
def _curve():
|
|
return DetectionMetricsService.precision_recall_curve(CANDIDATES, REFERENCES, iou_threshold=0.5)
|
|
|
|
|
|
class TestOperatingPoints:
|
|
def test_a_strict_cut_keeps_only_the_confident_detection(self) -> None:
|
|
point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.5)
|
|
|
|
assert point["true_positives"] == 1
|
|
assert point["false_positives"] == 0
|
|
assert point["false_negatives"] == 2
|
|
assert point["precision"] == pytest.approx(1.0)
|
|
assert point["recall"] == pytest.approx(1 / 3)
|
|
|
|
def test_a_looser_cut_finds_more_and_stays_exact(self) -> None:
|
|
point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.3)
|
|
|
|
assert point["true_positives"] == 2
|
|
assert point["false_positives"] == 0
|
|
assert point["recall"] == pytest.approx(2 / 3)
|
|
|
|
def test_the_loosest_cut_admits_the_false_positive(self) -> None:
|
|
point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.1)
|
|
|
|
assert point["true_positives"] == 2
|
|
assert point["false_positives"] == 1
|
|
assert point["precision"] == pytest.approx(2 / 3)
|
|
|
|
def test_a_cut_above_every_detection_finds_nothing_but_still_reports(self) -> None:
|
|
point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.99)
|
|
|
|
assert point["true_positives"] == 0
|
|
assert point["false_negatives"] == 3
|
|
assert point["recall"] == pytest.approx(0.0)
|
|
assert point["precision"] is None
|
|
|
|
def test_the_requested_threshold_is_echoed_back(self) -> None:
|
|
point = DetectionMetricsService.operating_point(_curve(), min_confidence=0.42)
|
|
|
|
assert point["min_confidence"] == pytest.approx(0.42)
|
|
# The nearest actual operating point sits at the detection's own
|
|
# confidence, which is what the numbers describe.
|
|
assert point["confidence_threshold"] == pytest.approx(0.9)
|
|
|
|
|
|
class TestSweep:
|
|
def test_a_sweep_returns_one_row_per_requested_threshold(self) -> None:
|
|
rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.5, 0.3, 0.1])
|
|
|
|
assert [row["min_confidence"] for row in rows] == [0.5, 0.3, 0.1]
|
|
|
|
def test_the_sweep_is_ordered_from_strict_to_loose(self) -> None:
|
|
rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.1, 0.5, 0.3])
|
|
|
|
assert [row["min_confidence"] for row in rows] == [0.5, 0.3, 0.1]
|
|
|
|
def test_a_repeated_threshold_is_asked_once(self) -> None:
|
|
rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.3, 0.3])
|
|
|
|
assert len(rows) == 1
|
|
|
|
def test_recall_never_falls_as_the_cut_loosens(self) -> None:
|
|
"""The monotonicity that makes one run sufficient."""
|
|
|
|
rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.9, 0.5, 0.3, 0.1])
|
|
recalls = [row["recall"] for row in rows]
|
|
|
|
assert recalls == sorted(recalls)
|
|
|
|
def test_an_empty_sweep_is_not_an_error(self) -> None:
|
|
assert DetectionMetricsService.calibration_sweep(_curve(), thresholds=[]) == []
|
|
|
|
def test_the_sweep_marks_the_f1_optimal_row(self) -> None:
|
|
rows = DetectionMetricsService.calibration_sweep(_curve(), thresholds=[0.5, 0.3, 0.1])
|
|
best = [row for row in rows if row["best_f1_in_sweep"]]
|
|
|
|
assert len(best) == 1
|
|
assert best[0]["f1_score"] == max(row["f1_score"] for row in rows)
|