JensandClaude Opus 5
2b968b74cf
The greedy IoU matcher gave a reference to whichever candidate was offered first. Row order decided that, and every detection in a run shares one transaction timestamp, so ordering by created_at left the assignment undefined: the same QA run over the same data produced different mean IoU, and the geometry shown to a reviewer as a false positive could be the better of two detections. Candidates are now ranked by confidence with feature identity as tiebreaker, which is also the COCO/PASCAL rule. A single precision/recall/F1 triple describes one operating point, so two models cannot be compared from it: a conservatively calibrated model looks worse at a low confidence cut and better at a high one without detecting anything differently. DetectionMetricsService adds the full curve, average precision and the threshold where F1 actually peaks. Also: - report the population the metrics were computed over, so matches + false_positives equals candidate_feature_count even under an area filter; raw dataset totals move to the _raw fields; - state whether candidates are axis-aligned boxes or footprint polygons. A box can never reach IoU 1 against a rotated building, so the strict score has a ceiling that has nothing to do with detection quality. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
117 lines
4.4 KiB
Python
117 lines
4.4 KiB
Python
"""A single F1 at one arbitrary confidence cut cannot compare two models.
|
|
|
|
Reporting F1 at whichever threshold the operator happened to type makes two
|
|
models look better or worse depending on their calibration rather than their
|
|
detection quality. The sweep produces the standard curve instead: precision
|
|
and recall at every operating point, average precision, and the threshold
|
|
where F1 actually peaks.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
from shapely.geometry import box
|
|
|
|
from app.services.detection_metrics_service import DetectionMetricsService
|
|
|
|
|
|
def _candidate(name: str, geometry, confidence: float):
|
|
return ({"id": name, "confidence": confidence}, geometry)
|
|
|
|
|
|
def _reference(name: str, geometry):
|
|
return ({"id": name}, geometry)
|
|
|
|
|
|
def test_perfect_detector_reaches_average_precision_one() -> None:
|
|
references = [_reference("r1", box(0, 0, 1, 1)), _reference("r2", box(5, 5, 6, 6))]
|
|
candidates = [
|
|
_candidate("c1", box(0, 0, 1, 1), 0.9),
|
|
_candidate("c2", box(5, 5, 6, 6), 0.8),
|
|
]
|
|
|
|
curve = DetectionMetricsService.precision_recall_curve(candidates, references, iou_threshold=0.5)
|
|
|
|
assert curve["average_precision"] == pytest.approx(1.0)
|
|
assert curve["best_f1"] == pytest.approx(1.0)
|
|
assert curve["reference_count"] == 2
|
|
|
|
|
|
def test_low_confidence_false_positive_is_only_penalised_below_its_threshold() -> None:
|
|
references = [_reference("r1", box(0, 0, 1, 1))]
|
|
candidates = [
|
|
_candidate("hit", box(0, 0, 1, 1), 0.9),
|
|
_candidate("junk", box(20, 20, 21, 21), 0.2),
|
|
]
|
|
|
|
curve = DetectionMetricsService.precision_recall_curve(candidates, references, iou_threshold=0.5)
|
|
|
|
# Cutting at 0.2 admits the junk box, so precision there is 0.5.
|
|
low = next(point for point in curve["points"] if point["confidence_threshold"] == pytest.approx(0.2))
|
|
assert low["precision"] == pytest.approx(0.5)
|
|
assert low["recall"] == pytest.approx(1.0)
|
|
|
|
# The optimum simply drops it.
|
|
assert curve["best_f1"] == pytest.approx(1.0)
|
|
assert curve["best_f1_threshold"] == pytest.approx(0.9)
|
|
# AP is computed over the ranking, so one trailing false positive after
|
|
# full recall does not reduce it.
|
|
assert curve["average_precision"] == pytest.approx(1.0)
|
|
|
|
|
|
def test_ranking_quality_is_visible_in_average_precision() -> None:
|
|
"""A detector that ranks its mistake above its hit scores worse."""
|
|
|
|
references = [_reference("r1", box(0, 0, 1, 1))]
|
|
good = DetectionMetricsService.precision_recall_curve(
|
|
[_candidate("hit", box(0, 0, 1, 1), 0.9), _candidate("junk", box(20, 20, 21, 21), 0.1)],
|
|
references,
|
|
iou_threshold=0.5,
|
|
)
|
|
bad = DetectionMetricsService.precision_recall_curve(
|
|
[_candidate("hit", box(0, 0, 1, 1), 0.1), _candidate("junk", box(20, 20, 21, 21), 0.9)],
|
|
references,
|
|
iou_threshold=0.5,
|
|
)
|
|
|
|
assert good["average_precision"] > bad["average_precision"]
|
|
assert bad["average_precision"] == pytest.approx(0.5)
|
|
|
|
|
|
def test_missed_reference_caps_recall_and_average_precision() -> None:
|
|
references = [_reference("r1", box(0, 0, 1, 1)), _reference("r2", box(9, 9, 10, 10))]
|
|
candidates = [_candidate("hit", box(0, 0, 1, 1), 0.9)]
|
|
|
|
curve = DetectionMetricsService.precision_recall_curve(candidates, references, iou_threshold=0.5)
|
|
|
|
assert curve["points"][0]["recall"] == pytest.approx(0.5)
|
|
assert curve["average_precision"] == pytest.approx(0.5)
|
|
assert curve["best_f1"] == pytest.approx(2 / 3)
|
|
|
|
|
|
def test_curve_is_independent_of_input_order() -> None:
|
|
references = [_reference("r1", box(0, 0, 1, 1)), _reference("r2", box(5, 5, 6, 6))]
|
|
candidates = [
|
|
_candidate("c1", box(0, 0, 1, 1), 0.9),
|
|
_candidate("c2", box(5, 5, 6, 6), 0.4),
|
|
_candidate("c3", box(30, 30, 31, 31), 0.6),
|
|
]
|
|
|
|
forward = DetectionMetricsService.precision_recall_curve(candidates, references, iou_threshold=0.5)
|
|
reverse = DetectionMetricsService.precision_recall_curve(
|
|
list(reversed(candidates)), list(reversed(references)), iou_threshold=0.5
|
|
)
|
|
|
|
assert forward == reverse
|
|
|
|
|
|
def test_empty_candidate_population_is_reported_not_crashed() -> None:
|
|
curve = DetectionMetricsService.precision_recall_curve(
|
|
[], [_reference("r1", box(0, 0, 1, 1))], iou_threshold=0.5
|
|
)
|
|
|
|
assert curve["average_precision"] == 0.0
|
|
assert curve["best_f1"] == 0.0
|
|
assert curve["best_f1_threshold"] is None
|
|
assert curve["points"] == []
|