Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
117 lines
4.4 KiB
Python
117 lines
4.4 KiB
Python
"""A single F1 at one arbitrary confidence cut cannot compare two models.
|
|
|
|
Reporting F1 at whichever threshold the operator happened to type makes two
|
|
models look better or worse depending on their calibration rather than their
|
|
detection quality. The sweep produces the standard curve instead: precision
|
|
and recall at every operating point, average precision, and the threshold
|
|
where F1 actually peaks.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
from shapely.geometry import box
|
|
|
|
from app.services.detection_metrics_service import DetectionMetricsService
|
|
|
|
|
|
def _candidate(name: str, geometry, confidence: float):
|
|
return ({"id": name, "confidence": confidence}, geometry)
|
|
|
|
|
|
def _reference(name: str, geometry):
|
|
return ({"id": name}, geometry)
|
|
|
|
|
|
def test_perfect_detector_reaches_average_precision_one() -> None:
|
|
references = [_reference("r1", box(0, 0, 1, 1)), _reference("r2", box(5, 5, 6, 6))]
|
|
candidates = [
|
|
_candidate("c1", box(0, 0, 1, 1), 0.9),
|
|
_candidate("c2", box(5, 5, 6, 6), 0.8),
|
|
]
|
|
|
|
curve = DetectionMetricsService.precision_recall_curve(candidates, references, iou_threshold=0.5)
|
|
|
|
assert curve["average_precision"] == pytest.approx(1.0)
|
|
assert curve["best_f1"] == pytest.approx(1.0)
|
|
assert curve["reference_count"] == 2
|
|
|
|
|
|
def test_low_confidence_false_positive_is_only_penalised_below_its_threshold() -> None:
|
|
references = [_reference("r1", box(0, 0, 1, 1))]
|
|
candidates = [
|
|
_candidate("hit", box(0, 0, 1, 1), 0.9),
|
|
_candidate("junk", box(20, 20, 21, 21), 0.2),
|
|
]
|
|
|
|
curve = DetectionMetricsService.precision_recall_curve(candidates, references, iou_threshold=0.5)
|
|
|
|
# Cutting at 0.2 admits the junk box, so precision there is 0.5.
|
|
low = next(point for point in curve["points"] if point["confidence_threshold"] == pytest.approx(0.2))
|
|
assert low["precision"] == pytest.approx(0.5)
|
|
assert low["recall"] == pytest.approx(1.0)
|
|
|
|
# The optimum simply drops it.
|
|
assert curve["best_f1"] == pytest.approx(1.0)
|
|
assert curve["best_f1_threshold"] == pytest.approx(0.9)
|
|
# AP is computed over the ranking, so one trailing false positive after
|
|
# full recall does not reduce it.
|
|
assert curve["average_precision"] == pytest.approx(1.0)
|
|
|
|
|
|
def test_ranking_quality_is_visible_in_average_precision() -> None:
|
|
"""A detector that ranks its mistake above its hit scores worse."""
|
|
|
|
references = [_reference("r1", box(0, 0, 1, 1))]
|
|
good = DetectionMetricsService.precision_recall_curve(
|
|
[_candidate("hit", box(0, 0, 1, 1), 0.9), _candidate("junk", box(20, 20, 21, 21), 0.1)],
|
|
references,
|
|
iou_threshold=0.5,
|
|
)
|
|
bad = DetectionMetricsService.precision_recall_curve(
|
|
[_candidate("hit", box(0, 0, 1, 1), 0.1), _candidate("junk", box(20, 20, 21, 21), 0.9)],
|
|
references,
|
|
iou_threshold=0.5,
|
|
)
|
|
|
|
assert good["average_precision"] > bad["average_precision"]
|
|
assert bad["average_precision"] == pytest.approx(0.5)
|
|
|
|
|
|
def test_missed_reference_caps_recall_and_average_precision() -> None:
|
|
references = [_reference("r1", box(0, 0, 1, 1)), _reference("r2", box(9, 9, 10, 10))]
|
|
candidates = [_candidate("hit", box(0, 0, 1, 1), 0.9)]
|
|
|
|
curve = DetectionMetricsService.precision_recall_curve(candidates, references, iou_threshold=0.5)
|
|
|
|
assert curve["points"][0]["recall"] == pytest.approx(0.5)
|
|
assert curve["average_precision"] == pytest.approx(0.5)
|
|
assert curve["best_f1"] == pytest.approx(2 / 3)
|
|
|
|
|
|
def test_curve_is_independent_of_input_order() -> None:
|
|
references = [_reference("r1", box(0, 0, 1, 1)), _reference("r2", box(5, 5, 6, 6))]
|
|
candidates = [
|
|
_candidate("c1", box(0, 0, 1, 1), 0.9),
|
|
_candidate("c2", box(5, 5, 6, 6), 0.4),
|
|
_candidate("c3", box(30, 30, 31, 31), 0.6),
|
|
]
|
|
|
|
forward = DetectionMetricsService.precision_recall_curve(candidates, references, iou_threshold=0.5)
|
|
reverse = DetectionMetricsService.precision_recall_curve(
|
|
list(reversed(candidates)), list(reversed(references)), iou_threshold=0.5
|
|
)
|
|
|
|
assert forward == reverse
|
|
|
|
|
|
def test_empty_candidate_population_is_reported_not_crashed() -> None:
|
|
curve = DetectionMetricsService.precision_recall_curve(
|
|
[], [_reference("r1", box(0, 0, 1, 1))], iou_threshold=0.5
|
|
)
|
|
|
|
assert curve["average_precision"] == 0.0
|
|
assert curve["best_f1"] == 0.0
|
|
assert curve["best_f1_threshold"] is None
|
|
assert curve["points"] == []
|