JensandClaude Opus 5
2b968b74cf
The greedy IoU matcher gave a reference to whichever candidate was offered first. Row order decided that, and every detection in a run shares one transaction timestamp, so ordering by created_at left the assignment undefined: the same QA run over the same data produced different mean IoU, and the geometry shown to a reviewer as a false positive could be the better of two detections. Candidates are now ranked by confidence with feature identity as tiebreaker, which is also the COCO/PASCAL rule. A single precision/recall/F1 triple describes one operating point, so two models cannot be compared from it: a conservatively calibrated model looks worse at a low confidence cut and better at a high one without detecting anything differently. DetectionMetricsService adds the full curve, average precision and the threshold where F1 actually peaks. Also: - report the population the metrics were computed over, so matches + false_positives equals candidate_feature_count even under an area filter; raw dataset totals move to the _raw fields; - state whether candidates are axis-aligned boxes or footprint polygons. A box can never reach IoU 1 against a rotated building, so the strict score has a ceiling that has nothing to do with detection quality. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
89 lines
3.0 KiB
Python
89 lines
3.0 KiB
Python
"""QA matching must be reproducible and must credit the best candidate.
|
|
|
|
The greedy IoU matcher decides which candidate is reported as a match and
|
|
which becomes false-positive evidence for an operator. If that decision
|
|
depends on database row order, the same run produces different scores and
|
|
points reviewers at the wrong geometry.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from shapely.geometry import box
|
|
|
|
from app.services.qa_service import QaService
|
|
|
|
|
|
REFERENCE = [({"id": "R1"}, box(0.0, 0.0, 10.0, 10.0))]
|
|
|
|
# Two detections of the same building. ``sloppy`` is far too tall (IoU 0.51),
|
|
# ``accurate`` is nearly exact (IoU 0.96).
|
|
SLOPPY = box(0.0, 0.0, 10.0, 19.5)
|
|
ACCURATE = box(0.0, 0.0, 10.0, 10.4)
|
|
|
|
|
|
def _candidates(order: list[tuple[str, float]]) -> list[tuple[dict, object]]:
|
|
geometries = {"sloppy": SLOPPY, "accurate": ACCURATE}
|
|
return [
|
|
({"id": name, "confidence": confidence}, geometries[name])
|
|
for name, confidence in order
|
|
]
|
|
|
|
|
|
def test_matching_is_independent_of_candidate_row_order() -> None:
|
|
forward = QaService._match_io_u_evidence(
|
|
_candidates([("sloppy", 0.42), ("accurate", 0.91)]), REFERENCE, 0.5
|
|
)
|
|
reverse = QaService._match_io_u_evidence(
|
|
_candidates([("accurate", 0.91), ("sloppy", 0.42)]), REFERENCE, 0.5
|
|
)
|
|
|
|
assert forward.matches == reverse.matches
|
|
assert forward.false_positives == reverse.false_positives
|
|
assert forward.false_negatives == reverse.false_negatives
|
|
assert forward.match_iou_values == reverse.match_iou_values
|
|
assert forward.match_evidence == reverse.match_evidence
|
|
assert forward.false_positive_evidence == reverse.false_positive_evidence
|
|
|
|
|
|
def test_highest_confidence_candidate_claims_the_reference() -> None:
|
|
evidence = QaService._match_io_u_evidence(
|
|
_candidates([("sloppy", 0.42), ("accurate", 0.91)]), REFERENCE, 0.5
|
|
)
|
|
|
|
assert evidence.matches == 1
|
|
assert evidence.match_evidence[0]["candidate_feature_id"] == "accurate"
|
|
assert evidence.false_positive_evidence == [{"candidate_feature_id": "sloppy"}]
|
|
assert evidence.match_iou_values[0] > 0.9
|
|
|
|
|
|
def test_matching_without_confidence_is_still_deterministic() -> None:
|
|
"""Vector-vs-vector QA has no confidence; identity keeps it reproducible."""
|
|
|
|
left = [
|
|
({"id": "b-second"}, SLOPPY),
|
|
({"id": "a-first"}, ACCURATE),
|
|
]
|
|
right = list(reversed(left))
|
|
|
|
assert QaService._match_io_u_evidence(left, REFERENCE, 0.5).match_evidence == (
|
|
QaService._match_io_u_evidence(right, REFERENCE, 0.5).match_evidence
|
|
)
|
|
|
|
|
|
def test_evidence_is_ordered_by_confidence_for_review() -> None:
|
|
evidence = QaService._match_io_u_evidence(
|
|
[
|
|
({"id": "low", "confidence": 0.30}, box(30.0, 30.0, 31.0, 31.0)),
|
|
({"id": "high", "confidence": 0.95}, box(40.0, 40.0, 41.0, 41.0)),
|
|
({"id": "mid", "confidence": 0.60}, box(50.0, 50.0, 51.0, 51.0)),
|
|
],
|
|
REFERENCE,
|
|
0.5,
|
|
)
|
|
|
|
assert [item["candidate_feature_id"] for item in evidence.false_positive_evidence] == [
|
|
"high",
|
|
"mid",
|
|
"low",
|
|
]
|