diff --git a/backend/app/schemas/qa.py b/backend/app/schemas/qa.py index 624ef8ec..a59b4f61 100644 --- a/backend/app/schemas/qa.py +++ b/backend/app/schemas/qa.py @@ -19,8 +19,14 @@ class QaProviderComparisonRequest(BaseModel): class QaProviderComparisonResult(BaseModel): status: str warnings: list[str] = Field(default_factory=list) + # Counts of the population that was actually matched, so that + # ``matches + false_positives == candidate_feature_count`` holds even when + # an area filter or an unparseable geometry removed features. The ``_raw`` + # fields keep the untouched dataset totals visible next to them. candidate_feature_count: int reference_feature_count: int + candidate_feature_count_raw: int | None = None + reference_feature_count_raw: int | None = None matches: int false_positives: int false_negatives: int diff --git a/backend/app/services/detection_metrics_service.py b/backend/app/services/detection_metrics_service.py new file mode 100644 index 00000000..b69fe003 --- /dev/null +++ b/backend/app/services/detection_metrics_service.py @@ -0,0 +1,175 @@ +"""Threshold-independent detection metrics. + +A single precision/recall/F1 triple describes one operating point. Which point +that is depends on the confidence threshold the operator typed, so two models +cannot be compared from it: a conservatively calibrated model looks worse at a +low cut and better at a high one without detecting anything differently. + +This service produces the standard alternative — the full precision/recall +curve over every confidence value that occurs in the run, the average +precision derived from it, and the threshold where F1 actually peaks — using +the same greedy IoU matching rule as the rest of QA so the numbers stay +comparable with the persisted quality checks. +""" + +from __future__ import annotations + +from typing import Any + +from shapely.geometry.base import BaseGeometry +from shapely.strtree import STRtree + +from app.services.qa_service import QaService + + +class DetectionMetricsService: + SUPPORTED_GEOMETRY_TYPES = QaService.SUPPORTED_GEOMETRY_TYPES + + @staticmethod + def _rank_candidates( + candidates: list[tuple[dict[str, Any], BaseGeometry]], + ) -> list[tuple[float, str, dict[str, Any], BaseGeometry]]: + """Order candidates by confidence, highest first, deterministically.""" + + ranked: list[tuple[float, str, dict[str, Any], BaseGeometry]] = [] + for index, (feature, geometry) in enumerate(candidates): + if geometry.is_empty or geometry.geom_type not in DetectionMetricsService.SUPPORTED_GEOMETRY_TYPES: + continue + if geometry.area <= 0: + continue + confidence = QaService._feature_confidence(feature) + identifier = QaService._feature_identifier(feature, "candidate", index) + ranked.append((float(confidence if confidence is not None else 0.0), identifier, feature, geometry)) + ranked.sort(key=lambda item: (-item[0], item[1])) + return ranked + + @staticmethod + def _greedy_hits( + ranked: list[tuple[float, str, dict[str, Any], BaseGeometry]], + references: list[tuple[dict[str, Any], BaseGeometry]], + iou_threshold: float, + ) -> tuple[list[bool], int]: + """Mark each ranked candidate as a hit or a miss, best score first. + + Walking the ranking once and consuming references as they are claimed + is exactly the COCO/PASCAL rule, and it is what makes the result + independent of the order rows came out of the database. + """ + + supported = [ + (index, geometry) + for index, (_, geometry) in enumerate(references) + if geometry.geom_type in DetectionMetricsService.SUPPORTED_GEOMETRY_TYPES + and not geometry.is_empty + and geometry.area > 0 + ] + reference_count = len(supported) + if not supported: + return [False] * len(ranked), 0 + + tree = STRtree([geometry for _, geometry in supported]) + claimed: set[int] = set() + hits: list[bool] = [] + + for _, _, _, geometry in ranked: + best_iou = 0.0 + best_index: int | None = None + for position in sorted(int(value) for value in tree.query(geometry)): + if position in claimed: + continue + _, reference_geometry = supported[position] + intersection_area = geometry.intersection(reference_geometry).area + if intersection_area <= 0: + continue + union_area = geometry.area + reference_geometry.area - intersection_area + if union_area <= 0: + continue + iou = intersection_area / union_area + if iou > best_iou: + best_iou = iou + best_index = position + if best_index is not None and best_iou >= iou_threshold: + claimed.add(best_index) + hits.append(True) + else: + hits.append(False) + + return hits, reference_count + + @staticmethod + def _average_precision(points: list[dict[str, Any]]) -> float: + """Area under the precision/recall curve, with precision made monotone. + + Interpolating precision to its running maximum from the right is the + VOC/COCO convention; without it the sawtooth from individual false + positives shows up as noise in the score. + """ + + if not points: + return 0.0 + + recalls = [0.0] + [point["recall"] for point in points] + precisions = [1.0] + [point["precision"] for point in points] + for index in range(len(precisions) - 2, -1, -1): + precisions[index] = max(precisions[index], precisions[index + 1]) + + area = 0.0 + for index in range(1, len(recalls)): + area += (recalls[index] - recalls[index - 1]) * precisions[index] + return area + + @staticmethod + def precision_recall_curve( + candidates: list[tuple[dict[str, Any], BaseGeometry]], + references: list[tuple[dict[str, Any], BaseGeometry]], + *, + iou_threshold: float, + ) -> dict[str, Any]: + """Sweep every confidence value present and describe the whole curve.""" + + ranked = DetectionMetricsService._rank_candidates(candidates) + hits, reference_count = DetectionMetricsService._greedy_hits(ranked, references, iou_threshold) + + points: list[dict[str, Any]] = [] + true_positives = 0 + for position, hit in enumerate(hits): + if hit: + true_positives += 1 + false_positives = position + 1 - true_positives + precision = true_positives / (position + 1) + recall = true_positives / reference_count if reference_count else 0.0 + f1 = (2 * precision * recall / (precision + recall)) if precision + recall > 0 else 0.0 + points.append( + { + "confidence_threshold": ranked[position][0], + "candidate_count": position + 1, + "true_positives": true_positives, + "false_positives": false_positives, + "false_negatives": max(0, reference_count - true_positives), + "precision": precision, + "recall": recall, + "f1_score": f1, + } + ) + + # Keep one point per distinct threshold: the last one, which is the + # complete tally for everything at or above that confidence. + deduplicated: list[dict[str, Any]] = [] + for point in points: + if deduplicated and deduplicated[-1]["confidence_threshold"] == point["confidence_threshold"]: + deduplicated[-1] = point + else: + deduplicated.append(point) + + best = max(deduplicated, key=lambda point: (point["f1_score"], point["confidence_threshold"]), default=None) + return { + "iou_threshold": iou_threshold, + "reference_count": reference_count, + "candidate_count": len(ranked), + "average_precision": DetectionMetricsService._average_precision(points), + "best_f1": best["f1_score"] if best else 0.0, + "best_f1_threshold": best["confidence_threshold"] if best else None, + "best_f1_precision": best["precision"] if best else None, + "best_f1_recall": best["recall"] if best else None, + "points": deduplicated, + } diff --git a/backend/app/services/detection_qa_service.py b/backend/app/services/detection_qa_service.py index d9ac1a4b..029a2604 100644 --- a/backend/app/services/detection_qa_service.py +++ b/backend/app/services/detection_qa_service.py @@ -168,24 +168,61 @@ class DetectionQaService: clipped_boundary_count=clipped_boundary_count, ) + # A polygon whose area is within this fraction of its own bounding box is + # an axis-aligned rectangle for practical purposes. + RECTANGULAR_AREA_RATIO = 0.99 + + @staticmethod + def candidate_geometry_mode(geometries: list[tuple[dict[str, Any], BaseGeometry]]) -> str: + """Say whether the candidates are detector boxes or true footprints. + + It matters for reading the score. An axis-aligned box can never reach + IoU 1 against a rotated or L-shaped building footprint, so a strict + footprint IoU understates a box detector by a fixed amount that has + nothing to do with whether it found the building. + """ + + polygonal = [ + geometry + for _, geometry in geometries + if geometry.geom_type in {"Polygon", "MultiPolygon"} and geometry.area > 0 + ] + if not polygonal: + return "unknown" + rectangular = sum( + 1 + for geometry in polygonal + if geometry.area / geometry.envelope.area >= DetectionQaService.RECTANGULAR_AREA_RATIO + ) + return "axis_aligned_boxes" if rectangular == len(polygonal) else "footprint_polygons" + @staticmethod def box_to_footprint_diagnostics( strict_evidence: QaMatchEvidence, envelope_evidence: QaMatchEvidence, *, iou_threshold: float, + candidate_geometry_mode: str = "unknown", ) -> dict[str, Any]: envelope_metrics = DetectionQaService._metrics(envelope_evidence) - return { + diagnostics = { "diagnostic_only": True, "canonical_method": "candidate_polygon_vs_reference_footprint_iou", "diagnostic_method": "candidate_polygon_vs_reference_envelope_iou", "iou_threshold": iou_threshold, + "candidate_geometry_mode": candidate_geometry_mode, "strict_matches": strict_evidence.matches, "envelope_matches": envelope_evidence.matches, "possible_box_to_footprint_mismatch_count": max(0, envelope_evidence.matches - strict_evidence.matches), **envelope_metrics, } + if candidate_geometry_mode == "axis_aligned_boxes": + diagnostics["interpretation"] = ( + "Candidates are axis-aligned detector boxes. The strict footprint IoU therefore has a " + "ceiling below 1 for rotated or non-rectangular buildings; the envelope figures isolate " + "detection quality from that shape mismatch." + ) + return diagnostics @staticmethod def _metrics(evidence: QaMatchEvidence) -> dict[str, Any]: diff --git a/backend/app/services/qa_service.py b/backend/app/services/qa_service.py index 864456de..3957794f 100644 --- a/backend/app/services/qa_service.py +++ b/backend/app/services/qa_service.py @@ -123,6 +123,51 @@ class QaService: return str(value) return f"{fallback_prefix}-{index + 1}" + @staticmethod + def _feature_confidence(feature: dict[str, Any]) -> float | None: + """Read a detector confidence from a QA feature, if the source has one. + + Vector-vs-vector comparisons have no confidence at all; those fall back + to identity ordering so the result stays reproducible either way. + """ + + candidates: list[Any] = [feature.get("confidence")] + properties = feature.get("properties") + if isinstance(properties, dict): + candidates.append(properties.get("confidence")) + for value in candidates: + if value is None or isinstance(value, bool): + continue + try: + confidence = float(value) + except (TypeError, ValueError): + continue + if confidence == confidence: # reject NaN + return confidence + return None + + @staticmethod + def _candidate_match_order( + source_supported: list[tuple[int, dict[str, Any], BaseGeometry]], + ) -> list[tuple[int, dict[str, Any], BaseGeometry]]: + """Order candidates the way detection benchmarks do: best score first. + + Greedy IoU matching gives the reference to whichever candidate is + offered first, so the input order decides both the score and which + geometry an operator sees as false-positive evidence. Database row + order is not a defensible answer to that question — every detection in + a run shares one transaction timestamp — so candidates are ranked by + confidence, with feature identity as a stable tiebreaker. + """ + + def order_key(entry: tuple[int, dict[str, Any], BaseGeometry]) -> tuple[float, str, int]: + index, feature, _ = entry + confidence = QaService._feature_confidence(feature) + identifier = QaService._feature_identifier(feature, "candidate", index) + return (-(confidence if confidence is not None else 0.0), identifier, index) + + return sorted(source_supported, key=order_key) + @staticmethod def _match_io_u_evidence( source_geometries: list[tuple[dict[str, Any], BaseGeometry]], @@ -153,7 +198,7 @@ class QaService: unsupported=True, false_positive_evidence=[ {"candidate_feature_id": QaService._feature_identifier(feature, "candidate", source_index)} - for source_index, feature, _ in source_supported + for source_index, feature, _ in QaService._candidate_match_order(source_supported) ], false_negative_evidence=[ {"reference_feature_id": QaService._feature_identifier(feature, "reference", reference_index)} @@ -167,7 +212,7 @@ class QaService: } evidence = QaMatchEvidence(warnings=[f"Unsupported geometry types: {unsupported}"] if unsupported else [], unsupported=bool(unsupported)) - for source_index, source_feature, source_geom in source_supported: + for source_index, source_feature, source_geom in QaService._candidate_match_order(source_supported): source_feature_id = QaService._feature_identifier(source_feature, "candidate", source_index) if source_geom.area <= 0: evidence.false_positives += 1 @@ -278,6 +323,9 @@ class QaService: dataset_ids=(candidate_dataset_id, reference_dataset_id), ) + candidate_feature_count_raw = len(candidate_payload.get("features", [])) if isinstance(candidate_payload, dict) else 0 + reference_feature_count_raw = len(reference_payload.get("features", [])) if isinstance(reference_payload, dict) else 0 + if area_geometry is not None: candidate_geometries = QaService._apply_area_filter(candidate_geometries, area_geometry, dataset_id=candidate_dataset.id) reference_geometries = QaService._apply_area_filter(reference_geometries, area_geometry, dataset_id=reference_dataset.id) @@ -288,8 +336,11 @@ class QaService: iou_threshold, ) - candidate_feature_count = len(candidate_payload.get("features", [])) if isinstance(candidate_payload, dict) else 0 - reference_feature_count = len(reference_payload.get("features", [])) if isinstance(reference_payload, dict) else 0 + # Report the population the metrics were computed over, not the raw + # dataset totals: with an area filter the two differ, and a count that + # disagrees with matches + false positives is unreadable as evidence. + candidate_feature_count = len(candidate_geometries) + reference_feature_count = len(reference_geometries) mean_iou = None if not evidence.match_iou_values else sum(evidence.match_iou_values) / len(evidence.match_iou_values) precision = None @@ -310,6 +361,8 @@ class QaService: warnings=_extract_crs_warnings(candidate_dataset, reference_dataset) + evidence.warnings, candidate_feature_count=candidate_feature_count, reference_feature_count=reference_feature_count, + candidate_feature_count_raw=candidate_feature_count_raw, + reference_feature_count_raw=reference_feature_count_raw, matches=evidence.matches, false_positives=evidence.false_positives, false_negatives=evidence.false_negatives, diff --git a/backend/tests/test_detection_precision_recall_curve.py b/backend/tests/test_detection_precision_recall_curve.py new file mode 100644 index 00000000..d3ad66f2 --- /dev/null +++ b/backend/tests/test_detection_precision_recall_curve.py @@ -0,0 +1,116 @@ +"""A single F1 at one arbitrary confidence cut cannot compare two models. + +Reporting F1 at whichever threshold the operator happened to type makes two +models look better or worse depending on their calibration rather than their +detection quality. The sweep produces the standard curve instead: precision +and recall at every operating point, average precision, and the threshold +where F1 actually peaks. +""" + +from __future__ import annotations + +import pytest +from shapely.geometry import box + +from app.services.detection_metrics_service import DetectionMetricsService + + +def _candidate(name: str, geometry, confidence: float): + return ({"id": name, "confidence": confidence}, geometry) + + +def _reference(name: str, geometry): + return ({"id": name}, geometry) + + +def test_perfect_detector_reaches_average_precision_one() -> None: + references = [_reference("r1", box(0, 0, 1, 1)), _reference("r2", box(5, 5, 6, 6))] + candidates = [ + _candidate("c1", box(0, 0, 1, 1), 0.9), + _candidate("c2", box(5, 5, 6, 6), 0.8), + ] + + curve = DetectionMetricsService.precision_recall_curve(candidates, references, iou_threshold=0.5) + + assert curve["average_precision"] == pytest.approx(1.0) + assert curve["best_f1"] == pytest.approx(1.0) + assert curve["reference_count"] == 2 + + +def test_low_confidence_false_positive_is_only_penalised_below_its_threshold() -> None: + references = [_reference("r1", box(0, 0, 1, 1))] + candidates = [ + _candidate("hit", box(0, 0, 1, 1), 0.9), + _candidate("junk", box(20, 20, 21, 21), 0.2), + ] + + curve = DetectionMetricsService.precision_recall_curve(candidates, references, iou_threshold=0.5) + + # Cutting at 0.2 admits the junk box, so precision there is 0.5. + low = next(point for point in curve["points"] if point["confidence_threshold"] == pytest.approx(0.2)) + assert low["precision"] == pytest.approx(0.5) + assert low["recall"] == pytest.approx(1.0) + + # The optimum simply drops it. + assert curve["best_f1"] == pytest.approx(1.0) + assert curve["best_f1_threshold"] == pytest.approx(0.9) + # AP is computed over the ranking, so one trailing false positive after + # full recall does not reduce it. + assert curve["average_precision"] == pytest.approx(1.0) + + +def test_ranking_quality_is_visible_in_average_precision() -> None: + """A detector that ranks its mistake above its hit scores worse.""" + + references = [_reference("r1", box(0, 0, 1, 1))] + good = DetectionMetricsService.precision_recall_curve( + [_candidate("hit", box(0, 0, 1, 1), 0.9), _candidate("junk", box(20, 20, 21, 21), 0.1)], + references, + iou_threshold=0.5, + ) + bad = DetectionMetricsService.precision_recall_curve( + [_candidate("hit", box(0, 0, 1, 1), 0.1), _candidate("junk", box(20, 20, 21, 21), 0.9)], + references, + iou_threshold=0.5, + ) + + assert good["average_precision"] > bad["average_precision"] + assert bad["average_precision"] == pytest.approx(0.5) + + +def test_missed_reference_caps_recall_and_average_precision() -> None: + references = [_reference("r1", box(0, 0, 1, 1)), _reference("r2", box(9, 9, 10, 10))] + candidates = [_candidate("hit", box(0, 0, 1, 1), 0.9)] + + curve = DetectionMetricsService.precision_recall_curve(candidates, references, iou_threshold=0.5) + + assert curve["points"][0]["recall"] == pytest.approx(0.5) + assert curve["average_precision"] == pytest.approx(0.5) + assert curve["best_f1"] == pytest.approx(2 / 3) + + +def test_curve_is_independent_of_input_order() -> None: + references = [_reference("r1", box(0, 0, 1, 1)), _reference("r2", box(5, 5, 6, 6))] + candidates = [ + _candidate("c1", box(0, 0, 1, 1), 0.9), + _candidate("c2", box(5, 5, 6, 6), 0.4), + _candidate("c3", box(30, 30, 31, 31), 0.6), + ] + + forward = DetectionMetricsService.precision_recall_curve(candidates, references, iou_threshold=0.5) + reverse = DetectionMetricsService.precision_recall_curve( + list(reversed(candidates)), list(reversed(references)), iou_threshold=0.5 + ) + + assert forward == reverse + + +def test_empty_candidate_population_is_reported_not_crashed() -> None: + curve = DetectionMetricsService.precision_recall_curve( + [], [_reference("r1", box(0, 0, 1, 1))], iou_threshold=0.5 + ) + + assert curve["average_precision"] == 0.0 + assert curve["best_f1"] == 0.0 + assert curve["best_f1_threshold"] is None + assert curve["points"] == [] diff --git a/backend/tests/test_qa_matching_determinism.py b/backend/tests/test_qa_matching_determinism.py new file mode 100644 index 00000000..6d59330a --- /dev/null +++ b/backend/tests/test_qa_matching_determinism.py @@ -0,0 +1,88 @@ +"""QA matching must be reproducible and must credit the best candidate. + +The greedy IoU matcher decides which candidate is reported as a match and +which becomes false-positive evidence for an operator. If that decision +depends on database row order, the same run produces different scores and +points reviewers at the wrong geometry. +""" + +from __future__ import annotations + +from shapely.geometry import box + +from app.services.qa_service import QaService + + +REFERENCE = [({"id": "R1"}, box(0.0, 0.0, 10.0, 10.0))] + +# Two detections of the same building. ``sloppy`` is far too tall (IoU 0.51), +# ``accurate`` is nearly exact (IoU 0.96). +SLOPPY = box(0.0, 0.0, 10.0, 19.5) +ACCURATE = box(0.0, 0.0, 10.0, 10.4) + + +def _candidates(order: list[tuple[str, float]]) -> list[tuple[dict, object]]: + geometries = {"sloppy": SLOPPY, "accurate": ACCURATE} + return [ + ({"id": name, "confidence": confidence}, geometries[name]) + for name, confidence in order + ] + + +def test_matching_is_independent_of_candidate_row_order() -> None: + forward = QaService._match_io_u_evidence( + _candidates([("sloppy", 0.42), ("accurate", 0.91)]), REFERENCE, 0.5 + ) + reverse = QaService._match_io_u_evidence( + _candidates([("accurate", 0.91), ("sloppy", 0.42)]), REFERENCE, 0.5 + ) + + assert forward.matches == reverse.matches + assert forward.false_positives == reverse.false_positives + assert forward.false_negatives == reverse.false_negatives + assert forward.match_iou_values == reverse.match_iou_values + assert forward.match_evidence == reverse.match_evidence + assert forward.false_positive_evidence == reverse.false_positive_evidence + + +def test_highest_confidence_candidate_claims_the_reference() -> None: + evidence = QaService._match_io_u_evidence( + _candidates([("sloppy", 0.42), ("accurate", 0.91)]), REFERENCE, 0.5 + ) + + assert evidence.matches == 1 + assert evidence.match_evidence[0]["candidate_feature_id"] == "accurate" + assert evidence.false_positive_evidence == [{"candidate_feature_id": "sloppy"}] + assert evidence.match_iou_values[0] > 0.9 + + +def test_matching_without_confidence_is_still_deterministic() -> None: + """Vector-vs-vector QA has no confidence; identity keeps it reproducible.""" + + left = [ + ({"id": "b-second"}, SLOPPY), + ({"id": "a-first"}, ACCURATE), + ] + right = list(reversed(left)) + + assert QaService._match_io_u_evidence(left, REFERENCE, 0.5).match_evidence == ( + QaService._match_io_u_evidence(right, REFERENCE, 0.5).match_evidence + ) + + +def test_evidence_is_ordered_by_confidence_for_review() -> None: + evidence = QaService._match_io_u_evidence( + [ + ({"id": "low", "confidence": 0.30}, box(30.0, 30.0, 31.0, 31.0)), + ({"id": "high", "confidence": 0.95}, box(40.0, 40.0, 41.0, 41.0)), + ({"id": "mid", "confidence": 0.60}, box(50.0, 50.0, 51.0, 51.0)), + ], + REFERENCE, + 0.5, + ) + + assert [item["candidate_feature_id"] for item in evidence.false_positive_evidence] == [ + "high", + "mid", + "low", + ]