rank model variants on average precision, and say when they are not comparable

The workbench ranks model variants by a stored F1, each measured at that
variant's own confidence threshold. A conservatively calibrated detector then
looks worse than a liberal one without detecting anything differently: the
number says as much about the threshold as about the model. POST
/detection/runs/compare ranks on average precision instead, which describes the
whole ranking a model produced, and keeps each run's own-threshold F1 visible
next to it so the difference between the two readings is auditable.

Comparability comes before the ranking. Runs over different source rasters,
scored against different references, without a proven inference footprint, or
covering a different evaluated population are not alternatives to one another,
and no metric makes them so. The report names which of those applies and still
returns the numbers — they are simply not a ranking.

Each run is scored through the same QA path the workbench uses, so a comparison
and the persisted quality checks cannot drift apart.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

backend/app/api/routes/detection.py

@@ -10,6 +10,8 @@ from app.schemas import (
     AnalysisQaResponse,
     DetectionListResponse,
     DetectionModelsResponse,
+    DetectionComparisonRequest,
+    DetectionComparisonResponse,
     DetectionQaRequest,
     DetectionRead,
     DetectionRunListResponse,
@@ -22,6 +24,7 @@ from app.schemas import (
     ModelAssetListResponse,
     YoloPreflightResponse,
 )
+from app.services.detection_comparison_service import DetectionComparisonService
 from app.services.detection_service import DetectionService
 from app.services.model_asset_catalog_service import ModelAssetCatalogService
 from app.services.model_registry_service import ModelRegistryService
@@ -232,6 +235,27 @@ def get_dataset_detection_geojson(
     )
 
 
+@router.post("/runs/compare", response_model=Envelope[DetectionComparisonResponse])
+def compare_detection_runs(payload: DetectionComparisonRequest, db: Session = Depends(get_db)) -> dict:
+    """Rank several runs against one reference on average precision.
+
+    The workbench ranks model variants by a stored F1 measured at each
+    variant's own confidence threshold, which orders the thresholds as much as
+    the models. Average precision describes the whole ranking a model produced.
+    Comparability is reported first: runs over different rasters, different
+    references or different inference coverage are not alternatives.
+    """
+
+    return envelope(
+        DetectionComparisonService.compare_runs(
+            db,
+            analysis_run_ids=payload.analysis_run_ids,
+            reference_dataset_id=payload.reference_dataset_id,
+            iou_threshold=payload.iou_threshold,
+        )
+    )
+
+
 @router.post(
     "/runs/{analysis_run_id}/qa/reference",
     response_model=Envelope[AnalysisQaResponse],

backend/app/schemas/__init__.py

@@ -56,6 +56,8 @@ from .detection import (
     DetectionRunListResponse,
     DetectionRunRead,
     DetectionRunRequest,
+    DetectionComparisonRequest,
+    DetectionComparisonResponse,
     DetectionRunResponse,
     ModelAssetListResponse,
     ModelAssetRead,
@@ -238,6 +240,8 @@ __all__ = [
     "DetectionRunListResponse",
     "DetectionRunRead",
     "DetectionRunRequest",
+    "DetectionComparisonRequest",
+    "DetectionComparisonResponse",
     "DetectionRunResponse",
     "ModelAssetListResponse",
     "ModelAssetRead",

backend/app/schemas/detection.py

@@ -78,6 +78,25 @@ class DetectionQaRequest(BaseModel):
     calibration_thresholds: list[float] = Field(default_factory=list, max_length=32)
 
 
+class DetectionComparisonRequest(BaseModel):
+    """Place several runs side by side against one reference."""
+
+    analysis_run_ids: list[UUID] = Field(min_length=2, max_length=12)
+    reference_dataset_id: UUID
+    iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0)
+
+
+class DetectionComparisonResponse(BaseModel):
+    reference_dataset_id: UUID
+    iou_threshold: float
+    # Whether these runs answer the same question at all, and why not if they
+    # do not. Numbers from incomparable runs are reported but never ranked as
+    # if they were alternatives.
+    comparability: dict
+    ranking_metric: str
+    rows: list[dict]
+
+
 class DetectionRunResponse(BaseModel):
     model_config = ConfigDict(protected_namespaces=())
 

backend/app/services/detection_comparison_service.py

@@ -0,0 +1,179 @@
+"""Placing two detection runs side by side, honestly.
+
+The workbench ranks model variants by a stored F1, each measured at that
+variant's own confidence threshold. That number says as much about the
+threshold as about the model: a conservatively calibrated detector looks worse
+than a liberal one without detecting anything differently. Average precision
+describes the whole ranking the model produced and is the comparable figure.
+
+Comparability comes first, though. Two runs over different rasters, scored
+against different references, or covering different ground are not two answers
+to one question, and no metric makes them so.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from app.core.errors import AppError
+
+
+class DetectionComparisonService:
+    # A run whose inference footprint was never established cannot be placed
+    # beside one that was: their recalls have different denominators.
+    PROVEN_COVERAGE_MODES = ("persisted_tile_manifest_union",)
+
+    @staticmethod
+    def assess_comparability(entries: list[dict[str, Any]]) -> dict[str, Any]:
+        """Say whether these runs answer the same question, and why not if they don't."""
+
+        if len(entries) < 2:
+            raise AppError(
+                code="DETECTION_COMPARISON_NEEDS_TWO_RUNS",
+                message="Comparing detection models requires at least two runs",
+                status_code=400,
+            )
+
+        reasons: list[str] = []
+        source_rasters = {str(entry.get("dataset_id")) for entry in entries}
+        if len(source_rasters) > 1:
+            reasons.append("different_source_raster")
+
+        references = {str(entry.get("reference_dataset_id")) for entry in entries}
+        if len(references) > 1:
+            reasons.append("different_reference_dataset")
+
+        if any(
+            str(entry.get("coverage_mode")) not in DetectionComparisonService.PROVEN_COVERAGE_MODES
+            for entry in entries
+        ):
+            reasons.append("coverage_not_proven")
+
+        populations = {int(entry.get("reference_evaluated_count") or 0) for entry in entries}
+        if len(populations) > 1:
+            reasons.append("different_evaluated_population")
+
+        return {
+            "comparable": not reasons,
+            "blocking_reasons": reasons,
+            "source_raster_count": len(source_rasters),
+            "reference_dataset_count": len(references),
+            "evaluated_population_counts": sorted(populations),
+        }
+
+    @staticmethod
+    def rank(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        """Order runs by average precision, stating the margin and any tie.
+
+        Ranking on the F1 each run happened to be read at would order the
+        thresholds, not the models.
+        """
+
+        ordered = sorted(
+            rows,
+            key=lambda row: (-(row.get("average_precision") or 0.0), str(row.get("model_asset_id") or "")),
+        )
+        if not ordered:
+            return []
+
+        leader = ordered[0].get("average_precision") or 0.0
+        tied_count = sum(1 for row in ordered if (row.get("average_precision") or 0.0) == leader)
+        runner_up = (ordered[1].get("average_precision") or 0.0) if len(ordered) > 1 else leader
+
+        ranked: list[dict[str, Any]] = []
+        for row in ordered:
+            average_precision = row.get("average_precision") or 0.0
+            is_leader = average_precision == leader
+            ranked.append(
+                {
+                    **row,
+                    "rank": 1 if is_leader else 1 + sum(
+                        1 for other in ordered if (other.get("average_precision") or 0.0) > average_precision
+                    ),
+                    # Distance behind the best run; zero for the leader itself.
+                    "average_precision_gap": leader - average_precision,
+                    "tied": is_leader and tied_count > 1,
+                    # Only the leader has a lead; stating it on every row would
+                    # invite reading a follower's gap as an advantage.
+                    "lead_over_next": (leader - runner_up) if is_leader and tied_count == 1 else None,
+                }
+            )
+        return ranked
+
+
+    @staticmethod
+    def compare_runs(
+        db,
+        *,
+        analysis_run_ids: list,
+        reference_dataset_id,
+        iou_threshold: float = 0.5,
+    ) -> dict[str, Any]:
+        """Score several runs against one reference and rank them on AP.
+
+        Each run is scored through the same QA path the workbench uses, so the
+        comparison and the persisted quality checks cannot drift apart.
+        """
+
+        # Lazy: detection_service imports this module's siblings at load.
+        from app.models import AnalysisRun
+        from app.services.detection_service import DetectionService
+
+        if len(set(analysis_run_ids)) < 2:
+            raise AppError(
+                code="DETECTION_COMPARISON_NEEDS_TWO_RUNS",
+                message="Comparing detection models requires at least two distinct runs",
+                status_code=400,
+            )
+
+        entries: list[dict[str, Any]] = []
+        rows: list[dict[str, Any]] = []
+        for analysis_run_id in analysis_run_ids:
+            result = DetectionService.compare_detections_with_reference(
+                db,
+                analysis_run_id=analysis_run_id,
+                reference_dataset_id=reference_dataset_id,
+                iou_threshold=iou_threshold,
+            )
+            run = db.get(AnalysisRun, analysis_run_id)
+            parameters = (run.parameters_json if run and isinstance(run.parameters_json, dict) else {}) or {}
+            coverage = result.get("coverage") if isinstance(result.get("coverage"), dict) else {}
+            curve = result.get("precision_recall_curve") or {}
+
+            entries.append(
+                {
+                    "analysis_run_id": analysis_run_id,
+                    "dataset_id": getattr(run, "dataset_id", None),
+                    "model_id": getattr(run, "model_name", None),
+                    "model_asset_id": parameters.get("model_asset_id"),
+                    "reference_dataset_id": reference_dataset_id,
+                    "coverage_mode": coverage.get("mode"),
+                    "reference_evaluated_count": coverage.get("reference_evaluated_count"),
+                }
+            )
+            rows.append(
+                {
+                    "analysis_run_id": str(analysis_run_id),
+                    "quality_check_id": result.get("quality_check_id"),
+                    "model_id": getattr(run, "model_name", None),
+                    "model_asset_id": parameters.get("model_asset_id"),
+                    "run_confidence_threshold": parameters.get("confidence_threshold"),
+                    "average_precision": curve.get("average_precision"),
+                    "best_f1": curve.get("best_f1"),
+                    "best_f1_threshold": curve.get("best_f1_threshold"),
+                    # The figure the workbench used to rank on, kept visible so
+                    # the difference between the two readings is auditable.
+                    "f1_at_run_threshold": result.get("f1_score"),
+                    "precision_at_run_threshold": result.get("precision"),
+                    "recall_at_run_threshold": result.get("recall"),
+                }
+            )
+
+        comparability = DetectionComparisonService.assess_comparability(entries)
+        return {
+            "reference_dataset_id": str(reference_dataset_id),
+            "iou_threshold": iou_threshold,
+            "comparability": comparability,
+            "ranking_metric": "average_precision",
+            "rows": DetectionComparisonService.rank(rows),
+        }

backend/tests/test_detection_model_comparison.py

@@ -0,0 +1,249 @@
+"""Comparing two models must not compare two different questions.
+
+The workbench ranks model variants by a stored F1, each measured at that
+model's own confidence threshold. A conservatively calibrated model then looks
+worse than a liberal one without detecting anything differently — the number
+says as much about the threshold as about the model.
+
+It also says nothing about whether the two runs are comparable at all. Two runs
+over different rasters, or with different inference coverage, produce numbers
+that cannot be placed side by side however they were measured.
+"""
+
+from __future__ import annotations
+
+from uuid import uuid4
+
+import pytest
+
+from app.core.errors import AppError
+from app.services.detection_comparison_service import DetectionComparisonService
+
+
+def _entry(
+    *,
+    dataset_id,
+    model_asset_id: str = "asset-a",
+    coverage_mode: str = "persisted_tile_manifest_union",
+    reference_dataset_id=None,
+    reference_evaluated: int = 100,
+):
+    return {
+        "analysis_run_id": uuid4(),
+        "dataset_id": dataset_id,
+        "model_id": "yolo-configured",
+        "model_asset_id": model_asset_id,
+        "reference_dataset_id": reference_dataset_id or uuid4(),
+        "coverage_mode": coverage_mode,
+        "reference_evaluated_count": reference_evaluated,
+    }
+
+
+class TestComparability:
+    def test_runs_over_the_same_raster_and_reference_are_comparable(self) -> None:
+        dataset_id, reference_id = uuid4(), uuid4()
+        entries = [
+            _entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="a"),
+            _entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="b"),
+        ]
+
+        report = DetectionComparisonService.assess_comparability(entries)
+
+        assert report["comparable"] is True
+        assert report["blocking_reasons"] == []
+
+    def test_runs_over_different_rasters_are_not_comparable(self) -> None:
+        reference_id = uuid4()
+        entries = [
+            _entry(dataset_id=uuid4(), reference_dataset_id=reference_id),
+            _entry(dataset_id=uuid4(), reference_dataset_id=reference_id),
+        ]
+
+        report = DetectionComparisonService.assess_comparability(entries)
+
+        assert report["comparable"] is False
+        assert "different_source_raster" in report["blocking_reasons"]
+
+    def test_runs_scored_against_different_references_are_not_comparable(self) -> None:
+        dataset_id = uuid4()
+        entries = [_entry(dataset_id=dataset_id), _entry(dataset_id=dataset_id)]
+
+        report = DetectionComparisonService.assess_comparability(entries)
+
+        assert report["comparable"] is False
+        assert "different_reference_dataset" in report["blocking_reasons"]
+
+    def test_a_run_without_proven_coverage_is_flagged(self) -> None:
+        dataset_id, reference_id = uuid4(), uuid4()
+        entries = [
+            _entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="a"),
+            _entry(
+                dataset_id=dataset_id,
+                reference_dataset_id=reference_id,
+                model_asset_id="b",
+                coverage_mode="unbounded_no_manifest",
+            ),
+        ]
+
+        report = DetectionComparisonService.assess_comparability(entries)
+
+        assert report["comparable"] is False
+        assert "coverage_not_proven" in report["blocking_reasons"]
+
+    def test_a_differing_evaluated_population_is_flagged(self) -> None:
+        """Same raster and reference, but the runs did not see the same ground."""
+
+        dataset_id, reference_id = uuid4(), uuid4()
+        entries = [
+            _entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="a", reference_evaluated=100),
+            _entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="b", reference_evaluated=60),
+        ]
+
+        report = DetectionComparisonService.assess_comparability(entries)
+
+        assert report["comparable"] is False
+        assert "different_evaluated_population" in report["blocking_reasons"]
+
+    def test_one_run_is_never_a_comparison(self) -> None:
+        with pytest.raises(AppError) as exc_info:
+            DetectionComparisonService.assess_comparability([_entry(dataset_id=uuid4())])
+
+        assert exc_info.value.code == "DETECTION_COMPARISON_NEEDS_TWO_RUNS"
+
+
+class TestRanking:
+    def _row(self, name: str, *, ap: float, best_f1: float, threshold_f1: float):
+        return {
+            "model_asset_id": name,
+            "average_precision": ap,
+            "best_f1": best_f1,
+            "best_f1_threshold": 0.3,
+            "f1_at_run_threshold": threshold_f1,
+        }
+
+    def test_ranking_uses_average_precision_not_the_run_threshold_f1(self) -> None:
+        rows = [
+            self._row("liberal", ap=0.55, best_f1=0.60, threshold_f1=0.61),
+            self._row("conservative", ap=0.72, best_f1=0.71, threshold_f1=0.44),
+        ]
+
+        ranked = DetectionComparisonService.rank(rows)
+
+        # The conservative model detects better; its stored F1 only looked worse
+        # because it was measured at a stricter cut.
+        assert [row["model_asset_id"] for row in ranked] == ["conservative", "liberal"]
+        assert ranked[0]["rank"] == 1
+
+    def test_every_row_states_how_far_behind_the_leader_it_is(self) -> None:
+        rows = [
+            self._row("a", ap=0.72, best_f1=0.71, threshold_f1=0.44),
+            self._row("b", ap=0.55, best_f1=0.60, threshold_f1=0.61),
+        ]
+
+        ranked = DetectionComparisonService.rank(rows)
+
+        assert ranked[0]["average_precision_gap"] == pytest.approx(0.0)
+        assert ranked[1]["average_precision_gap"] == pytest.approx(0.17)
+
+    def test_only_the_leader_carries_a_lead(self) -> None:
+        """A follower's gap must never be readable as an advantage."""
+
+        rows = [
+            self._row("a", ap=0.72, best_f1=0.71, threshold_f1=0.44),
+            self._row("b", ap=0.55, best_f1=0.60, threshold_f1=0.61),
+        ]
+
+        ranked = DetectionComparisonService.rank(rows)
+
+        assert ranked[0]["lead_over_next"] == pytest.approx(0.17)
+        assert ranked[1]["lead_over_next"] is None
+
+    def test_a_tied_leader_claims_no_lead(self) -> None:
+        rows = [
+            self._row("a", ap=0.6, best_f1=0.6, threshold_f1=0.6),
+            self._row("b", ap=0.6, best_f1=0.5, threshold_f1=0.5),
+        ]
+
+        assert all(row["lead_over_next"] is None for row in DetectionComparisonService.rank(rows))
+
+    def test_a_tie_is_reported_as_a_tie_rather_than_an_arbitrary_winner(self) -> None:
+        rows = [
+            self._row("a", ap=0.6, best_f1=0.6, threshold_f1=0.6),
+            self._row("b", ap=0.6, best_f1=0.5, threshold_f1=0.5),
+        ]
+
+        ranked = DetectionComparisonService.rank(rows)
+
+        assert [row["rank"] for row in ranked] == [1, 1]
+        assert all(row["tied"] for row in ranked)
+
+
+class TestComparingRealRuns:
+    """Through the same QA path the workbench uses, so the two cannot drift."""
+
+    def test_two_runs_are_scored_ranked_and_judged_comparable(self, monkeypatch) -> None:
+        from app.services.detection_comparison_service import DetectionComparisonService as Service
+        from app.services.detection_service import DetectionService
+
+        dataset_id, reference_id = uuid4(), uuid4()
+        run_a, run_b = uuid4(), uuid4()
+
+        class _Run:
+            def __init__(self, asset: str, threshold: float) -> None:
+                self.dataset_id = dataset_id
+                self.model_name = "yolo-configured"
+                self.parameters_json = {"model_asset_id": asset, "confidence_threshold": threshold}
+
+        runs = {run_a: _Run("liberal", 0.15), run_b: _Run("conservative", 0.45)}
+        scores = {
+            run_a: {"average_precision": 0.55, "best_f1": 0.60, "f1": 0.61},
+            run_b: {"average_precision": 0.72, "best_f1": 0.71, "f1": 0.44},
+        }
+
+        class _Session:
+            def get(self, _model, item_id):
+                return runs.get(item_id)
+
+        def fake_qa(_db, *, analysis_run_id, reference_dataset_id, iou_threshold, **_kwargs):
+            score = scores[analysis_run_id]
+            return {
+                "quality_check_id": str(uuid4()),
+                "f1_score": score["f1"],
+                "precision": 0.6,
+                "recall": 0.6,
+                "coverage": {"mode": "persisted_tile_manifest_union", "reference_evaluated_count": 100},
+                "precision_recall_curve": {
+                    "average_precision": score["average_precision"],
+                    "best_f1": score["best_f1"],
+                    "best_f1_threshold": 0.3,
+                },
+            }
+
+        monkeypatch.setattr(DetectionService, "compare_detections_with_reference", staticmethod(fake_qa))
+
+        report = Service.compare_runs(
+            _Session(),
+            analysis_run_ids=[run_a, run_b],
+            reference_dataset_id=reference_id,
+            iou_threshold=0.5,
+        )
+
+        assert report["comparability"]["comparable"] is True
+        assert report["ranking_metric"] == "average_precision"
+        # Ranked on AP, so the conservative model leads despite the lower F1 at
+        # its own threshold — which stays visible next to it.
+        assert [row["model_asset_id"] for row in report["rows"]] == ["conservative", "liberal"]
+        assert report["rows"][0]["f1_at_run_threshold"] == pytest.approx(0.44)
+        assert report["rows"][1]["f1_at_run_threshold"] == pytest.approx(0.61)
+
+    def test_one_run_twice_is_refused(self) -> None:
+        run_id = uuid4()
+
+        with pytest.raises(AppError) as exc_info:
+            DetectionComparisonService.compare_runs(
+                object(),
+                analysis_run_ids=[run_id, run_id],
+                reference_dataset_id=uuid4(),
+            )
+
+        assert exc_info.value.code == "DETECTION_COMPARISON_NEEDS_TWO_RUNS"

docs/API_CONTRACTS.md

@@ -1850,6 +1850,28 @@ Each feature includes:
 
 Returns persisted detections for a dataset as a GeoJSON FeatureCollection. Optional filters match the detection list endpoint.
 
+### POST `/api/v1/detection/runs/compare`
+
+Scores several persisted runs against one reference and ranks them on average
+precision.
+
+The workbench ranks model variants by a stored F1, each measured at that
+variant's own confidence threshold — a figure that says as much about the
+threshold as about the model, so a conservatively calibrated detector looks
+worse than a liberal one without detecting anything differently. Average
+precision describes the whole ranking the model produced. The F1 at each run's
+own threshold stays in the response next to it, so the difference between the
+two readings is auditable rather than hidden.
+
+Comparability is reported before any ranking. Runs over different source
+rasters, scored against different references, without a proven inference
+footprint, or covering a different evaluated population are not alternatives to
+one another, and `comparability.blocking_reasons` names which of those applies.
+The numbers are still returned — they are simply not a ranking.
+
+Each run is scored through the same QA path the workbench uses, so a comparison
+and the persisted quality checks cannot drift apart.
+
 ### POST `/api/v1/detection/runs/{analysis_run_id}/qa/reference`
 
 Compares persisted detection geometries from an analysis run against persisted `vector_features` from a reference vector dataset.