From f6eced1b94cb21c11d8179fb6636ab84907f62f3 Mon Sep 17 00:00:00 2001 From: Jens Date: Sat, 22 Aug 2026 19:47:14 +0200 Subject: [PATCH] rank model variants on average precision, and say when they are not comparable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The workbench ranks model variants by a stored F1, each measured at that variant's own confidence threshold. A conservatively calibrated detector then looks worse than a liberal one without detecting anything differently: the number says as much about the threshold as about the model. POST /detection/runs/compare ranks on average precision instead, which describes the whole ranking a model produced, and keeps each run's own-threshold F1 visible next to it so the difference between the two readings is auditable. Comparability comes before the ranking. Runs over different source rasters, scored against different references, without a proven inference footprint, or covering a different evaluated population are not alternatives to one another, and no metric makes them so. The report names which of those applies and still returns the numbers — they are simply not a ranking. Each run is scored through the same QA path the workbench uses, so a comparison and the persisted quality checks cannot drift apart. Co-Authored-By: Claude Opus 5 --- backend/app/api/routes/detection.py | 24 ++ backend/app/schemas/__init__.py | 4 + backend/app/schemas/detection.py | 19 ++ .../services/detection_comparison_service.py | 179 +++++++++++++ .../tests/test_detection_model_comparison.py | 249 ++++++++++++++++++ docs/API_CONTRACTS.md | 22 ++ 6 files changed, 497 insertions(+) create mode 100644 backend/app/services/detection_comparison_service.py create mode 100644 backend/tests/test_detection_model_comparison.py diff --git a/backend/app/api/routes/detection.py b/backend/app/api/routes/detection.py index 3384f613..038f77c6 100644 --- a/backend/app/api/routes/detection.py +++ b/backend/app/api/routes/detection.py @@ -10,6 +10,8 @@ from app.schemas import ( AnalysisQaResponse, DetectionListResponse, DetectionModelsResponse, + DetectionComparisonRequest, + DetectionComparisonResponse, DetectionQaRequest, DetectionRead, DetectionRunListResponse, @@ -22,6 +24,7 @@ from app.schemas import ( ModelAssetListResponse, YoloPreflightResponse, ) +from app.services.detection_comparison_service import DetectionComparisonService from app.services.detection_service import DetectionService from app.services.model_asset_catalog_service import ModelAssetCatalogService from app.services.model_registry_service import ModelRegistryService @@ -232,6 +235,27 @@ def get_dataset_detection_geojson( ) +@router.post("/runs/compare", response_model=Envelope[DetectionComparisonResponse]) +def compare_detection_runs(payload: DetectionComparisonRequest, db: Session = Depends(get_db)) -> dict: + """Rank several runs against one reference on average precision. + + The workbench ranks model variants by a stored F1 measured at each + variant's own confidence threshold, which orders the thresholds as much as + the models. Average precision describes the whole ranking a model produced. + Comparability is reported first: runs over different rasters, different + references or different inference coverage are not alternatives. + """ + + return envelope( + DetectionComparisonService.compare_runs( + db, + analysis_run_ids=payload.analysis_run_ids, + reference_dataset_id=payload.reference_dataset_id, + iou_threshold=payload.iou_threshold, + ) + ) + + @router.post( "/runs/{analysis_run_id}/qa/reference", response_model=Envelope[AnalysisQaResponse], diff --git a/backend/app/schemas/__init__.py b/backend/app/schemas/__init__.py index d1d18db9..e3b07798 100644 --- a/backend/app/schemas/__init__.py +++ b/backend/app/schemas/__init__.py @@ -56,6 +56,8 @@ from .detection import ( DetectionRunListResponse, DetectionRunRead, DetectionRunRequest, + DetectionComparisonRequest, + DetectionComparisonResponse, DetectionRunResponse, ModelAssetListResponse, ModelAssetRead, @@ -238,6 +240,8 @@ __all__ = [ "DetectionRunListResponse", "DetectionRunRead", "DetectionRunRequest", + "DetectionComparisonRequest", + "DetectionComparisonResponse", "DetectionRunResponse", "ModelAssetListResponse", "ModelAssetRead", diff --git a/backend/app/schemas/detection.py b/backend/app/schemas/detection.py index d706a084..4b717397 100644 --- a/backend/app/schemas/detection.py +++ b/backend/app/schemas/detection.py @@ -78,6 +78,25 @@ class DetectionQaRequest(BaseModel): calibration_thresholds: list[float] = Field(default_factory=list, max_length=32) +class DetectionComparisonRequest(BaseModel): + """Place several runs side by side against one reference.""" + + analysis_run_ids: list[UUID] = Field(min_length=2, max_length=12) + reference_dataset_id: UUID + iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0) + + +class DetectionComparisonResponse(BaseModel): + reference_dataset_id: UUID + iou_threshold: float + # Whether these runs answer the same question at all, and why not if they + # do not. Numbers from incomparable runs are reported but never ranked as + # if they were alternatives. + comparability: dict + ranking_metric: str + rows: list[dict] + + class DetectionRunResponse(BaseModel): model_config = ConfigDict(protected_namespaces=()) diff --git a/backend/app/services/detection_comparison_service.py b/backend/app/services/detection_comparison_service.py new file mode 100644 index 00000000..8b3293cc --- /dev/null +++ b/backend/app/services/detection_comparison_service.py @@ -0,0 +1,179 @@ +"""Placing two detection runs side by side, honestly. + +The workbench ranks model variants by a stored F1, each measured at that +variant's own confidence threshold. That number says as much about the +threshold as about the model: a conservatively calibrated detector looks worse +than a liberal one without detecting anything differently. Average precision +describes the whole ranking the model produced and is the comparable figure. + +Comparability comes first, though. Two runs over different rasters, scored +against different references, or covering different ground are not two answers +to one question, and no metric makes them so. +""" + +from __future__ import annotations + +from typing import Any + +from app.core.errors import AppError + + +class DetectionComparisonService: + # A run whose inference footprint was never established cannot be placed + # beside one that was: their recalls have different denominators. + PROVEN_COVERAGE_MODES = ("persisted_tile_manifest_union",) + + @staticmethod + def assess_comparability(entries: list[dict[str, Any]]) -> dict[str, Any]: + """Say whether these runs answer the same question, and why not if they don't.""" + + if len(entries) < 2: + raise AppError( + code="DETECTION_COMPARISON_NEEDS_TWO_RUNS", + message="Comparing detection models requires at least two runs", + status_code=400, + ) + + reasons: list[str] = [] + source_rasters = {str(entry.get("dataset_id")) for entry in entries} + if len(source_rasters) > 1: + reasons.append("different_source_raster") + + references = {str(entry.get("reference_dataset_id")) for entry in entries} + if len(references) > 1: + reasons.append("different_reference_dataset") + + if any( + str(entry.get("coverage_mode")) not in DetectionComparisonService.PROVEN_COVERAGE_MODES + for entry in entries + ): + reasons.append("coverage_not_proven") + + populations = {int(entry.get("reference_evaluated_count") or 0) for entry in entries} + if len(populations) > 1: + reasons.append("different_evaluated_population") + + return { + "comparable": not reasons, + "blocking_reasons": reasons, + "source_raster_count": len(source_rasters), + "reference_dataset_count": len(references), + "evaluated_population_counts": sorted(populations), + } + + @staticmethod + def rank(rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Order runs by average precision, stating the margin and any tie. + + Ranking on the F1 each run happened to be read at would order the + thresholds, not the models. + """ + + ordered = sorted( + rows, + key=lambda row: (-(row.get("average_precision") or 0.0), str(row.get("model_asset_id") or "")), + ) + if not ordered: + return [] + + leader = ordered[0].get("average_precision") or 0.0 + tied_count = sum(1 for row in ordered if (row.get("average_precision") or 0.0) == leader) + runner_up = (ordered[1].get("average_precision") or 0.0) if len(ordered) > 1 else leader + + ranked: list[dict[str, Any]] = [] + for row in ordered: + average_precision = row.get("average_precision") or 0.0 + is_leader = average_precision == leader + ranked.append( + { + **row, + "rank": 1 if is_leader else 1 + sum( + 1 for other in ordered if (other.get("average_precision") or 0.0) > average_precision + ), + # Distance behind the best run; zero for the leader itself. + "average_precision_gap": leader - average_precision, + "tied": is_leader and tied_count > 1, + # Only the leader has a lead; stating it on every row would + # invite reading a follower's gap as an advantage. + "lead_over_next": (leader - runner_up) if is_leader and tied_count == 1 else None, + } + ) + return ranked + + + @staticmethod + def compare_runs( + db, + *, + analysis_run_ids: list, + reference_dataset_id, + iou_threshold: float = 0.5, + ) -> dict[str, Any]: + """Score several runs against one reference and rank them on AP. + + Each run is scored through the same QA path the workbench uses, so the + comparison and the persisted quality checks cannot drift apart. + """ + + # Lazy: detection_service imports this module's siblings at load. + from app.models import AnalysisRun + from app.services.detection_service import DetectionService + + if len(set(analysis_run_ids)) < 2: + raise AppError( + code="DETECTION_COMPARISON_NEEDS_TWO_RUNS", + message="Comparing detection models requires at least two distinct runs", + status_code=400, + ) + + entries: list[dict[str, Any]] = [] + rows: list[dict[str, Any]] = [] + for analysis_run_id in analysis_run_ids: + result = DetectionService.compare_detections_with_reference( + db, + analysis_run_id=analysis_run_id, + reference_dataset_id=reference_dataset_id, + iou_threshold=iou_threshold, + ) + run = db.get(AnalysisRun, analysis_run_id) + parameters = (run.parameters_json if run and isinstance(run.parameters_json, dict) else {}) or {} + coverage = result.get("coverage") if isinstance(result.get("coverage"), dict) else {} + curve = result.get("precision_recall_curve") or {} + + entries.append( + { + "analysis_run_id": analysis_run_id, + "dataset_id": getattr(run, "dataset_id", None), + "model_id": getattr(run, "model_name", None), + "model_asset_id": parameters.get("model_asset_id"), + "reference_dataset_id": reference_dataset_id, + "coverage_mode": coverage.get("mode"), + "reference_evaluated_count": coverage.get("reference_evaluated_count"), + } + ) + rows.append( + { + "analysis_run_id": str(analysis_run_id), + "quality_check_id": result.get("quality_check_id"), + "model_id": getattr(run, "model_name", None), + "model_asset_id": parameters.get("model_asset_id"), + "run_confidence_threshold": parameters.get("confidence_threshold"), + "average_precision": curve.get("average_precision"), + "best_f1": curve.get("best_f1"), + "best_f1_threshold": curve.get("best_f1_threshold"), + # The figure the workbench used to rank on, kept visible so + # the difference between the two readings is auditable. + "f1_at_run_threshold": result.get("f1_score"), + "precision_at_run_threshold": result.get("precision"), + "recall_at_run_threshold": result.get("recall"), + } + ) + + comparability = DetectionComparisonService.assess_comparability(entries) + return { + "reference_dataset_id": str(reference_dataset_id), + "iou_threshold": iou_threshold, + "comparability": comparability, + "ranking_metric": "average_precision", + "rows": DetectionComparisonService.rank(rows), + } diff --git a/backend/tests/test_detection_model_comparison.py b/backend/tests/test_detection_model_comparison.py new file mode 100644 index 00000000..387085e1 --- /dev/null +++ b/backend/tests/test_detection_model_comparison.py @@ -0,0 +1,249 @@ +"""Comparing two models must not compare two different questions. + +The workbench ranks model variants by a stored F1, each measured at that +model's own confidence threshold. A conservatively calibrated model then looks +worse than a liberal one without detecting anything differently — the number +says as much about the threshold as about the model. + +It also says nothing about whether the two runs are comparable at all. Two runs +over different rasters, or with different inference coverage, produce numbers +that cannot be placed side by side however they were measured. +""" + +from __future__ import annotations + +from uuid import uuid4 + +import pytest + +from app.core.errors import AppError +from app.services.detection_comparison_service import DetectionComparisonService + + +def _entry( + *, + dataset_id, + model_asset_id: str = "asset-a", + coverage_mode: str = "persisted_tile_manifest_union", + reference_dataset_id=None, + reference_evaluated: int = 100, +): + return { + "analysis_run_id": uuid4(), + "dataset_id": dataset_id, + "model_id": "yolo-configured", + "model_asset_id": model_asset_id, + "reference_dataset_id": reference_dataset_id or uuid4(), + "coverage_mode": coverage_mode, + "reference_evaluated_count": reference_evaluated, + } + + +class TestComparability: + def test_runs_over_the_same_raster_and_reference_are_comparable(self) -> None: + dataset_id, reference_id = uuid4(), uuid4() + entries = [ + _entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="a"), + _entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="b"), + ] + + report = DetectionComparisonService.assess_comparability(entries) + + assert report["comparable"] is True + assert report["blocking_reasons"] == [] + + def test_runs_over_different_rasters_are_not_comparable(self) -> None: + reference_id = uuid4() + entries = [ + _entry(dataset_id=uuid4(), reference_dataset_id=reference_id), + _entry(dataset_id=uuid4(), reference_dataset_id=reference_id), + ] + + report = DetectionComparisonService.assess_comparability(entries) + + assert report["comparable"] is False + assert "different_source_raster" in report["blocking_reasons"] + + def test_runs_scored_against_different_references_are_not_comparable(self) -> None: + dataset_id = uuid4() + entries = [_entry(dataset_id=dataset_id), _entry(dataset_id=dataset_id)] + + report = DetectionComparisonService.assess_comparability(entries) + + assert report["comparable"] is False + assert "different_reference_dataset" in report["blocking_reasons"] + + def test_a_run_without_proven_coverage_is_flagged(self) -> None: + dataset_id, reference_id = uuid4(), uuid4() + entries = [ + _entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="a"), + _entry( + dataset_id=dataset_id, + reference_dataset_id=reference_id, + model_asset_id="b", + coverage_mode="unbounded_no_manifest", + ), + ] + + report = DetectionComparisonService.assess_comparability(entries) + + assert report["comparable"] is False + assert "coverage_not_proven" in report["blocking_reasons"] + + def test_a_differing_evaluated_population_is_flagged(self) -> None: + """Same raster and reference, but the runs did not see the same ground.""" + + dataset_id, reference_id = uuid4(), uuid4() + entries = [ + _entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="a", reference_evaluated=100), + _entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="b", reference_evaluated=60), + ] + + report = DetectionComparisonService.assess_comparability(entries) + + assert report["comparable"] is False + assert "different_evaluated_population" in report["blocking_reasons"] + + def test_one_run_is_never_a_comparison(self) -> None: + with pytest.raises(AppError) as exc_info: + DetectionComparisonService.assess_comparability([_entry(dataset_id=uuid4())]) + + assert exc_info.value.code == "DETECTION_COMPARISON_NEEDS_TWO_RUNS" + + +class TestRanking: + def _row(self, name: str, *, ap: float, best_f1: float, threshold_f1: float): + return { + "model_asset_id": name, + "average_precision": ap, + "best_f1": best_f1, + "best_f1_threshold": 0.3, + "f1_at_run_threshold": threshold_f1, + } + + def test_ranking_uses_average_precision_not_the_run_threshold_f1(self) -> None: + rows = [ + self._row("liberal", ap=0.55, best_f1=0.60, threshold_f1=0.61), + self._row("conservative", ap=0.72, best_f1=0.71, threshold_f1=0.44), + ] + + ranked = DetectionComparisonService.rank(rows) + + # The conservative model detects better; its stored F1 only looked worse + # because it was measured at a stricter cut. + assert [row["model_asset_id"] for row in ranked] == ["conservative", "liberal"] + assert ranked[0]["rank"] == 1 + + def test_every_row_states_how_far_behind_the_leader_it_is(self) -> None: + rows = [ + self._row("a", ap=0.72, best_f1=0.71, threshold_f1=0.44), + self._row("b", ap=0.55, best_f1=0.60, threshold_f1=0.61), + ] + + ranked = DetectionComparisonService.rank(rows) + + assert ranked[0]["average_precision_gap"] == pytest.approx(0.0) + assert ranked[1]["average_precision_gap"] == pytest.approx(0.17) + + def test_only_the_leader_carries_a_lead(self) -> None: + """A follower's gap must never be readable as an advantage.""" + + rows = [ + self._row("a", ap=0.72, best_f1=0.71, threshold_f1=0.44), + self._row("b", ap=0.55, best_f1=0.60, threshold_f1=0.61), + ] + + ranked = DetectionComparisonService.rank(rows) + + assert ranked[0]["lead_over_next"] == pytest.approx(0.17) + assert ranked[1]["lead_over_next"] is None + + def test_a_tied_leader_claims_no_lead(self) -> None: + rows = [ + self._row("a", ap=0.6, best_f1=0.6, threshold_f1=0.6), + self._row("b", ap=0.6, best_f1=0.5, threshold_f1=0.5), + ] + + assert all(row["lead_over_next"] is None for row in DetectionComparisonService.rank(rows)) + + def test_a_tie_is_reported_as_a_tie_rather_than_an_arbitrary_winner(self) -> None: + rows = [ + self._row("a", ap=0.6, best_f1=0.6, threshold_f1=0.6), + self._row("b", ap=0.6, best_f1=0.5, threshold_f1=0.5), + ] + + ranked = DetectionComparisonService.rank(rows) + + assert [row["rank"] for row in ranked] == [1, 1] + assert all(row["tied"] for row in ranked) + + +class TestComparingRealRuns: + """Through the same QA path the workbench uses, so the two cannot drift.""" + + def test_two_runs_are_scored_ranked_and_judged_comparable(self, monkeypatch) -> None: + from app.services.detection_comparison_service import DetectionComparisonService as Service + from app.services.detection_service import DetectionService + + dataset_id, reference_id = uuid4(), uuid4() + run_a, run_b = uuid4(), uuid4() + + class _Run: + def __init__(self, asset: str, threshold: float) -> None: + self.dataset_id = dataset_id + self.model_name = "yolo-configured" + self.parameters_json = {"model_asset_id": asset, "confidence_threshold": threshold} + + runs = {run_a: _Run("liberal", 0.15), run_b: _Run("conservative", 0.45)} + scores = { + run_a: {"average_precision": 0.55, "best_f1": 0.60, "f1": 0.61}, + run_b: {"average_precision": 0.72, "best_f1": 0.71, "f1": 0.44}, + } + + class _Session: + def get(self, _model, item_id): + return runs.get(item_id) + + def fake_qa(_db, *, analysis_run_id, reference_dataset_id, iou_threshold, **_kwargs): + score = scores[analysis_run_id] + return { + "quality_check_id": str(uuid4()), + "f1_score": score["f1"], + "precision": 0.6, + "recall": 0.6, + "coverage": {"mode": "persisted_tile_manifest_union", "reference_evaluated_count": 100}, + "precision_recall_curve": { + "average_precision": score["average_precision"], + "best_f1": score["best_f1"], + "best_f1_threshold": 0.3, + }, + } + + monkeypatch.setattr(DetectionService, "compare_detections_with_reference", staticmethod(fake_qa)) + + report = Service.compare_runs( + _Session(), + analysis_run_ids=[run_a, run_b], + reference_dataset_id=reference_id, + iou_threshold=0.5, + ) + + assert report["comparability"]["comparable"] is True + assert report["ranking_metric"] == "average_precision" + # Ranked on AP, so the conservative model leads despite the lower F1 at + # its own threshold — which stays visible next to it. + assert [row["model_asset_id"] for row in report["rows"]] == ["conservative", "liberal"] + assert report["rows"][0]["f1_at_run_threshold"] == pytest.approx(0.44) + assert report["rows"][1]["f1_at_run_threshold"] == pytest.approx(0.61) + + def test_one_run_twice_is_refused(self) -> None: + run_id = uuid4() + + with pytest.raises(AppError) as exc_info: + DetectionComparisonService.compare_runs( + object(), + analysis_run_ids=[run_id, run_id], + reference_dataset_id=uuid4(), + ) + + assert exc_info.value.code == "DETECTION_COMPARISON_NEEDS_TWO_RUNS" diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index c6757ee2..2748a634 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -1850,6 +1850,28 @@ Each feature includes: Returns persisted detections for a dataset as a GeoJSON FeatureCollection. Optional filters match the detection list endpoint. +### POST `/api/v1/detection/runs/compare` + +Scores several persisted runs against one reference and ranks them on average +precision. + +The workbench ranks model variants by a stored F1, each measured at that +variant's own confidence threshold — a figure that says as much about the +threshold as about the model, so a conservatively calibrated detector looks +worse than a liberal one without detecting anything differently. Average +precision describes the whole ranking the model produced. The F1 at each run's +own threshold stays in the response next to it, so the difference between the +two readings is auditable rather than hidden. + +Comparability is reported before any ranking. Runs over different source +rasters, scored against different references, without a proven inference +footprint, or covering a different evaluated population are not alternatives to +one another, and `comparability.blocking_reasons` names which of those applies. +The numbers are still returned — they are simply not a ranking. + +Each run is scored through the same QA path the workbench uses, so a comparison +and the persisted quality checks cannot drift apart. + ### POST `/api/v1/detection/runs/{analysis_run_id}/qa/reference` Compares persisted detection geometries from an analysis run against persisted `vector_features` from a reference vector dataset.