geointel/backend/tests/test_detection_model_comparison.py

"""Comparing two models must not compare two different questions.

The workbench ranks model variants by a stored F1, each measured at that
model's own confidence threshold. A conservatively calibrated model then looks
worse than a liberal one without detecting anything differently — the number
says as much about the threshold as about the model.

It also says nothing about whether the two runs are comparable at all. Two runs
over different rasters, or with different inference coverage, produce numbers
that cannot be placed side by side however they were measured.
"""

from __future__ import annotations

from uuid import uuid4

import pytest

from app.core.errors import AppError
from app.services.detection_comparison_service import DetectionComparisonService


def _entry(
    *,
    dataset_id,
    model_asset_id: str = "asset-a",
    coverage_mode: str = "persisted_tile_manifest_union",
    reference_dataset_id=None,
    reference_evaluated: int = 100,
):
    return {
        "analysis_run_id": uuid4(),
        "dataset_id": dataset_id,
        "model_id": "yolo-configured",
        "model_asset_id": model_asset_id,
        "reference_dataset_id": reference_dataset_id or uuid4(),
        "coverage_mode": coverage_mode,
        "reference_evaluated_count": reference_evaluated,
    }


class TestComparability:
    def test_runs_over_the_same_raster_and_reference_are_comparable(self) -> None:
        dataset_id, reference_id = uuid4(), uuid4()
        entries = [
            _entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="a"),
            _entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="b"),
        ]

        report = DetectionComparisonService.assess_comparability(entries)

        assert report["comparable"] is True
        assert report["blocking_reasons"] == []

    def test_runs_over_different_rasters_are_not_comparable(self) -> None:
        reference_id = uuid4()
        entries = [
            _entry(dataset_id=uuid4(), reference_dataset_id=reference_id),
            _entry(dataset_id=uuid4(), reference_dataset_id=reference_id),
        ]

        report = DetectionComparisonService.assess_comparability(entries)

        assert report["comparable"] is False
        assert "different_source_raster" in report["blocking_reasons"]

    def test_runs_scored_against_different_references_are_not_comparable(self) -> None:
        dataset_id = uuid4()
        entries = [_entry(dataset_id=dataset_id), _entry(dataset_id=dataset_id)]

        report = DetectionComparisonService.assess_comparability(entries)

        assert report["comparable"] is False
        assert "different_reference_dataset" in report["blocking_reasons"]

    def test_a_run_without_proven_coverage_is_flagged(self) -> None:
        dataset_id, reference_id = uuid4(), uuid4()
        entries = [
            _entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="a"),
            _entry(
                dataset_id=dataset_id,
                reference_dataset_id=reference_id,
                model_asset_id="b",
                coverage_mode="unbounded_no_manifest",
            ),
        ]

        report = DetectionComparisonService.assess_comparability(entries)

        assert report["comparable"] is False
        assert "coverage_not_proven" in report["blocking_reasons"]

    def test_a_differing_evaluated_population_is_flagged(self) -> None:
        """Same raster and reference, but the runs did not see the same ground."""

        dataset_id, reference_id = uuid4(), uuid4()
        entries = [
            _entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="a", reference_evaluated=100),
            _entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="b", reference_evaluated=60),
        ]

        report = DetectionComparisonService.assess_comparability(entries)

        assert report["comparable"] is False
        assert "different_evaluated_population" in report["blocking_reasons"]

    def test_one_run_is_never_a_comparison(self) -> None:
        with pytest.raises(AppError) as exc_info:
            DetectionComparisonService.assess_comparability([_entry(dataset_id=uuid4())])

        assert exc_info.value.code == "DETECTION_COMPARISON_NEEDS_TWO_RUNS"


class TestRanking:
    def _row(self, name: str, *, ap: float, best_f1: float, threshold_f1: float):
        return {
            "model_asset_id": name,
            "average_precision": ap,
            "best_f1": best_f1,
            "best_f1_threshold": 0.3,
            "f1_at_run_threshold": threshold_f1,
        }

    def test_ranking_uses_average_precision_not_the_run_threshold_f1(self) -> None:
        rows = [
            self._row("liberal", ap=0.55, best_f1=0.60, threshold_f1=0.61),
            self._row("conservative", ap=0.72, best_f1=0.71, threshold_f1=0.44),
        ]

        ranked = DetectionComparisonService.rank(rows)

        # The conservative model detects better; its stored F1 only looked worse
        # because it was measured at a stricter cut.
        assert [row["model_asset_id"] for row in ranked] == ["conservative", "liberal"]
        assert ranked[0]["rank"] == 1

    def test_every_row_states_how_far_behind_the_leader_it_is(self) -> None:
        rows = [
            self._row("a", ap=0.72, best_f1=0.71, threshold_f1=0.44),
            self._row("b", ap=0.55, best_f1=0.60, threshold_f1=0.61),
        ]

        ranked = DetectionComparisonService.rank(rows)

        assert ranked[0]["average_precision_gap"] == pytest.approx(0.0)
        assert ranked[1]["average_precision_gap"] == pytest.approx(0.17)

    def test_only_the_leader_carries_a_lead(self) -> None:
        """A follower's gap must never be readable as an advantage."""

        rows = [
            self._row("a", ap=0.72, best_f1=0.71, threshold_f1=0.44),
            self._row("b", ap=0.55, best_f1=0.60, threshold_f1=0.61),
        ]

        ranked = DetectionComparisonService.rank(rows)

        assert ranked[0]["lead_over_next"] == pytest.approx(0.17)
        assert ranked[1]["lead_over_next"] is None

    def test_a_tied_leader_claims_no_lead(self) -> None:
        rows = [
            self._row("a", ap=0.6, best_f1=0.6, threshold_f1=0.6),
            self._row("b", ap=0.6, best_f1=0.5, threshold_f1=0.5),
        ]

        assert all(row["lead_over_next"] is None for row in DetectionComparisonService.rank(rows))

    def test_a_tie_is_reported_as_a_tie_rather_than_an_arbitrary_winner(self) -> None:
        rows = [
            self._row("a", ap=0.6, best_f1=0.6, threshold_f1=0.6),
            self._row("b", ap=0.6, best_f1=0.5, threshold_f1=0.5),
        ]

        ranked = DetectionComparisonService.rank(rows)

        assert [row["rank"] for row in ranked] == [1, 1]
        assert all(row["tied"] for row in ranked)


class TestComparingRealRuns:
    """Through the same QA path the workbench uses, so the two cannot drift."""

    def test_two_runs_are_scored_ranked_and_judged_comparable(self, monkeypatch) -> None:
        from app.services.detection_comparison_service import DetectionComparisonService as Service
        from app.services.detection_service import DetectionService

        dataset_id, reference_id = uuid4(), uuid4()
        run_a, run_b = uuid4(), uuid4()

        class _Run:
            def __init__(self, asset: str, threshold: float) -> None:
                self.dataset_id = dataset_id
                self.model_name = "yolo-configured"
                self.parameters_json = {"model_asset_id": asset, "confidence_threshold": threshold}
                # Both runs post-processed identically, so they stay comparable.
                self.result_json = {
                    "containment_suppression_threshold": 0.85,
                    "duplicate_iou_threshold": 0.5,
                }

        runs = {run_a: _Run("liberal", 0.15), run_b: _Run("conservative", 0.45)}
        scores = {
            run_a: {"average_precision": 0.55, "best_f1": 0.60, "f1": 0.61},
            run_b: {"average_precision": 0.72, "best_f1": 0.71, "f1": 0.44},
        }

        class _Session:
            def get(self, _model, item_id):
                return runs.get(item_id)

        def fake_qa(_db, *, analysis_run_id, reference_dataset_id, iou_threshold, **_kwargs):
            score = scores[analysis_run_id]
            return {
                "quality_check_id": str(uuid4()),
                "f1_score": score["f1"],
                "precision": 0.6,
                "recall": 0.6,
                "coverage": {"mode": "persisted_tile_manifest_union", "reference_evaluated_count": 100},
                "precision_recall_curve": {
                    "average_precision": score["average_precision"],
                    "best_f1": score["best_f1"],
                    "best_f1_threshold": 0.3,
                },
            }

        monkeypatch.setattr(DetectionService, "compare_detections_with_reference", staticmethod(fake_qa))

        report = Service.compare_runs(
            _Session(),
            analysis_run_ids=[run_a, run_b],
            reference_dataset_id=reference_id,
            iou_threshold=0.5,
        )

        assert report["comparability"]["comparable"] is True
        assert report["ranking_metric"] == "average_precision"
        # Ranked on AP, so the conservative model leads despite the lower F1 at
        # its own threshold — which stays visible next to it.
        assert [row["model_asset_id"] for row in report["rows"]] == ["conservative", "liberal"]
        assert report["rows"][0]["f1_at_run_threshold"] == pytest.approx(0.44)
        assert report["rows"][1]["f1_at_run_threshold"] == pytest.approx(0.61)

    def test_one_run_twice_is_refused(self) -> None:
        run_id = uuid4()

        with pytest.raises(AppError) as exc_info:
            DetectionComparisonService.compare_runs(
                object(),
                analysis_run_ids=[run_id, run_id],
                reference_dataset_id=uuid4(),
            )

        assert exc_info.value.code == "DETECTION_COMPARISON_NEEDS_TWO_RUNS"