JensandClaude Opus 5
c8d32a4801
scripts/evaluate_belgium_building_candidate.py freezes its post-processing before the protected test — NMS IoU and a containment threshold selected during calibration, defaulting to 1.0. The runtime applied a hardcoded 0.85, so a promoted candidate was served under suppression its evaluation never measured and dropped detections the gate had counted. Neither report showed the difference. That constant was mine, added without noticing the evaluation pipeline already had a tuned value for the same concept. Containment is now configuration, recorded on every run beside the duplicate IoU threshold, so an operator can serve a candidate at the value it was gated at and afterwards see which value produced a given score. Two runs that post-processed differently produced different candidate sets from the same model output, so the comparison endpoint refuses to rank them. Runs recorded before those values were persisted carry none, and absence is not treated as a difference. The remaining gap is deliberate and documented rather than closed: the gate scores the model on its protected test set, the workbench scores the whole pipeline including coverage clipping and the tile-edge filter. A promoted candidate will not reproduce its gate F1 exactly, and pretending otherwise would be the worse answer. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
255 lines
9.9 KiB
Python
255 lines
9.9 KiB
Python
"""Comparing two models must not compare two different questions.
|
|
|
|
The workbench ranks model variants by a stored F1, each measured at that
|
|
model's own confidence threshold. A conservatively calibrated model then looks
|
|
worse than a liberal one without detecting anything differently — the number
|
|
says as much about the threshold as about the model.
|
|
|
|
It also says nothing about whether the two runs are comparable at all. Two runs
|
|
over different rasters, or with different inference coverage, produce numbers
|
|
that cannot be placed side by side however they were measured.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
|
|
from app.core.errors import AppError
|
|
from app.services.detection_comparison_service import DetectionComparisonService
|
|
|
|
|
|
def _entry(
|
|
*,
|
|
dataset_id,
|
|
model_asset_id: str = "asset-a",
|
|
coverage_mode: str = "persisted_tile_manifest_union",
|
|
reference_dataset_id=None,
|
|
reference_evaluated: int = 100,
|
|
):
|
|
return {
|
|
"analysis_run_id": uuid4(),
|
|
"dataset_id": dataset_id,
|
|
"model_id": "yolo-configured",
|
|
"model_asset_id": model_asset_id,
|
|
"reference_dataset_id": reference_dataset_id or uuid4(),
|
|
"coverage_mode": coverage_mode,
|
|
"reference_evaluated_count": reference_evaluated,
|
|
}
|
|
|
|
|
|
class TestComparability:
|
|
def test_runs_over_the_same_raster_and_reference_are_comparable(self) -> None:
|
|
dataset_id, reference_id = uuid4(), uuid4()
|
|
entries = [
|
|
_entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="a"),
|
|
_entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="b"),
|
|
]
|
|
|
|
report = DetectionComparisonService.assess_comparability(entries)
|
|
|
|
assert report["comparable"] is True
|
|
assert report["blocking_reasons"] == []
|
|
|
|
def test_runs_over_different_rasters_are_not_comparable(self) -> None:
|
|
reference_id = uuid4()
|
|
entries = [
|
|
_entry(dataset_id=uuid4(), reference_dataset_id=reference_id),
|
|
_entry(dataset_id=uuid4(), reference_dataset_id=reference_id),
|
|
]
|
|
|
|
report = DetectionComparisonService.assess_comparability(entries)
|
|
|
|
assert report["comparable"] is False
|
|
assert "different_source_raster" in report["blocking_reasons"]
|
|
|
|
def test_runs_scored_against_different_references_are_not_comparable(self) -> None:
|
|
dataset_id = uuid4()
|
|
entries = [_entry(dataset_id=dataset_id), _entry(dataset_id=dataset_id)]
|
|
|
|
report = DetectionComparisonService.assess_comparability(entries)
|
|
|
|
assert report["comparable"] is False
|
|
assert "different_reference_dataset" in report["blocking_reasons"]
|
|
|
|
def test_a_run_without_proven_coverage_is_flagged(self) -> None:
|
|
dataset_id, reference_id = uuid4(), uuid4()
|
|
entries = [
|
|
_entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="a"),
|
|
_entry(
|
|
dataset_id=dataset_id,
|
|
reference_dataset_id=reference_id,
|
|
model_asset_id="b",
|
|
coverage_mode="unbounded_no_manifest",
|
|
),
|
|
]
|
|
|
|
report = DetectionComparisonService.assess_comparability(entries)
|
|
|
|
assert report["comparable"] is False
|
|
assert "coverage_not_proven" in report["blocking_reasons"]
|
|
|
|
def test_a_differing_evaluated_population_is_flagged(self) -> None:
|
|
"""Same raster and reference, but the runs did not see the same ground."""
|
|
|
|
dataset_id, reference_id = uuid4(), uuid4()
|
|
entries = [
|
|
_entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="a", reference_evaluated=100),
|
|
_entry(dataset_id=dataset_id, reference_dataset_id=reference_id, model_asset_id="b", reference_evaluated=60),
|
|
]
|
|
|
|
report = DetectionComparisonService.assess_comparability(entries)
|
|
|
|
assert report["comparable"] is False
|
|
assert "different_evaluated_population" in report["blocking_reasons"]
|
|
|
|
def test_one_run_is_never_a_comparison(self) -> None:
|
|
with pytest.raises(AppError) as exc_info:
|
|
DetectionComparisonService.assess_comparability([_entry(dataset_id=uuid4())])
|
|
|
|
assert exc_info.value.code == "DETECTION_COMPARISON_NEEDS_TWO_RUNS"
|
|
|
|
|
|
class TestRanking:
|
|
def _row(self, name: str, *, ap: float, best_f1: float, threshold_f1: float):
|
|
return {
|
|
"model_asset_id": name,
|
|
"average_precision": ap,
|
|
"best_f1": best_f1,
|
|
"best_f1_threshold": 0.3,
|
|
"f1_at_run_threshold": threshold_f1,
|
|
}
|
|
|
|
def test_ranking_uses_average_precision_not_the_run_threshold_f1(self) -> None:
|
|
rows = [
|
|
self._row("liberal", ap=0.55, best_f1=0.60, threshold_f1=0.61),
|
|
self._row("conservative", ap=0.72, best_f1=0.71, threshold_f1=0.44),
|
|
]
|
|
|
|
ranked = DetectionComparisonService.rank(rows)
|
|
|
|
# The conservative model detects better; its stored F1 only looked worse
|
|
# because it was measured at a stricter cut.
|
|
assert [row["model_asset_id"] for row in ranked] == ["conservative", "liberal"]
|
|
assert ranked[0]["rank"] == 1
|
|
|
|
def test_every_row_states_how_far_behind_the_leader_it_is(self) -> None:
|
|
rows = [
|
|
self._row("a", ap=0.72, best_f1=0.71, threshold_f1=0.44),
|
|
self._row("b", ap=0.55, best_f1=0.60, threshold_f1=0.61),
|
|
]
|
|
|
|
ranked = DetectionComparisonService.rank(rows)
|
|
|
|
assert ranked[0]["average_precision_gap"] == pytest.approx(0.0)
|
|
assert ranked[1]["average_precision_gap"] == pytest.approx(0.17)
|
|
|
|
def test_only_the_leader_carries_a_lead(self) -> None:
|
|
"""A follower's gap must never be readable as an advantage."""
|
|
|
|
rows = [
|
|
self._row("a", ap=0.72, best_f1=0.71, threshold_f1=0.44),
|
|
self._row("b", ap=0.55, best_f1=0.60, threshold_f1=0.61),
|
|
]
|
|
|
|
ranked = DetectionComparisonService.rank(rows)
|
|
|
|
assert ranked[0]["lead_over_next"] == pytest.approx(0.17)
|
|
assert ranked[1]["lead_over_next"] is None
|
|
|
|
def test_a_tied_leader_claims_no_lead(self) -> None:
|
|
rows = [
|
|
self._row("a", ap=0.6, best_f1=0.6, threshold_f1=0.6),
|
|
self._row("b", ap=0.6, best_f1=0.5, threshold_f1=0.5),
|
|
]
|
|
|
|
assert all(row["lead_over_next"] is None for row in DetectionComparisonService.rank(rows))
|
|
|
|
def test_a_tie_is_reported_as_a_tie_rather_than_an_arbitrary_winner(self) -> None:
|
|
rows = [
|
|
self._row("a", ap=0.6, best_f1=0.6, threshold_f1=0.6),
|
|
self._row("b", ap=0.6, best_f1=0.5, threshold_f1=0.5),
|
|
]
|
|
|
|
ranked = DetectionComparisonService.rank(rows)
|
|
|
|
assert [row["rank"] for row in ranked] == [1, 1]
|
|
assert all(row["tied"] for row in ranked)
|
|
|
|
|
|
class TestComparingRealRuns:
|
|
"""Through the same QA path the workbench uses, so the two cannot drift."""
|
|
|
|
def test_two_runs_are_scored_ranked_and_judged_comparable(self, monkeypatch) -> None:
|
|
from app.services.detection_comparison_service import DetectionComparisonService as Service
|
|
from app.services.detection_service import DetectionService
|
|
|
|
dataset_id, reference_id = uuid4(), uuid4()
|
|
run_a, run_b = uuid4(), uuid4()
|
|
|
|
class _Run:
|
|
def __init__(self, asset: str, threshold: float) -> None:
|
|
self.dataset_id = dataset_id
|
|
self.model_name = "yolo-configured"
|
|
self.parameters_json = {"model_asset_id": asset, "confidence_threshold": threshold}
|
|
# Both runs post-processed identically, so they stay comparable.
|
|
self.result_json = {
|
|
"containment_suppression_threshold": 0.85,
|
|
"duplicate_iou_threshold": 0.5,
|
|
}
|
|
|
|
runs = {run_a: _Run("liberal", 0.15), run_b: _Run("conservative", 0.45)}
|
|
scores = {
|
|
run_a: {"average_precision": 0.55, "best_f1": 0.60, "f1": 0.61},
|
|
run_b: {"average_precision": 0.72, "best_f1": 0.71, "f1": 0.44},
|
|
}
|
|
|
|
class _Session:
|
|
def get(self, _model, item_id):
|
|
return runs.get(item_id)
|
|
|
|
def fake_qa(_db, *, analysis_run_id, reference_dataset_id, iou_threshold, **_kwargs):
|
|
score = scores[analysis_run_id]
|
|
return {
|
|
"quality_check_id": str(uuid4()),
|
|
"f1_score": score["f1"],
|
|
"precision": 0.6,
|
|
"recall": 0.6,
|
|
"coverage": {"mode": "persisted_tile_manifest_union", "reference_evaluated_count": 100},
|
|
"precision_recall_curve": {
|
|
"average_precision": score["average_precision"],
|
|
"best_f1": score["best_f1"],
|
|
"best_f1_threshold": 0.3,
|
|
},
|
|
}
|
|
|
|
monkeypatch.setattr(DetectionService, "compare_detections_with_reference", staticmethod(fake_qa))
|
|
|
|
report = Service.compare_runs(
|
|
_Session(),
|
|
analysis_run_ids=[run_a, run_b],
|
|
reference_dataset_id=reference_id,
|
|
iou_threshold=0.5,
|
|
)
|
|
|
|
assert report["comparability"]["comparable"] is True
|
|
assert report["ranking_metric"] == "average_precision"
|
|
# Ranked on AP, so the conservative model leads despite the lower F1 at
|
|
# its own threshold — which stays visible next to it.
|
|
assert [row["model_asset_id"] for row in report["rows"]] == ["conservative", "liberal"]
|
|
assert report["rows"][0]["f1_at_run_threshold"] == pytest.approx(0.44)
|
|
assert report["rows"][1]["f1_at_run_threshold"] == pytest.approx(0.61)
|
|
|
|
def test_one_run_twice_is_refused(self) -> None:
|
|
run_id = uuid4()
|
|
|
|
with pytest.raises(AppError) as exc_info:
|
|
DetectionComparisonService.compare_runs(
|
|
object(),
|
|
analysis_run_ids=[run_id, run_id],
|
|
reference_dataset_id=uuid4(),
|
|
)
|
|
|
|
assert exc_info.value.code == "DETECTION_COMPARISON_NEEDS_TWO_RUNS"
|