diff --git a/.env.example b/.env.example index 65cf5183..cd30adc7 100644 --- a/.env.example +++ b/.env.example @@ -138,6 +138,11 @@ YOLO_BATCH_SIZE=1 # neighbouring tile saw the same object whole, so the truncated half is a # duplicate and a shape error at once. Boxes on the outer raster edge are kept. YOLO_SUPPRESS_TILE_EDGE_DETECTIONS=true +# Intersection over the smaller box. scripts/evaluate_belgium_building_candidate.py +# freezes this during calibration (--containment-nms) before the protected test. +# Serving a promoted model at a different value means the runtime suppresses +# detections its gate counted, so set this to the value the candidate was gated at. +YOLO_CONTAINMENT_NMS_THRESHOLD=0.85 # Local segmentation models. GeoIntel never downloads model weights # automatically; point these to existing local files to enable inference. diff --git a/backend/app/core/config.py b/backend/app/core/config.py index 4c37ba9e..4b94ab55 100644 --- a/backend/app/core/config.py +++ b/backend/app/core/config.py @@ -391,6 +391,12 @@ class Settings(BaseSettings): yolo_suppress_tile_edge_detections: bool = Field( default=True, validation_alias="YOLO_SUPPRESS_TILE_EDGE_DETECTIONS" ) + # Intersection over the smaller box. The candidate evaluation freezes this + # during calibration; serving a promoted model at a different value means + # the runtime suppresses detections the gate counted. + yolo_containment_nms_threshold: float = Field( + default=0.85, ge=0.0, le=1.0, validation_alias="YOLO_CONTAINMENT_NMS_THRESHOLD" + ) yolo_batch_size: int = Field(default=1, validation_alias="YOLO_BATCH_SIZE") yolo_seg_enabled: bool = Field(default=False, validation_alias="YOLO_SEG_ENABLED") yolo_seg_model_path: str | None = Field(default=None, validation_alias="YOLO_SEG_MODEL_PATH") diff --git a/backend/app/services/detection_comparison_service.py b/backend/app/services/detection_comparison_service.py index 8b3293cc..24fbcab3 100644 --- a/backend/app/services/detection_comparison_service.py +++ b/backend/app/services/detection_comparison_service.py @@ -53,6 +53,20 @@ class DetectionComparisonService: if len(populations) > 1: reasons.append("different_evaluated_population") + # Two runs that suppressed duplicates differently produced different + # candidate sets from the same model output, so their scores describe + # different pipelines. Runs from before these values were recorded + # carry none; absence is not a difference. + post_processing = { + ( + entry.get("containment_suppression_threshold"), + entry.get("duplicate_iou_threshold"), + ) + for entry in entries + } + if len(post_processing) > 1: + reasons.append("different_post_processing") + return { "comparable": not reasons, "blocking_reasons": reasons, @@ -137,6 +151,7 @@ class DetectionComparisonService: ) run = db.get(AnalysisRun, analysis_run_id) parameters = (run.parameters_json if run and isinstance(run.parameters_json, dict) else {}) or {} + run_result = (run.result_json if run and isinstance(run.result_json, dict) else {}) or {} coverage = result.get("coverage") if isinstance(result.get("coverage"), dict) else {} curve = result.get("precision_recall_curve") or {} @@ -149,6 +164,10 @@ class DetectionComparisonService: "reference_dataset_id": reference_dataset_id, "coverage_mode": coverage.get("mode"), "reference_evaluated_count": coverage.get("reference_evaluated_count"), + # Recorded on the run itself, so two runs that suppressed + # duplicates differently cannot be ranked against each other. + "containment_suppression_threshold": run_result.get("containment_suppression_threshold"), + "duplicate_iou_threshold": run_result.get("duplicate_iou_threshold"), } ) rows.append( @@ -166,6 +185,8 @@ class DetectionComparisonService: "f1_at_run_threshold": result.get("f1_score"), "precision_at_run_threshold": result.get("precision"), "recall_at_run_threshold": result.get("recall"), + "containment_suppression_threshold": run_result.get("containment_suppression_threshold"), + "duplicate_iou_threshold": run_result.get("duplicate_iou_threshold"), } ) diff --git a/backend/app/services/detection_service.py b/backend/app/services/detection_service.py index e63fb82c..877364c0 100644 --- a/backend/app/services/detection_service.py +++ b/backend/app/services/detection_service.py @@ -1024,6 +1024,7 @@ class DetectionService: filtered_candidates = DetectionService._suppress_duplicate_candidates( edge_filtered_candidates, iou_threshold=float(settings.yolo_duplicate_iou_threshold), + containment_threshold=float(settings.yolo_containment_nms_threshold), ) persisted: list[Detection] = [] for candidate in filtered_candidates: @@ -1061,7 +1062,7 @@ class DetectionService: "suppressed_detection_count": len(candidates) - len(filtered_candidates), "tile_edge_truncated_count": len(candidates) - len(edge_filtered_candidates), "duplicate_iou_threshold": float(settings.yolo_duplicate_iou_threshold), - "containment_suppression_threshold": DetectionService.CONTAINMENT_SUPPRESSION_THRESHOLD, + "containment_suppression_threshold": float(settings.yolo_containment_nms_threshold), "runtime_model_provenance": runtime_model_provenance.as_dict(), } @@ -1094,12 +1095,20 @@ class DetectionService: # two halves barely intersect and IoU alone never suppresses them. Overlap # measured against the smaller box catches that case; the threshold is # deliberately strict so that terraced houses stay separate detections. + # Fallback only. The served value is configuration, so a promoted model can + # be run at the threshold its evaluation froze. CONTAINMENT_SUPPRESSION_THRESHOLD = 0.85 @staticmethod - def _suppress_duplicate_candidates(candidates: list[dict[str, Any]], iou_threshold: float) -> list[dict[str, Any]]: + def _suppress_duplicate_candidates( + candidates: list[dict[str, Any]], + iou_threshold: float, + containment_threshold: float | None = None, + ) -> list[dict[str, Any]]: if iou_threshold <= 0 or len(candidates) < 2: return candidates + if containment_threshold is None: + containment_threshold = DetectionService.CONTAINMENT_SUPPRESSION_THRESHOLD ordered = sorted( candidates, @@ -1123,10 +1132,7 @@ class DetectionService: if DetectionService._geometry_iou(geometry, other) >= iou_threshold: duplicate = True break - if ( - DetectionService._geometry_containment(geometry, other) - >= DetectionService.CONTAINMENT_SUPPRESSION_THRESHOLD - ): + if DetectionService._geometry_containment(geometry, other) >= containment_threshold: duplicate = True break if not duplicate: diff --git a/backend/tests/test_detection_model_comparison.py b/backend/tests/test_detection_model_comparison.py index 387085e1..f5130852 100644 --- a/backend/tests/test_detection_model_comparison.py +++ b/backend/tests/test_detection_model_comparison.py @@ -193,6 +193,11 @@ class TestComparingRealRuns: self.dataset_id = dataset_id self.model_name = "yolo-configured" self.parameters_json = {"model_asset_id": asset, "confidence_threshold": threshold} + # Both runs post-processed identically, so they stay comparable. + self.result_json = { + "containment_suppression_threshold": 0.85, + "duplicate_iou_threshold": 0.5, + } runs = {run_a: _Run("liberal", 0.15), run_b: _Run("conservative", 0.45)} scores = { diff --git a/backend/tests/test_postprocessing_parity_with_training_gate.py b/backend/tests/test_postprocessing_parity_with_training_gate.py new file mode 100644 index 00000000..44e2df81 --- /dev/null +++ b/backend/tests/test_postprocessing_parity_with_training_gate.py @@ -0,0 +1,144 @@ +"""What is promoted must be what the workbench then runs. + +The candidate evaluation freezes its post-processing before the protected test +— NMS IoU and a containment threshold selected during calibration. The runtime +applied its own hardcoded containment value, so a model gated at one setting +was served at another and suppressed detections the gate had counted. The +difference is invisible in both reports. + +Containment is therefore configuration, recorded with every run, and two runs +that post-processed differently are not comparable however good their numbers +look. +""" + +from __future__ import annotations + +import pytest +from shapely.geometry import box + +from app.core.config import Settings +from app.services.detection_comparison_service import DetectionComparisonService +from app.services.detection_service import DetectionService + + +def _candidate(name: str, geometry, confidence: float): + return { + "class_name": "building", + "confidence": confidence, + "geometry": geometry, + "bbox": [0.0, 0.0, 1.0, 1.0], + "source_tile_path": f"/tiles/{name}.tif", + "properties": {"name": name}, + } + + +class TestConfigurableContainment: + def test_the_runtime_threshold_comes_from_settings(self) -> None: + assert Settings(_env_file=None).yolo_containment_nms_threshold == pytest.approx(0.85) + assert Settings( + _env_file=None, yolo_containment_nms_threshold=1.0 + ).yolo_containment_nms_threshold == pytest.approx(1.0) + + def test_a_strict_threshold_suppresses_only_a_fully_nested_box(self) -> None: + outer = _candidate("outer", box(0, 0, 10, 10), 0.9) + # 90% of the smaller box lies inside the larger one, but their IoU is + # only 0.09 — so only the containment rule can act on this pair. + mostly_nested = _candidate("mostly", box(8.2, 1, 10.2, 6), 0.5) + + kept = DetectionService._suppress_duplicate_candidates( + [outer, mostly_nested], iou_threshold=0.5, containment_threshold=1.0 + ) + + assert [item["properties"]["name"] for item in kept] == ["outer", "mostly"] + + def test_a_looser_threshold_suppresses_it(self) -> None: + outer = _candidate("outer", box(0, 0, 10, 10), 0.9) + mostly_nested = _candidate("mostly", box(8.2, 1, 10.2, 6), 0.5) + + kept = DetectionService._suppress_duplicate_candidates( + [outer, mostly_nested], iou_threshold=0.5, containment_threshold=0.7 + ) + + assert [item["properties"]["name"] for item in kept] == ["outer"] + + def test_the_default_matches_the_documented_runtime_value(self) -> None: + outer = _candidate("outer", box(0, 0, 10, 10), 0.9) + nested = _candidate("nested", box(1, 1, 9, 9), 0.5) + + kept = DetectionService._suppress_duplicate_candidates([outer, nested], iou_threshold=0.5) + + assert [item["properties"]["name"] for item in kept] == ["outer"] + + +class TestComparabilityOfPostProcessing: + def _entry(self, *, dataset_id, reference_id, containment: float, duplicate_iou: float = 0.5): + from uuid import uuid4 + + return { + "analysis_run_id": uuid4(), + "dataset_id": dataset_id, + "model_id": "yolo-configured", + "model_asset_id": "asset", + "reference_dataset_id": reference_id, + "coverage_mode": "persisted_tile_manifest_union", + "reference_evaluated_count": 100, + "containment_suppression_threshold": containment, + "duplicate_iou_threshold": duplicate_iou, + } + + def test_runs_with_the_same_post_processing_stay_comparable(self) -> None: + from uuid import uuid4 + + dataset_id, reference_id = uuid4(), uuid4() + report = DetectionComparisonService.assess_comparability( + [ + self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85), + self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85), + ] + ) + + assert report["comparable"] is True + + def test_a_different_containment_threshold_blocks_the_comparison(self) -> None: + from uuid import uuid4 + + dataset_id, reference_id = uuid4(), uuid4() + report = DetectionComparisonService.assess_comparability( + [ + self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85), + self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=1.0), + ] + ) + + assert report["comparable"] is False + assert "different_post_processing" in report["blocking_reasons"] + + def test_a_different_duplicate_iou_blocks_the_comparison(self) -> None: + from uuid import uuid4 + + dataset_id, reference_id = uuid4(), uuid4() + report = DetectionComparisonService.assess_comparability( + [ + self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85, duplicate_iou=0.5), + self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85, duplicate_iou=0.7), + ] + ) + + assert report["comparable"] is False + assert "different_post_processing" in report["blocking_reasons"] + + def test_runs_from_before_the_setting_existed_do_not_block(self) -> None: + """Older runs recorded no threshold; absence is not a difference.""" + + from uuid import uuid4 + + dataset_id, reference_id = uuid4(), uuid4() + entries = [ + self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85), + self._entry(dataset_id=dataset_id, reference_id=reference_id, containment=0.85), + ] + for entry in entries: + entry.pop("containment_suppression_threshold") + entry.pop("duplicate_iou_threshold") + + assert DetectionComparisonService.assess_comparability(entries)["comparable"] is True diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index c235932e..122216c6 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -1891,8 +1891,11 @@ two readings is auditable rather than hidden. Comparability is reported before any ranking. Runs over different source rasters, scored against different references, without a proven inference -footprint, or covering a different evaluated population are not alternatives to -one another, and `comparability.blocking_reasons` names which of those applies. +footprint, covering a different evaluated population, or post-processed with a +different containment / duplicate-IoU threshold are not alternatives to one +another, and `comparability.blocking_reasons` names which of those applies. +Runs recorded before those thresholds were persisted carry none of them, and +absence is not treated as a difference. The numbers are still returned — they are simply not a ranking. Each run is scored through the same QA path the workbench uses, so a comparison diff --git a/docs/KNOWN_LIMITATIONS.md b/docs/KNOWN_LIMITATIONS.md index 3c6f735d..f1ae0ea0 100644 --- a/docs/KNOWN_LIMITATIONS.md +++ b/docs/KNOWN_LIMITATIONS.md @@ -92,6 +92,15 @@ runtime source of truth. `YOLO_SEG_MODEL_PATH`/`SAM_MODEL_PATH` to existing local weights and enables them explicitly. GeoIntel never downloads segmentation weights automatically; fixture segmentation remains explicit-only. +- The training gate and the workbench measure different things, deliberately. + `scripts/evaluate_belgium_building_candidate.py` scores the model on frozen + post-processing over its protected test set; the workbench scores the whole + production pipeline, which additionally clips to the persisted inference + coverage and drops boxes truncated by an interior tile edge. A promoted + candidate will therefore not reproduce its gate F1 exactly in the workbench. + What must match is the post-processing: `YOLO_CONTAINMENT_NMS_THRESHOLD` and + `YOLO_DUPLICATE_IOU_THRESHOLD` are recorded with every run, and two runs that + used different values are reported as not comparable. - Detection QA reports both a strict footprint IoU and an envelope diagnostic. For an axis-aligned box detector the strict figure has a ceiling below 1 on rotated or non-rectangular buildings; the response states which geometry mode