"""Segmentation QA must score against the area it actually inferred. Detection QA already clips both populations to the union of the persisted inference tiles. Segmentation QA compared candidates against every reference feature in the dataset, so every building outside the inferred tiles counted as a false negative and recall collapsed for no modelling reason. """ from __future__ import annotations import json from pathlib import Path from uuid import uuid4 import pytest from geoalchemy2.shape import from_shape from shapely.geometry import MultiPolygon, box from app.core.errors import AppError from app.models import AnalysisRun, Dataset, Segmentation, VectorFeature from app.services.segmentation_service import SegmentationService from tests.test_sprint9_segmentation_foundation import ( # noqa: F401 FakeSession, _authoritative_reference, ) def _manifest(tmp_path: Path, dataset_id, bounds: list[float]) -> str: manifest_path = tmp_path / "manifest.json" manifest_path.write_text( json.dumps( { "source_dataset_id": str(dataset_id), "crs": "EPSG:4326", "tiles": [{"path": "tile_0000.tif", "bounds": bounds, "crs": "EPSG:4326"}], } ), encoding="utf-8", ) return str(manifest_path) def _segmentation(project_id, dataset_id, analysis_run_id, geom): return Segmentation( id=uuid4(), project_id=project_id, dataset_id=dataset_id, analysis_run_id=analysis_run_id, job_id=uuid4(), model_name="fixture-segmenter", model_version="fixture-v1", class_name="building", confidence=0.9, geometry=from_shape(MultiPolygon([geom]), srid=4326), ) def _reference(dataset_id, geom) -> VectorFeature: return VectorFeature( id=uuid4(), dataset_id=dataset_id, feature_class="building", geometry=from_shape(geom, srid=4326), ) def _session(tmp_path: Path, *, with_manifest: bool): project_id = uuid4() dataset_id = uuid4() reference_dataset_id = uuid4() analysis_run_id = uuid4() parameters = {} if with_manifest: parameters = {"tile_manifest_path": _manifest(tmp_path, dataset_id, [0.0, 0.0, 1.0, 1.0])} reference_dataset = _authoritative_reference( Dataset( id=reference_dataset_id, project_id=project_id, name="reference.geojson", dataset_type="vector", source="test", dataset_role="reference", ) ) db = FakeSession( objects={ (AnalysisRun, analysis_run_id): AnalysisRun( id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="segmentation", status="success", parameters_json=parameters, ), (Dataset, dataset_id): Dataset( id=dataset_id, project_id=project_id, name="fixture.tif", dataset_type="raster", source="test" ), (Dataset, reference_dataset_id): reference_dataset, }, query_rows={ Segmentation: [_segmentation(project_id, dataset_id, analysis_run_id, box(0.1, 0.1, 0.2, 0.2))], VectorFeature: [ # Inside the inferred tile: a genuine match. _reference(reference_dataset_id, box(0.1, 0.1, 0.2, 0.2)), # Far outside it: never looked at by the model. _reference(reference_dataset_id, box(8.0, 8.0, 8.1, 8.1)), _reference(reference_dataset_id, box(9.0, 9.0, 9.1, 9.1)), ], }, ) return db, analysis_run_id, reference_dataset_id def test_segmentation_qa_scores_only_inside_persisted_tile_coverage(tmp_path: Path, monkeypatch) -> None: # A manifest written into tmp_path is only a governed artifact if # tmp_path is the storage root. monkeypatch.setenv("STORAGE_ROOT", str(tmp_path)) db, analysis_run_id, reference_dataset_id = _session(tmp_path, with_manifest=True) result = SegmentationService.compare_segmentations_with_reference( db=db, analysis_run_id=analysis_run_id, reference_dataset_id=reference_dataset_id, iou_threshold=0.5, ) assert result["matches"] == 1 assert result["false_negatives"] == 0 assert result["recall"] == 1.0 assert result["coverage"]["applied"] is True assert result["coverage"]["reference_raw_count"] == 3 assert result["coverage"]["reference_evaluated_count"] == 1 assert result["coverage"]["reference_excluded_outside_count"] == 2 assert any("tile" in warning for warning in result["warnings"]) def test_segmentation_qa_without_manifest_reports_unbounded_coverage(tmp_path: Path) -> None: db, analysis_run_id, reference_dataset_id = _session(tmp_path, with_manifest=False) result = SegmentationService.compare_segmentations_with_reference( db=db, analysis_run_id=analysis_run_id, reference_dataset_id=reference_dataset_id, iou_threshold=0.5, ) # Unchanged behaviour, but the response now says the score was not bounded # by an inference footprint so the recall can be read correctly. assert result["false_negatives"] == 2 assert result["coverage"]["applied"] is False assert result["coverage"]["mode"] == "unbounded_no_manifest" def test_segmentation_qa_rejects_reference_entirely_outside_coverage(tmp_path: Path, monkeypatch) -> None: # A manifest written into tmp_path is only a governed artifact if # tmp_path is the storage root. monkeypatch.setenv("STORAGE_ROOT", str(tmp_path)) db, analysis_run_id, reference_dataset_id = _session(tmp_path, with_manifest=True) db.query_rows[VectorFeature] = [ _reference(reference_dataset_id, box(8.0, 8.0, 8.1, 8.1)), ] with pytest.raises(AppError) as exc_info: SegmentationService.compare_segmentations_with_reference( db=db, analysis_run_id=analysis_run_id, reference_dataset_id=reference_dataset_id, iou_threshold=0.5, ) assert exc_info.value.code == "REFERENCE_FEATURES_OUTSIDE_COVERAGE"