From 0cad8fdf76b3fc63a2381b109a97f05d74a1d9f6 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 15 Jul 2026 00:35:39 +0200 Subject: [PATCH] fix: bound detection QA to inference coverage --- CHANGELOG.md | 4 ++ backend/app/services/detection_qa_service.py | 9 ++- backend/app/services/detection_service.py | 57 ++++++++++++------- backend/app/services/qa_service.py | 14 +++-- .../test_sprint184_detection_qa_coverage.py | 44 ++++++++++++++ ...est_sprint8c_detection_visualization_qa.py | 3 + docs/AI_PIPELINES.md | 11 +++- docs/API_CONTRACTS.md | 6 +- docs/CODEX_EXECUTION_LOG.md | 16 ++++++ frontend/src/App.tsx | 10 ++-- frontend/src/components/map/MapWorkspace.tsx | 22 +++++-- frontend/src/hooks/useMapWorkspaceState.ts | 8 +-- 12 files changed, 159 insertions(+), 45 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 078f7036..021a6d80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,10 @@ - Kept manual manifest execution, model assets, preflight and calibration available under technical/management disclosures while making persisted detection QA a primary user step. - Added clear preparation progress, understandable Dutch QA diagnostics, result-to-map navigation and focused regression coverage. - Did not change API contracts, database migrations, model dependencies or backend inference behavior. +- Live Mol validation persisted 1,953 configured-YOLO detections from nine georeferenced tiles and exposed a regional QA scaling defect before release. +- Detection QA now applies the persisted tile coverage through the existing GiST-indexed PostGIS geometry column before loading reference rows, while retaining the complete reference population in audit counts. +- The unchanged exact IoU matcher now uses a Shapely spatial index to avoid testing geometries whose envelopes cannot intersect. +- Clarified the primary map source and legend whenever an AI result is active so detections are never presented as the underlying official GRB source. ## Sprint 194 Regional official time-series synchronization (2026-07-14) diff --git a/backend/app/services/detection_qa_service.py b/backend/app/services/detection_qa_service.py index f4911741..d9ac1a4b 100644 --- a/backend/app/services/detection_qa_service.py +++ b/backend/app/services/detection_qa_service.py @@ -129,9 +129,14 @@ class DetectionQaService: def filter_population( geometries: list[tuple[dict[str, Any], BaseGeometry]], coverage: DetectionQaCoverage, + *, + raw_count: int | None = None, ) -> CoveragePopulation: evaluated: list[tuple[dict[str, Any], BaseGeometry]] = [] - excluded_outside_count = 0 + resolved_raw_count = len(geometries) if raw_count is None else raw_count + if resolved_raw_count < len(geometries): + raise ValueError("raw_count cannot be smaller than the supplied geometry population") + excluded_outside_count = resolved_raw_count - len(geometries) clipped_boundary_count = 0 for feature, geometry in geometries: @@ -157,7 +162,7 @@ class DetectionQaService: return CoveragePopulation( geometries=evaluated, - raw_count=len(geometries), + raw_count=resolved_raw_count, evaluated_count=len(evaluated), excluded_outside_count=excluded_outside_count, clipped_boundary_count=clipped_boundary_count, diff --git a/backend/app/services/detection_service.py b/backend/app/services/detection_service.py index 3fd0012f..7c21926b 100644 --- a/backend/app/services/detection_service.py +++ b/backend/app/services/detection_service.py @@ -9,6 +9,7 @@ from typing import Type from geoalchemy2.shape import from_shape, to_shape from shapely.geometry import mapping, shape +from sqlalchemy import func from app.core.config import Settings, get_settings from app.core.errors import AppError @@ -288,18 +289,8 @@ class DetectionService: class_name=class_name, min_confidence=min_confidence, ) - references = db.query(VectorFeature).filter(VectorFeature.dataset_id == reference_dataset_id).all() - if not references: - raise AppError( - code="REFERENCE_FEATURES_NOT_FOUND", - message="Reference dataset has no persisted vector features for QA", - status_code=422, - ) - raw_candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in detections] - raw_reference_geometries = [({"id": str(row.id), "feature_class": row.feature_class}, to_shape(row.geometry)) for row in references] candidate_geometries = raw_candidate_geometries - reference_geometries = raw_reference_geometries run_parameters = run.parameters_json if isinstance(run.parameters_json, dict) else {} manifest_path = DetectionQaService.tile_manifest_path(run_parameters) resolved_settings = get_settings() @@ -314,6 +305,34 @@ class DetectionService: status_code=422, ) + coverage = None + if manifest_path: + manifest = DetectionService._load_tile_manifest(manifest_path, resolved_settings.yolo_max_tiles) + coverage = DetectionQaService.build_tile_coverage( + manifest, + manifest_path=manifest_path, + expected_dataset_id=run.dataset_id, + ) + + reference_query = db.query(VectorFeature).filter(VectorFeature.dataset_id == reference_dataset_id) + if coverage is not None and hasattr(reference_query, "count"): + reference_raw_count = reference_query.count() + references = reference_query.filter( + func.ST_Intersects(VectorFeature.geometry, from_shape(coverage.geometry, srid=4326)) + ).all() + else: + references = reference_query.all() + reference_raw_count = len(references) + if reference_raw_count == 0: + raise AppError( + code="REFERENCE_FEATURES_NOT_FOUND", + message="Reference dataset has no persisted vector features for QA", + status_code=422, + ) + + raw_reference_geometries = [({"id": str(row.id), "feature_class": row.feature_class}, to_shape(row.geometry)) for row in references] + reference_geometries = raw_reference_geometries + coverage_summary: dict[str, Any] = { "applied": False, "mode": "unbounded_no_manifest", @@ -324,21 +343,19 @@ class DetectionService: "candidate_evaluated_count": len(raw_candidate_geometries), "candidate_excluded_outside_count": 0, "candidate_clipped_boundary_count": 0, - "reference_raw_count": len(raw_reference_geometries), + "reference_raw_count": reference_raw_count, "reference_evaluated_count": len(raw_reference_geometries), "reference_excluded_outside_count": 0, "reference_clipped_boundary_count": 0, } coverage_warnings: list[str] = [] - if manifest_path: - manifest = DetectionService._load_tile_manifest(manifest_path, resolved_settings.yolo_max_tiles) - coverage = DetectionQaService.build_tile_coverage( - manifest, - manifest_path=manifest_path, - expected_dataset_id=run.dataset_id, - ) + if coverage is not None: candidate_population = DetectionQaService.filter_population(raw_candidate_geometries, coverage) - reference_population = DetectionQaService.filter_population(raw_reference_geometries, coverage) + reference_population = DetectionQaService.filter_population( + raw_reference_geometries, + coverage, + raw_count=reference_raw_count, + ) candidate_geometries = candidate_population.geometries reference_geometries = reference_population.geometries if not reference_geometries: @@ -434,7 +451,7 @@ class DetectionService: "candidate_feature_count": len(candidate_geometries), "reference_feature_count": len(reference_geometries), "candidate_feature_count_raw": len(raw_candidate_geometries), - "reference_feature_count_raw": len(raw_reference_geometries), + "reference_feature_count_raw": reference_raw_count, "matches": evidence.matches, "false_positives": evidence.false_positives, "false_negatives": evidence.false_negatives, diff --git a/backend/app/services/qa_service.py b/backend/app/services/qa_service.py index b67fe9d1..c0528ea4 100644 --- a/backend/app/services/qa_service.py +++ b/backend/app/services/qa_service.py @@ -8,6 +8,7 @@ from uuid import UUID from geoalchemy2.shape import to_shape from shapely.geometry import GeometryCollection from shapely.geometry.base import BaseGeometry +from shapely.strtree import STRtree from shapely.ops import unary_union from shapely.validation import make_valid from shapely.geometry import shape @@ -160,7 +161,10 @@ class QaService: ], ) - unmatched_reference_indices = set(range(len(reference_supported))) + reference_tree = STRtree([geometry for _, _, geometry in reference_supported]) + unmatched_reference_indices = { + index for index, (_, _, geometry) in enumerate(reference_supported) if geometry.area > 0 + } evidence = QaMatchEvidence(warnings=[f"Unsupported geometry types: {unsupported}"] if unsupported else [], unsupported=bool(unsupported)) for source_index, source_feature, source_geom in source_supported: @@ -172,11 +176,11 @@ class QaService: best_iou = 0.0 best_index = None - for reference_index in list(unmatched_reference_indices): - _, _, reference_geom = reference_supported[reference_index] - if reference_geom.area <= 0: - unmatched_reference_indices.discard(reference_index) + candidate_reference_indices = sorted(int(index) for index in reference_tree.query(source_geom)) + for reference_index in candidate_reference_indices: + if reference_index not in unmatched_reference_indices: continue + _, _, reference_geom = reference_supported[reference_index] try: intersection = source_geom.intersection(reference_geom) except Exception as exc: # pragma: no cover - robustness path diff --git a/backend/tests/test_sprint184_detection_qa_coverage.py b/backend/tests/test_sprint184_detection_qa_coverage.py index be6ec269..9758fcc8 100644 --- a/backend/tests/test_sprint184_detection_qa_coverage.py +++ b/backend/tests/test_sprint184_detection_qa_coverage.py @@ -8,6 +8,7 @@ from shapely.geometry import box from app.core.errors import AppError from app.services.detection_qa_service import DetectionQaService +from app.services.qa_service import QaService def test_tile_coverage_transforms_projected_manifest_bounds_to_epsg4326() -> None: @@ -78,3 +79,46 @@ def test_coverage_filter_reports_outside_and_boundary_clipped_population() -> No assert population.excluded_outside_count == 1 assert population.clipped_boundary_count == 1 assert population.geometries[1][1].bounds == pytest.approx((0.8, 0.8, 1.0, 1.0)) + + +def test_coverage_filter_preserves_prefiltered_database_population_count() -> None: + dataset_id = uuid4() + coverage = DetectionQaService.build_tile_coverage( + { + "source_dataset_id": str(dataset_id), + "crs": "EPSG:4326", + "tiles": [{"bounds": [0.0, 0.0, 1.0, 1.0]}], + }, + manifest_path="/app/storage/tiles/manifest.json", + expected_dataset_id=dataset_id, + ) + + population = DetectionQaService.filter_population( + [ + ({"id": "inside"}, box(0.1, 0.1, 0.2, 0.2)), + ({"id": "crossing"}, box(0.8, 0.8, 1.2, 1.2)), + ], + coverage, + raw_count=3, + ) + + assert population.raw_count == 3 + assert population.evaluated_count == 2 + assert population.excluded_outside_count == 1 + assert population.clipped_boundary_count == 1 + + +def test_iou_matching_keeps_exact_results_with_many_spatially_disjoint_references() -> None: + references = [({"id": f"outside-{index}"}, box(index + 10, 10, index + 10.5, 10.5)) for index in range(100)] + references.append(({"id": "match"}, box(0.0, 0.0, 1.0, 1.0))) + + evidence = QaService._match_io_u_evidence( + [({"id": "candidate"}, box(0.0, 0.0, 1.0, 1.0))], + references, + 0.5, + ) + + assert evidence.matches == 1 + assert evidence.false_positives == 0 + assert evidence.false_negatives == 100 + assert evidence.match_evidence[0]["reference_feature_id"] == "match" diff --git a/backend/tests/test_sprint8c_detection_visualization_qa.py b/backend/tests/test_sprint8c_detection_visualization_qa.py index 6ed1b1ba..19b44e82 100644 --- a/backend/tests/test_sprint8c_detection_visualization_qa.py +++ b/backend/tests/test_sprint8c_detection_visualization_qa.py @@ -42,6 +42,9 @@ class FakeQuery: def first(self): return self.rows[0] if self.rows else None + def count(self): + return len(self.rows) + class FakeSession: def __init__(self, objects=None, query_rows=None) -> None: diff --git a/docs/AI_PIPELINES.md b/docs/AI_PIPELINES.md index ff18445a..d5656b06 100644 --- a/docs/AI_PIPELINES.md +++ b/docs/AI_PIPELINES.md @@ -548,9 +548,14 @@ Sprint 8C makes persisted detections reviewable: - QA results reuse `quality_checks` and `metrics`; no parallel QA persistence system is introduced. - Configured-YOLO QA derives its evaluation extent from the persisted tile manifest. Tile bounds are transformed from their explicit source CRS to - EPSG:4326, unioned, and used to clip candidate/reference populations before - canonical footprint-IoU matching. Reference features wholly outside the - imagery presented to the model no longer count as false negatives. + EPSG:4326 and unioned. The union is first applied as a GiST-backed PostGIS + spatial predicate, then used to clip the bounded candidate/reference + populations before canonical footprint-IoU matching. Complete source counts + remain in QA evidence, but regional geometries outside inference coverage are + not materialized in application memory and do not count as false negatives. +- Canonical one-to-one IoU matching uses an in-memory spatial index only to + discard geometries whose envelopes cannot intersect. It does not change the + configured IoU threshold, greedy match ownership or persisted metrics. - A separate reference-envelope IoU pass is persisted as `box_to_footprint_diagnostics`. It quantifies possible matching artifacts from comparing rectangular detections with irregular building footprints, but is diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index 36ee8b0d..1f82aa24 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -893,7 +893,11 @@ Response persists a `quality_check` and `metrics` rows through the existing QA/Q Configured-YOLO QA automatically reads `tile_manifest_path` from the persisted `AnalysisRun.parameters_json`. Candidate and reference geometries are clipped to the union of the manifest's tile bounds after explicit CRS transformation to -EPSG:4326. The response additionally returns: +EPSG:4326. Before reference geometries are materialized, the service applies +that coverage with an indexed PostGIS `ST_Intersects` predicate. The full +dataset count is retained separately so raw/evaluated/excluded counts remain +auditable without transferring a regional reference dataset to Python. The +response additionally returns: - `candidate_feature_count_raw` and `reference_feature_count_raw`; - `coverage`, including raw/evaluated/excluded/boundary-clipped population diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 19ba1e9f..3a46a2f7 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -8052,3 +8052,19 @@ Live operational proof: Next: - Reuse the bounded regional operator pattern for current roads, water and parcels, then add official population and land-use time series without changing Mol semantics or introducing interactive external fetching. + +## Sprint 195 - Live guided detection and bounded regional QA (2026-07-14) + +Implemented: +- Added one guided Detection Lab action that uploads a georeferenced raster through `DatasetService`, creates or reuses the canonical tile manifest, runs preflight, invokes the configured local YOLO adapter and opens persisted detection geometry on MapLibre. +- Live Tower validation used the official Mol orthophoto sample, produced nine 512 px tiles and persisted 1,953 detections in analysis run `3912e179-3d1d-4080-8093-f883bbe95d9c`. +- A real QA attempt against the 466,078-feature regional GRB building dataset revealed that coverage clipping happened after every reference row had already been materialized. +- Moved configured-YOLO reference bounding into the existing GiST-indexed PostGIS query with the persisted manifest coverage. Full source count, evaluated count and excluded count remain explicit in the persisted evidence. +- Added a Shapely STRtree candidate index around the unchanged exact IoU matcher and clarified the simple map legend/source whenever AI detections are active. + +Validation before final deployment: +- Focused detection, coverage, QA and guided-workflow tests passed. +- Frontend TypeScript typecheck passed after the source/legend correction. + +Next: +- Deploy the bounded QA correction, rerun the live persisted detection-versus-GRB comparison, verify persisted metrics and inspect the result in MapLibre before declaring the guided operational flow complete. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 720db380..45a85d2a 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -533,18 +533,18 @@ function App(): JSX.Element { const activeWorkspaceItem = workspaceNavItems.find((item) => item.key === activeWorkspace) ?? workspaceNavItems[0] const mapLayerSourceLabel = useMemo(() => { if (analysisMapLayerActive && changeDetectionResult?.geojson) { - return 'Change detection' + return 'Veranderingsanalyse' } if (analysisMapLayerActive && segmentationGeoJson) { - return 'Segmentation run' + return 'Segmentatierun' } if (analysisMapLayerActive && detectionGeoJson) { - return 'Detection run' + return 'Detectierun' } if ((datasetMapContent || viewportVectorLayer.enabled) && selectedDataset) { - return `${selectedDataset.dataset_type} dataset` + return `${selectedDataset.dataset_type}-dataset` } - return 'No active vector or result layer' + return 'Geen actieve gegevens- of analyselaag' }, [analysisMapLayerActive, changeDetectionResult?.geojson, datasetMapContent, detectionGeoJson, segmentationGeoJson, selectedDataset, viewportVectorLayer.enabled]) const mapLayerProvenance = useMemo(() => { if (analysisMapLayerActive && changeDetectionResult?.geojson) { diff --git a/frontend/src/components/map/MapWorkspace.tsx b/frontend/src/components/map/MapWorkspace.tsx index bfb121aa..ab68db5a 100644 --- a/frontend/src/components/map/MapWorkspace.tsx +++ b/frontend/src/components/map/MapWorkspace.tsx @@ -588,6 +588,7 @@ export function MapWorkspace({ ) const activeTheme = DATA_THEMES.find((theme) => theme.id === activeThemeId) ?? DATA_THEMES[0] const activeThemeMapStyle = DATA_THEME_MAP_STYLES[activeTheme.id] + const analysisOverlayActive = mapContentMode === 'analysis' && analysisLayerAvailable && Boolean(mapFeatureCollection) const activeThemeDataset = themeDatasetMap[activeTheme.id] const activeScopeProject = projects.find((project) => project.id === selectedProjectId) ?? null const activeScopeLabel = activeScopeProject ? operationalScopeProjectLabel(activeScopeProject) : 'Werkgebied' @@ -986,14 +987,18 @@ export function MapWorkspace({
- {analysisMode === 'evolution' ? 'Tijdreeks' : 'Actieve bron'} + {analysisOverlayActive ? 'Actieve analyselaag' : analysisMode === 'evolution' ? 'Tijdreeks' : 'Actieve bron'} - {analysisMode === 'evolution' + {analysisOverlayActive + ? mapLayerLabel + : analysisMode === 'evolution' ? activeTemporalSeriesGroup?.label ?? 'Geen tijdreeks beschikbaar' : activeThemeDataset ? getDatasetDisplayName(activeThemeDataset) : 'Geen databron beschikbaar'} - {analysisMode === 'evolution' + {analysisOverlayActive + ? `${mapLayerSourceLabel} · AI-resultaat, controle vereist` + : analysisMode === 'evolution' ? activeTemporalSeries.length >= 2 ? `${activeTemporalSeries.length} officiële meetmomenten · ${formatObservationDate(activeTemporalSeries[0].observed_at)} tot ${formatObservationDate(activeTemporalSeries[activeTemporalSeries.length - 1].observed_at)}` : 'Minstens twee expliciet gedateerde snapshots zijn vereist.' @@ -1113,7 +1118,12 @@ export function MapWorkspace({ />
Werkgebied - {analysisMode === 'evolution' && temporalComparison?.object_changes.available ? ( + {analysisOverlayActive ? ( + <> + Gevonden gebouwen + Selectie + + ) : analysisMode === 'evolution' && temporalComparison?.object_changes.available ? ( <> Nieuw Verdwenen @@ -1301,7 +1311,9 @@ export function MapWorkspace({ Werkgebied: {selectedMapArea?.name ?? 'Geen werkgebied geselecteerd'} Bron:{' '} - {analysisMode === 'evolution' + {analysisOverlayActive + ? `${mapLayerLabel} · ${mapLayerSourceLabel}` + : analysisMode === 'evolution' ? activeTemporalSeriesGroup?.label ?? 'geen vergelijkbare tijdreeks' : activeThemeDataset ? getDatasetDisplayName(activeThemeDataset) diff --git a/frontend/src/hooks/useMapWorkspaceState.ts b/frontend/src/hooks/useMapWorkspaceState.ts index 431c6b4d..1511ad36 100644 --- a/frontend/src/hooks/useMapWorkspaceState.ts +++ b/frontend/src/hooks/useMapWorkspaceState.ts @@ -76,18 +76,18 @@ export function useMapWorkspaceState({ ) const mapLayerLabel = useMemo(() => { if (changeDetectionGeoJson) { - return 'Change detection result' + return 'Veranderingsanalyse' } if (segmentationGeoJson) { - return 'Segmentation result' + return 'AI-segmentaties' } if (detectionGeoJson) { - return 'Detection result' + return 'AI-detecties' } if (datasetLayerActive && selectedDataset) { return selectedDataset.name } - return 'No active vector layer' + return 'Geen actieve kaartlaag' }, [changeDetectionGeoJson, datasetLayerActive, detectionGeoJson, segmentationGeoJson, selectedDataset]) const mapFeatureCount = mapFeatureCollection?.features.length ?? 0 const areaFeatureCount = areaFeatureCollection?.features.length ?? 0