From 5b3839dc89e50e7685824b52053132e649c64c6d Mon Sep 17 00:00:00 2001 From: Jens Date: Sat, 22 Aug 2026 15:24:46 +0200 Subject: [PATCH] page detection and segmentation results instead of returning all of them /detection/runs/{id}/detections and its GeoJSON sibling returned every persisted detection, as did the segmentation equivalents. A regional run holds tens of thousands, and these are the endpoints the results table and the map overlay call after every run. They now take limit and offset, default to 2.000, and report total, limit, offset and truncated so the complete population stays visible while what is transferred does not. The GeoJSON responses carry the same window in a geointel_result_window foreign member. Rows are ordered by confidence, so a capped overlay draws the strongest detections rather than an arbitrary slice, and the lab says how many of how many are being shown rather than silently presenting a page as the whole run. Co-Authored-By: Claude Opus 5 --- backend/app/api/routes/detection.py | 34 +++++++- backend/app/api/routes/segmentation.py | 35 +++++++- backend/app/schemas/detection.py | 4 + backend/app/services/detection_service.py | 46 +++++++++- backend/app/services/segmentation_service.py | 11 ++- .../tests/test_detection_result_pagination.py | 86 +++++++++++++++++++ docs/API_CONTRACTS.md | 16 ++++ frontend/src/App.tsx | 4 + .../src/components/detection/DetectionLab.tsx | 12 +++ frontend/src/hooks/useDetectionWorkflow.ts | 11 +++ frontend/src/services/api/detection.ts | 12 ++- frontend/src/types.ts | 8 ++ 12 files changed, 271 insertions(+), 8 deletions(-) create mode 100644 backend/tests/test_detection_result_pagination.py diff --git a/backend/app/api/routes/detection.py b/backend/app/api/routes/detection.py index 324787af..cb5506b4 100644 --- a/backend/app/api/routes/detection.py +++ b/backend/app/api/routes/detection.py @@ -2,7 +2,7 @@ from __future__ import annotations from uuid import UUID -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Query from sqlalchemy.orm import Session from app.db.session import get_db @@ -118,6 +118,13 @@ def list_detection_run_detections( dataset_id: UUID | None = None, class_name: str | None = None, min_confidence: float | None = None, + limit: int = Query( + default=DetectionService.DEFAULT_RESULT_LIMIT, + ge=0, + le=50_000, + description="Maximum results to return; 0 returns everything. Highest confidence first.", + ), + offset: int = Query(default=0, ge=0), db: Session = Depends(get_db), ) -> dict: return envelope( @@ -127,6 +134,8 @@ def list_detection_run_detections( dataset_id=dataset_id, class_name=class_name, min_confidence=min_confidence, + limit=limit, + offset=offset, ).model_dump() ) @@ -140,6 +149,13 @@ def list_dataset_detections( analysis_run_id: UUID | None = None, class_name: str | None = None, min_confidence: float | None = None, + limit: int = Query( + default=DetectionService.DEFAULT_RESULT_LIMIT, + ge=0, + le=50_000, + description="Maximum results to return; 0 returns everything. Highest confidence first.", + ), + offset: int = Query(default=0, ge=0), db: Session = Depends(get_db), ) -> dict: return envelope( @@ -149,6 +165,8 @@ def list_dataset_detections( dataset_id=dataset_id, class_name=class_name, min_confidence=min_confidence, + limit=limit, + offset=offset, ).model_dump() ) @@ -166,11 +184,18 @@ def get_detection_run_geojson( analysis_run_id: UUID, class_name: str | None = None, min_confidence: float | None = None, + limit: int = Query( + default=DetectionService.DEFAULT_RESULT_LIMIT, + ge=0, + le=50_000, + description="Maximum results to return; 0 returns everything. Highest confidence first.", + ), db: Session = Depends(get_db), ) -> dict: return envelope( DetectionService.detections_to_geojson( db, + limit=limit, analysis_run_id=analysis_run_id, class_name=class_name, min_confidence=min_confidence, @@ -187,11 +212,18 @@ def get_dataset_detection_geojson( analysis_run_id: UUID | None = None, class_name: str | None = None, min_confidence: float | None = None, + limit: int = Query( + default=DetectionService.DEFAULT_RESULT_LIMIT, + ge=0, + le=50_000, + description="Maximum results to return; 0 returns everything. Highest confidence first.", + ), db: Session = Depends(get_db), ) -> dict: return envelope( DetectionService.detections_to_geojson( db, + limit=limit, analysis_run_id=analysis_run_id, dataset_id=dataset_id, class_name=class_name, diff --git a/backend/app/api/routes/segmentation.py b/backend/app/api/routes/segmentation.py index 6bed8eca..7894de3c 100644 --- a/backend/app/api/routes/segmentation.py +++ b/backend/app/api/routes/segmentation.py @@ -2,7 +2,7 @@ from __future__ import annotations from uuid import UUID -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Query from sqlalchemy.orm import Session from app.db.session import get_db @@ -21,6 +21,7 @@ from app.schemas import ( SegmentationRunResponse, ) from app.services.model_registry_service import ModelRegistryService +from app.services.detection_service import DetectionService from app.services.segmentation_service import SegmentationService from app.utils.response import envelope @@ -91,11 +92,20 @@ def list_segmentation_run_outputs( dataset_id: UUID | None = None, class_name: str | None = None, min_confidence: float | None = None, + limit: int = Query( + default=DetectionService.DEFAULT_RESULT_LIMIT, + ge=0, + le=50_000, + description="Maximum results to return; 0 returns everything. Highest confidence first.", + ), + offset: int = Query(default=0, ge=0), db: Session = Depends(get_db), ) -> dict: return envelope( SegmentationService.list_segmentations( db, + limit=limit, + offset=offset, analysis_run_id=analysis_run_id, dataset_id=dataset_id, class_name=class_name, @@ -113,11 +123,20 @@ def list_dataset_segmentations( analysis_run_id: UUID | None = None, class_name: str | None = None, min_confidence: float | None = None, + limit: int = Query( + default=DetectionService.DEFAULT_RESULT_LIMIT, + ge=0, + le=50_000, + description="Maximum results to return; 0 returns everything. Highest confidence first.", + ), + offset: int = Query(default=0, ge=0), db: Session = Depends(get_db), ) -> dict: return envelope( SegmentationService.list_segmentations( db, + limit=limit, + offset=offset, analysis_run_id=analysis_run_id, dataset_id=dataset_id, class_name=class_name, @@ -139,11 +158,18 @@ def get_segmentation_run_geojson( analysis_run_id: UUID, class_name: str | None = None, min_confidence: float | None = None, + limit: int = Query( + default=DetectionService.DEFAULT_RESULT_LIMIT, + ge=0, + le=50_000, + description="Maximum results to return; 0 returns everything. Highest confidence first.", + ), db: Session = Depends(get_db), ) -> dict: return envelope( SegmentationService.segmentations_to_geojson( db, + limit=limit, analysis_run_id=analysis_run_id, class_name=class_name, min_confidence=min_confidence, @@ -160,11 +186,18 @@ def get_dataset_segmentation_geojson( analysis_run_id: UUID | None = None, class_name: str | None = None, min_confidence: float | None = None, + limit: int = Query( + default=DetectionService.DEFAULT_RESULT_LIMIT, + ge=0, + le=50_000, + description="Maximum results to return; 0 returns everything. Highest confidence first.", + ), db: Session = Depends(get_db), ) -> dict: return envelope( SegmentationService.segmentations_to_geojson( db, + limit=limit, analysis_run_id=analysis_run_id, dataset_id=dataset_id, class_name=class_name, diff --git a/backend/app/schemas/detection.py b/backend/app/schemas/detection.py index 36f1777c..ddae2bb1 100644 --- a/backend/app/schemas/detection.py +++ b/backend/app/schemas/detection.py @@ -133,7 +133,11 @@ class DetectionRead(BaseModel): class DetectionListResponse(BaseModel): items: list[DetectionRead] + # ``total`` is the complete population; ``items`` is one page of it. total: int + limit: int | None = None + offset: int = 0 + truncated: bool = False class YoloPreflightChecks(BaseModel): diff --git a/backend/app/services/detection_service.py b/backend/app/services/detection_service.py index 52ab48c4..2f052566 100644 --- a/backend/app/services/detection_service.py +++ b/backend/app/services/detection_service.py @@ -297,6 +297,8 @@ class DetectionService: dataset_id: uuid.UUID | None = None, class_name: str | None = None, min_confidence: float | None = None, + limit: int | None = None, + offset: int = 0, ) -> DetectionListResponse: if analysis_run_id is not None: run = db.get(AnalysisRun, analysis_run_id) @@ -309,8 +311,15 @@ class DetectionService: class_name=class_name, min_confidence=min_confidence, ) - items = [DetectionRead.model_validate(row) for row in rows] - return DetectionListResponse(items=items, total=len(items)) + resolved_limit = DetectionService.DEFAULT_RESULT_LIMIT if limit is None else int(limit) + page, total, truncated = DetectionService.paginate(rows, limit=resolved_limit, offset=offset) + return DetectionListResponse( + items=[DetectionRead.model_validate(row) for row in page], + total=total, + limit=resolved_limit, + offset=max(0, int(offset)), + truncated=truncated, + ) @staticmethod def get_detection(db, detection_id: uuid.UUID) -> DetectionRead: @@ -327,16 +336,27 @@ class DetectionService: dataset_id: uuid.UUID | None = None, class_name: str | None = None, min_confidence: float | None = None, + limit: int | None = None, ) -> dict[str, Any]: - detections = DetectionService._query_detection_rows( + rows = DetectionService._query_detection_rows( db, analysis_run_id=analysis_run_id, dataset_id=dataset_id, class_name=class_name, min_confidence=min_confidence, ) + resolved_limit = DetectionService.DEFAULT_RESULT_LIMIT if limit is None else int(limit) + # Rows arrive ranked by confidence, so a capped overlay draws the + # strongest detections rather than an arbitrary slice. + detections, total, truncated = DetectionService.paginate(rows, limit=resolved_limit, offset=0) return { "type": "FeatureCollection", + "geointel_result_window": { + "feature_count": len(detections), + "total_feature_count": total, + "limit": resolved_limit, + "truncated": truncated, + }, "features": [ { "type": "Feature", @@ -738,6 +758,26 @@ class DetectionService: ) return dataset + # A regional run holds tens of thousands of detections; the results table + # and the map overlay both read them after every run. + DEFAULT_RESULT_LIMIT = 2_000 + + @staticmethod + def paginate(rows: list[Any], *, limit: int, offset: int) -> tuple[list[Any], int, bool]: + """Slice a result population, keeping the total intact. + + ``limit <= 0`` means "everything", for callers that genuinely need the + whole population and know what they are asking for. + """ + + total = len(rows) + start = max(0, int(offset)) + if limit <= 0: + return rows[start:], total, False + page = rows[start : start + int(limit)] + # Truncated means: this page is not the whole population. + return page, total, len(page) < total + @staticmethod def _query_detection_rows( db, diff --git a/backend/app/services/segmentation_service.py b/backend/app/services/segmentation_service.py index a606d65e..4022b9f8 100644 --- a/backend/app/services/segmentation_service.py +++ b/backend/app/services/segmentation_service.py @@ -290,16 +290,25 @@ class SegmentationService: dataset_id: uuid.UUID | None = None, class_name: str | None = None, min_confidence: float | None = None, + limit: int | None = None, ) -> dict[str, Any]: - segmentations = SegmentationService._query_segmentation_rows( + rows = SegmentationService._query_segmentation_rows( db, analysis_run_id=analysis_run_id, dataset_id=dataset_id, class_name=class_name, min_confidence=min_confidence, ) + resolved_limit = DetectionService.DEFAULT_RESULT_LIMIT if limit is None else int(limit) + segmentations, total, truncated = DetectionService.paginate(rows, limit=resolved_limit, offset=0) return { "type": "FeatureCollection", + "geointel_result_window": { + "feature_count": len(segmentations), + "total_feature_count": total, + "limit": resolved_limit, + "truncated": truncated, + }, "features": [ { "type": "Feature", diff --git a/backend/tests/test_detection_result_pagination.py b/backend/tests/test_detection_result_pagination.py new file mode 100644 index 00000000..37725168 --- /dev/null +++ b/backend/tests/test_detection_result_pagination.py @@ -0,0 +1,86 @@ +"""Loading a run's results must not depend on the run being small. + +``/detection/runs/{id}/detections`` and its GeoJSON sibling returned every +persisted detection. A regional run holds tens of thousands, so the endpoints +the map and the results table call after every run grew without bound. The +counts stay complete; what is transferred does not. +""" + +from __future__ import annotations + +import uuid + +import pytest + +from app.services.detection_service import DetectionService + + +class _Detection: + def __init__(self, index: int) -> None: + self.id = uuid.uuid4() + self.index = index + + +def _rows(count: int) -> list[_Detection]: + return [_Detection(index) for index in range(count)] + + +def test_a_page_is_returned_with_the_complete_total() -> None: + page, total, truncated = DetectionService.paginate(_rows(1_000), limit=100, offset=0) + + assert len(page) == 100 + assert total == 1_000 + assert truncated is True + + +def test_the_offset_walks_the_population() -> None: + page, total, _ = DetectionService.paginate(_rows(10), limit=3, offset=6) + + assert [row.index for row in page] == [6, 7, 8] + assert total == 10 + + +def test_an_offset_past_the_end_yields_an_empty_page_not_an_error() -> None: + page, total, truncated = DetectionService.paginate(_rows(5), limit=10, offset=50) + + assert page == [] + assert total == 5 + assert truncated is True + + +def test_a_population_inside_one_page_is_not_reported_as_truncated() -> None: + page, total, truncated = DetectionService.paginate(_rows(7), limit=100, offset=0) + + assert len(page) == 7 + assert total == 7 + assert truncated is False + + +def test_a_zero_limit_returns_everything_for_callers_that_need_it() -> None: + page, total, truncated = DetectionService.paginate(_rows(2_500), limit=0, offset=0) + + assert len(page) == 2_500 + assert total == 2_500 + assert truncated is False + + +def test_a_negative_offset_is_treated_as_the_start() -> None: + page, _, _ = DetectionService.paginate(_rows(4), limit=2, offset=-5) + + assert [row.index for row in page] == [0, 1] + + +@pytest.mark.parametrize("limit", [1, 2, 3]) +def test_paging_covers_the_population_exactly_once(limit: int) -> None: + rows = _rows(7) + seen: list[int] = [] + offset = 0 + while True: + page, total, _ = DetectionService.paginate(rows, limit=limit, offset=offset) + if not page: + break + seen.extend(row.index for row in page) + offset += limit + + assert seen == list(range(7)) + assert total == 7 diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index f5f6e57d..37eb2a50 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -1081,6 +1081,22 @@ Return vector stats (feature counts and geometry summary). Return vector bounds and feature count. +### Detection and segmentation result windows + +`GET /detection/runs/{id}/detections`, `/detection/datasets/{id}/detections` +and their segmentation equivalents accept `limit` (default 2.000, `0` for +everything) and `offset`, and return `total`, `limit`, `offset` and +`truncated`. `total` always describes the complete population; `items` is one +page of it. + +The `/geojson` siblings accept `limit` and report the window in a +`geointel_result_window` foreign member. Rows are ordered by confidence, so a +capped overlay draws the strongest detections rather than an arbitrary slice. + +A regional run holds tens of thousands of detections, and these are the +endpoints the results table and the map overlay call after every run; they +previously returned all of them. + ### GET `/api/v1/projects/{project_id}/quality-checks/{id}/evidence/geojson` Returns the reviewable geometry behind one quality check: the objects the model diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 27bc2235..3f255793 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -350,6 +350,8 @@ function WorkbenchApp({ username, accessMode, loggingOut, onLogout }: WorkbenchA detectionRuns, selectedDetectionRunId, detectionItems, + detectionTotal, + detectionTruncated, detectionGeoJson, detectionClassFilter, detectionMinConfidenceFilter, @@ -1233,6 +1235,8 @@ function WorkbenchApp({ username, accessMode, loggingOut, onLogout }: WorkbenchA detectionWorkflowStage={detectionWorkflowStage} selectedDetectionRunId={selectedDetectionRunId} detectionItems={detectionItems} + detectionTotal={detectionTotal} + detectionTruncated={detectionTruncated} detectionClassFilter={detectionClassFilter} detectionMinConfidenceFilter={detectionMinConfidenceFilter} loadingDetectionResults={loadingDetectionResults} diff --git a/frontend/src/components/detection/DetectionLab.tsx b/frontend/src/components/detection/DetectionLab.tsx index 91a133a1..8ddd6106 100644 --- a/frontend/src/components/detection/DetectionLab.tsx +++ b/frontend/src/components/detection/DetectionLab.tsx @@ -98,6 +98,9 @@ interface DetectionLabProps { detectionWorkflowStage: DetectionWorkflowStage selectedDetectionRunId: string detectionItems: DetectionRead[] + /** Complete population behind the returned page. */ + detectionTotal?: number + detectionTruncated?: boolean detectionClassFilter: string detectionMinConfidenceFilter: number loadingDetectionResults: boolean @@ -159,6 +162,8 @@ export function DetectionLab({ detectionWorkflowStage, selectedDetectionRunId, detectionItems, + detectionTotal, + detectionTruncated = false, detectionClassFilter, detectionMinConfidenceFilter, loadingDetectionResults, @@ -679,6 +684,13 @@ export function DetectionLab({ : null}
+ {detectionTruncated ? ( +

+ Deze run leverde {(detectionTotal ?? detectionItems.length).toLocaleString('nl-BE')} detecties op; + hieronder en op de kaart staan de {detectionItems.length.toLocaleString('nl-BE')} met de hoogste + zekerheid. De tellingen in de kwaliteitscontrole gebruiken de volledige run. +

+ ) : null}

Gevonden objecten

diff --git a/frontend/src/hooks/useDetectionWorkflow.ts b/frontend/src/hooks/useDetectionWorkflow.ts index 46458efb..334a4350 100644 --- a/frontend/src/hooks/useDetectionWorkflow.ts +++ b/frontend/src/hooks/useDetectionWorkflow.ts @@ -109,6 +109,9 @@ export function useDetectionWorkflow({ const [detectionRuns, setDetectionRuns] = useState([]) const [selectedDetectionRunId, setSelectedDetectionRunId] = useState('') const [detectionItems, setDetectionItems] = useState([]) + // The server returns one page; a regional run holds far more than it draws. + const [detectionTotal, setDetectionTotal] = useState(0) + const [detectionTruncated, setDetectionTruncated] = useState(false) const [detectionGeoJson, setDetectionGeoJson] = useState(null) const [detectionClassFilter, setDetectionClassFilter] = useState('') const [detectionMinConfidenceFilter, setDetectionMinConfidenceFilter] = useState(0) @@ -200,6 +203,10 @@ export function useDetectionWorkflow({ const loadDetectionResults = async (analysisRunId = selectedDetectionRunId) => { if (!analysisRunId) { setDetectionItems([]) + setDetectionTotal(0) + setDetectionTruncated(false) + setDetectionTotal(0) + setDetectionTruncated(false) setDetectionGeoJson(null) return } @@ -216,6 +223,8 @@ export function useDetectionWorkflow({ detectionApi.getRunGeoJson(analysisRunId, params), ]) setDetectionItems(detectionsResponse.items) + setDetectionTotal(detectionsResponse.total) + setDetectionTruncated(Boolean(detectionsResponse.truncated)) setDetectionGeoJson(geoJsonResponse) } catch (error) { setDetectionRunError(formatError(error, 'Failed to load detection results')) @@ -563,6 +572,8 @@ export function useDetectionWorkflow({ detectionRuns, selectedDetectionRunId, detectionItems, + detectionTotal, + detectionTruncated, detectionGeoJson, detectionClassFilter, detectionMinConfidenceFilter, diff --git a/frontend/src/services/api/detection.ts b/frontend/src/services/api/detection.ts index 7588c7d3..cb536073 100644 --- a/frontend/src/services/api/detection.ts +++ b/frontend/src/services/api/detection.ts @@ -36,12 +36,20 @@ export const detectionApi = { apiGet(`/api/v1/detection/runs/${analysisRunId}`), listDetections: ( analysisRunId: string, - params: { project_id: string; dataset_id?: string | null; class_name?: string | null; min_confidence?: number | null }, + params: { + project_id: string + dataset_id?: string | null + class_name?: string | null + min_confidence?: number | null + // A run holds tens of thousands of detections; the response is one page. + limit?: number | null + offset?: number | null + }, ): Promise => apiGet(`/api/v1/detection/runs/${analysisRunId}/detections${queryString(params)}`), getRunGeoJson: ( analysisRunId: string, - params: { project_id: string; class_name?: string | null; min_confidence?: number | null }, + params: { project_id: string; class_name?: string | null; min_confidence?: number | null; limit?: number | null }, ): Promise => apiGet(`/api/v1/detection/runs/${analysisRunId}/geojson${queryString(params)}`), compareWithReference: (analysisRunId: string, projectId: string, payload: DetectionQaRequest): Promise => diff --git a/frontend/src/types.ts b/frontend/src/types.ts index d5bea5a7..ac0301e4 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -1418,8 +1418,12 @@ export interface DetectionRead { } export interface DetectionListResponse { + /** One page of results; `total` is the complete population. */ items: DetectionRead[] total: number + limit?: number | null + offset?: number + truncated?: boolean } export interface DetectionQaRequest { @@ -1583,8 +1587,12 @@ export interface SegmentationRead { } export interface SegmentationListResponse { + /** One page of results; `total` is the complete population. */ items: SegmentationRead[] total: number + limit?: number | null + offset?: number + truncated?: boolean } export interface SegmentationQaRequest {