diff --git a/CHANGELOG.md b/CHANGELOG.md index 21e6ccea..51191e45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,14 @@ # Changelog +## Sprint 182 Municipality viewport delivery and bounded AI handoff (2026-07-14) + +- Replaced full-file loading for vector datasets above 5,000 features with debounced, zoom-aware requests to the existing persisted PostGIS bbox-selection endpoint. +- Kept each viewport response bounded to the canonical 1,000-feature maximum and made zoom-required, loading, visible/total, truncation and error states explicit in the Map workspace. +- Prevented viewport responses from repeatedly fitting the map back to their own bounds while preserving AOI framing and existing behavior for small vectors, detection/segmentation results, selections and QA evidence. +- Hardened the real raster/detection/QA operator smoke so it can reuse a validated existing project, attach both uploads to a persisted analysis Area and apply safe distinct upload filenames. +- Added focused contract and frontend-wiring regression coverage. No migration, model dependency, provider fetch behavior or persistence schema changed. + ## Sprint 181 Complete Mol municipality workspace (2026-07-14) - Added an explicit operator provisioner for the official Digitaal Vlaanderen Mol municipality boundary (NIS `13025`) and the complete GRB GBG building population clipped to that boundary. diff --git a/backend/README.md b/backend/README.md index 100e6ee2..b3d8d3eb 100644 --- a/backend/README.md +++ b/backend/README.md @@ -907,6 +907,11 @@ returns a canonical-envelope GeoJSON FeatureCollection. It is intended for the Map workspace area-extract flow and does not create derived datasets or export records by itself. +The same bounded endpoint is the canonical large-layer map delivery path. The +frontend requests at most 1,000 features for the current viewport and surfaces +the response `truncated` flag; the backend does not provide or imply an +unbounded municipality-wide map response. + `POST /api/v1/projects/{project_id}/datasets/{dataset_id}/vector/select/derive` uses the same persisted `vector_features` selection but writes the result as a new derived vector dataset. The created dataset uses diff --git a/backend/tests/test_sprint121_real_data_detection_qa_smoke.py b/backend/tests/test_sprint121_real_data_detection_qa_smoke.py index 85cb0a88..99860359 100644 --- a/backend/tests/test_sprint121_real_data_detection_qa_smoke.py +++ b/backend/tests/test_sprint121_real_data_detection_qa_smoke.py @@ -14,10 +14,13 @@ def test_real_data_detection_qa_smoke_requires_operator_inputs_and_checks_full_c assert "bash -n scripts/verify_real_data_detection_qa_workflow.sh" in readiness assert "REAL_RASTER_PATH" in script assert "REAL_REFERENCE_VECTOR_PATH" in script + assert "REAL_PROJECT_ID" in script + assert "REAL_DATASET_NAME_PREFIX" in script assert "usage()" in script assert "/api/v1/projects" in script assert "/datasets/upload" in script assert "dataset_role=reference" in script + assert 'area_upload_args=(-F "area_id=${area_id}")' in script assert "reference_layer_name=buildings" in script assert "/raster/inspect" in script assert "/raster/tile" in script diff --git a/backend/tests/test_sprint182_viewport_vector_delivery.py b/backend/tests/test_sprint182_viewport_vector_delivery.py new file mode 100644 index 00000000..e448ffc0 --- /dev/null +++ b/backend/tests/test_sprint182_viewport_vector_delivery.py @@ -0,0 +1,48 @@ +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from app.schemas.operations import VectorSelectionBBox, VectorSelectionRequest + + +ROOT = Path(__file__).resolve().parents[2] + + +def test_vector_selection_contract_keeps_an_explicit_bounded_limit() -> None: + bbox = VectorSelectionBBox(min_x=5.03, min_y=51.15, max_x=5.25, max_y=51.33) + + request = VectorSelectionRequest(bbox=bbox, limit=1000) + + assert request.limit == 1000 + with pytest.raises(ValidationError): + VectorSelectionRequest(bbox=bbox, limit=1001) + + +def test_large_vector_delivery_uses_existing_postgis_bbox_contract() -> None: + config = (ROOT / "frontend" / "src" / "config" / "vectorDelivery.ts").read_text(encoding="utf-8") + viewport_hook = (ROOT / "frontend" / "src" / "hooks" / "useViewportVectorLayer.ts").read_text(encoding="utf-8") + dataset_hook = (ROOT / "frontend" / "src" / "hooks" / "useDatasetWorkflow.ts").read_text(encoding="utf-8") + + assert "VECTOR_VIEWPORT_FEATURE_THRESHOLD = 5_000" in config + assert "VECTOR_VIEWPORT_MIN_ZOOM = 14" in config + assert "VECTOR_VIEWPORT_FEATURE_LIMIT = 1_000" in config + assert "datasetsApi.selectVectorFeatures" in viewport_hook + assert "response.truncated" in viewport_hook + assert "requestSequence" in viewport_hook + assert "summary.feature_count ?? dataset.feature_count" in dataset_hook + assert "datasetsApi.getContent" in dataset_hook + + +def test_map_reports_viewport_and_does_not_refit_each_slice() -> None: + app = (ROOT / "frontend" / "src" / "App.tsx").read_text(encoding="utf-8") + geo_map = (ROOT / "frontend" / "src" / "components" / "GeoMap.tsx").read_text(encoding="utf-8") + workspace = (ROOT / "frontend" / "src" / "components" / "map" / "MapWorkspace.tsx").read_text(encoding="utf-8") + + assert "useViewportVectorLayer" in app + assert "fitMapDataOnChange={!viewportVectorLayerActive}" in app + assert "map.on('moveend', emitViewport)" in geo_map + assert "fitDataOnChange" in geo_map + assert "onViewportChange" in geo_map + assert "Zoom in to level" in (ROOT / "frontend" / "src" / "hooks" / "useViewportVectorLayer.ts").read_text(encoding="utf-8") + assert "viewport-vector-status" in workspace diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index 2e61bc4b..8e5448a8 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -405,6 +405,8 @@ Rules: - Coordinates are EPSG:4326 longitude/latitude. - Results are generated from persisted PostGIS `vector_features`, not from client-side map data. - The response is capped by `limit` and returns `truncated=true` when more matching rows exist. +- `limit` is bounded to `1..1000`. Municipality-scale clients must page spatially by viewport instead of requesting an unbounded municipality FeatureCollection. +- The Map workspace uses this existing endpoint for vector datasets above 5,000 features. It starts delivery at zoom level 14, debounces `moveend` requests and explicitly reports `truncated=true` as a request to zoom further in. This is a client delivery policy, not a second API or persistence path. ### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/vector/select/derive` diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 4137e6f8..c866cc37 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -7474,3 +7474,39 @@ Open: layering additional municipality-wide reference classes. Then acquire and tile georeferenced imagery only for an explicitly selected Mol analysis zone and run configured-YOLO plus QA/QC against the persisted GRB reference. + +# Sprint 182 - Municipality viewport delivery and bounded AI handoff + +## Implementation + +- Added `useViewportVectorLayer` with centralized feature threshold, minimum + zoom, response limit and debounce policy. Vector datasets above 5,000 + features now use the existing canonical PostGIS bbox-selection endpoint from + zoom level 14 instead of downloading the full stored GeoJSON file. +- Added MapLibre `moveend` viewport reporting, stale-request protection and + disabled automatic fit for viewport slices. AOI changes still frame the map; + small vectors and persisted analysis/evidence layers retain their existing + complete-layer fit behavior. +- Added explicit low-zoom, loading, visible/total, truncation and error states. + A selected large database layer is no longer presented as an empty or missing + layer before the first detail request. +- Extended the real-data detection/QA workflow with validated existing-project + reuse, safe distinct upload filenames and actual Area linkage for the raster + and matching reference vector. The workflow still consumes operator-provided + files and an existing local model asset; it performs no provider fetch or + model download. + +## Local validation + +- `python -m compileall backend/app` passed. +- Full backend suite passed: `500` tests. +- `npm run typecheck` and `npm run build` passed; the production build contains + 85 transformed modules and preserves the dedicated MapLibre chunk. +- `bash scripts/run_readiness_check.sh` passed with the 81-route API contract + audit, one Alembic head `202606120900`, shell syntax checks and full frontend + build. +- Internal browser validation against the local frontend and live Tower API + selected `mol_grb_gbg_buildings.geojson`, showed the zoom-14 PostGIS delivery + guard, suppressed the incorrect empty-layer state and loaded an actual + viewport slice with `2 visible of 36,941 total features`. No browser warnings + or errors were recorded. diff --git a/docs/TODO.md b/docs/TODO.md index 304554d0..9d628552 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -525,6 +525,16 @@ This file now starts with the current implementation status. Older preparation/b - [ ] Acquire and tile a georeferenced raster only for an explicitly selected Mol analysis zone before running the next configured-YOLO validation. +# Sprint 182 - Municipality viewport delivery and bounded AI handoff + +- [x] Stop downloading municipality-scale vector datasets as one frontend GeoJSON payload. +- [x] Deliver large persisted layers through the existing PostGIS bbox endpoint from zoom level 14. +- [x] Report visible/total counts and explicit truncation instead of silently hiding omitted features. +- [x] Preserve complete loading and auto-fit behavior for small vectors and persisted AI/QA overlays. +- [x] Allow the real raster/detection/QA operator smoke to reuse the definitive Mol project. +- [x] Link operator raster/reference uploads to the persisted analysis Area and keep their names distinct from municipality-wide layers. +- [ ] Execute the bounded Mol-Centrum configured-YOLO/QA workflow in the deployed runtime and retain its persisted ids as release evidence. + # Sprint 171 - Positive AOI expansion and small-building recovery - [x] Reject cross-model false-negative comparisons when reference populations differ. diff --git a/frontend/README.md b/frontend/README.md index a70bcbec..167dcb0e 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -20,6 +20,8 @@ When the public OpenStreetMap fallback is active, the Map workspace shows a base Operational GIS testing is now available directly in the Map workspace. Users can choose a persisted vector database layer, load it on the map, reuse the selected AOI or active layer extent, run the existing persisted `vector_features` bbox query, save the result as a derived dataset, export the selection GeoJSON, choose a reference dataset and launch QA/QC without creating fake data or a parallel backend path. The guided workflow also includes a one-click full run action that executes query, derived dataset save, GeoJSON export and optional QA/QC in sequence with visible status. For repeated review, switch the run mode from `Create new dataset/export` to `Reuse latest saved dataset for QA`; this reruns QA/QC against the latest saved derived dataset without creating another dataset/export pair. +Large vector layers use viewport delivery instead of downloading one unbounded GeoJSON file. Datasets above 5,000 features load their persisted PostGIS geometries from the existing bbox-selection endpoint at zoom level 14 or closer, with a 1,000-feature cap per view. The Map workspace shows visible versus total feature counts and asks the operator to zoom further when the response is truncated. Small layers, AI result overlays, selections and QA evidence keep their existing complete-FeatureCollection behavior. + QA/QC and Exports follow the same calmer density model. QA/QC keeps metric evidence, feature ids and raw findings available but compresses provenance and history surfaces so review starts from the selected check and map evidence actions. Exports uses denser handoff cards, latest-artifact cards and history filters so artifact creation and download paths are easier to scan. When project data loads and no dataset is selected yet, the workbench auto-opens the first ready vector dataset. This gives Data, Map and Exports an immediately usable default context while preserving explicit user selection once the user picks another dataset. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 4c38a43c..e1572791 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -29,6 +29,7 @@ import { useProjectWorkspace } from './hooks/useProjectWorkspace' import { useQualityWorkflow } from './hooks/useQualityWorkflow' import { useSegmentationWorkflow } from './hooks/useSegmentationWorkflow' import { useWorkbenchBootstrap } from './hooks/useWorkbenchBootstrap' +import { useViewportVectorLayer } from './hooks/useViewportVectorLayer' function isVectorDatasetType(datasetType: string): boolean { return datasetType === 'vector' || datasetType === 'geojson' @@ -344,6 +345,15 @@ function App(): JSX.Element { loadProjectData, loadQualityChecks, }) + const viewportVectorLayer = useViewportVectorLayer({ + selectedProjectId, + selectedDataset, + featureCount: selectedDatasetSummary?.feature_count ?? selectedDataset?.feature_count ?? null, + isVectorDatasetType, + }) + const analysisMapLayerActive = Boolean(changeDetectionResult?.geojson || segmentationGeoJson || detectionGeoJson) + const viewportVectorLayerActive = viewportVectorLayer.enabled && !analysisMapLayerActive + const datasetMapContent = viewportVectorLayer.enabled ? viewportVectorLayer.data : datasetContent const useRasterTileManifestForSegmentation = () => { const manifestPath = latestRasterTileManifestPath.trim() if (!manifestPath || !selectedDataset || selectedDataset.dataset_type !== 'raster') { @@ -403,8 +413,9 @@ function App(): JSX.Element { changeDetectionGeoJson: changeDetectionResult?.geojson ?? null, segmentationGeoJson, detectionGeoJson, - datasetContent, + datasetContent: datasetMapContent, selectedDataset, + datasetLayerActive: Boolean(selectedDataset && isVectorDatasetType(selectedDataset.dataset_type)), }) const { mapSelectionBbox, @@ -519,11 +530,11 @@ function App(): JSX.Element { if (detectionGeoJson) { return 'Detection run' } - if (datasetContent && selectedDataset) { + if ((datasetMapContent || viewportVectorLayer.enabled) && selectedDataset) { return `${selectedDataset.dataset_type} dataset` } return 'No active vector or result layer' - }, [changeDetectionResult?.geojson, datasetContent, detectionGeoJson, segmentationGeoJson, selectedDataset]) + }, [changeDetectionResult?.geojson, datasetMapContent, detectionGeoJson, segmentationGeoJson, selectedDataset, viewportVectorLayer.enabled]) const mapLayerProvenance = useMemo(() => { if (changeDetectionResult?.geojson) { return `source ${changeSourceDatasetId || 'n/a'} -> target ${changeTargetDatasetId || 'n/a'}` @@ -534,7 +545,7 @@ function App(): JSX.Element { if (detectionGeoJson) { return selectedDetectionRunId ? `analysis run ${selectedDetectionRunId}` : 'detection results loaded' } - if (datasetContent && selectedDataset) { + if ((datasetMapContent || viewportVectorLayer.enabled) && selectedDataset) { return `${selectedDataset.dataset_role ?? 'source'} / ${selectedDataset.source_name ?? selectedDataset.source}` } return 'Open a dataset, detection run, segmentation run or change result to draw it here.' @@ -542,12 +553,13 @@ function App(): JSX.Element { changeDetectionResult?.geojson, changeSourceDatasetId, changeTargetDatasetId, - datasetContent, + datasetMapContent, detectionGeoJson, segmentationGeoJson, selectedDataset, selectedDetectionRunId, selectedSegmentationRunId, + viewportVectorLayer.enabled, ]) const openDatasetInMap = (dataset: DatasetCreateResponse) => { if (selectedProjectId) { @@ -643,7 +655,11 @@ function App(): JSX.Element { const projectContextLabel = selectedProject?.name ?? 'No project' const areaContextLabel = selectedArea?.name ?? (areas.length > 0 ? 'Select area' : 'No AOI') const datasetContextLabel = selectedDataset?.name ?? (datasets.length > 0 ? 'Select dataset' : 'No dataset') - const layerContextLabel = mapFeatureCollection ? `${mapFeatureCount} features` : 'No active layer' + const layerContextLabel = mapFeatureCollection + ? `${mapFeatureCount} features` + : viewportVectorLayerActive + ? 'Viewport layer selected' + : 'No active layer' return (
Spatial review
Open a dataset, detection run, segmentation run or change result to draw it here.
diff --git a/frontend/src/config/vectorDelivery.ts b/frontend/src/config/vectorDelivery.ts new file mode 100644 index 00000000..6effa90a --- /dev/null +++ b/frontend/src/config/vectorDelivery.ts @@ -0,0 +1,4 @@ +export const VECTOR_VIEWPORT_FEATURE_THRESHOLD = 5_000 +export const VECTOR_VIEWPORT_MIN_ZOOM = 14 +export const VECTOR_VIEWPORT_FEATURE_LIMIT = 1_000 +export const VECTOR_VIEWPORT_DEBOUNCE_MS = 250 diff --git a/frontend/src/hooks/useDatasetWorkflow.ts b/frontend/src/hooks/useDatasetWorkflow.ts index ff408606..b39adea8 100644 --- a/frontend/src/hooks/useDatasetWorkflow.ts +++ b/frontend/src/hooks/useDatasetWorkflow.ts @@ -12,6 +12,7 @@ import type { } from '../types' import { formatError } from '../lib/formatError' import { isPrimaryFocusMunicipalityBoundaryDataset } from '../config/primaryFocus' +import { VECTOR_VIEWPORT_FEATURE_THRESHOLD } from '../config/vectorDelivery' interface DatasetWorkflowOptions { selectedProjectId: string | null @@ -165,12 +166,13 @@ export function useDatasetWorkflow({ setJobs([]) try { if (isVectorDatasetType(dataset.dataset_type)) { - const [content, summary] = await Promise.all([ - datasetsApi.getContent(projectId, dataset.id), - datasetsApi.vectorSummary(projectId, dataset.id), - ]) - setDatasetContent(content) + const summary = await datasetsApi.vectorSummary(projectId, dataset.id) setSelectedDatasetSummary(summary) + const featureCount = summary.feature_count ?? dataset.feature_count + if (featureCount == null || featureCount <= VECTOR_VIEWPORT_FEATURE_THRESHOLD) { + const content = await datasetsApi.getContent(projectId, dataset.id) + setDatasetContent(content) + } } else if (dataset.dataset_type === 'raster') { try { const rasterInspection = await datasetsApi.rasterInspect(projectId, dataset.id) diff --git a/frontend/src/hooks/useMapWorkspaceState.ts b/frontend/src/hooks/useMapWorkspaceState.ts index e90283dd..ac914503 100644 --- a/frontend/src/hooks/useMapWorkspaceState.ts +++ b/frontend/src/hooks/useMapWorkspaceState.ts @@ -8,6 +8,7 @@ interface MapWorkspaceStateOptions { detectionGeoJson: GeoJSON.FeatureCollection | null datasetContent: GeoJSON.FeatureCollection | null selectedDataset: DatasetCreateResponse | null + datasetLayerActive?: boolean } export function useMapWorkspaceState({ @@ -17,6 +18,7 @@ export function useMapWorkspaceState({ detectionGeoJson, datasetContent, selectedDataset, + datasetLayerActive = Boolean(datasetContent), }: MapWorkspaceStateOptions) { const [mapLayerVisible, setMapLayerVisible] = useState(true) const [mapLayerOpacity, setMapLayerOpacity] = useState(0.4) @@ -72,11 +74,11 @@ export function useMapWorkspaceState({ if (detectionGeoJson) { return 'Detection result' } - if (datasetContent && selectedDataset) { + if (datasetLayerActive && selectedDataset) { return selectedDataset.name } return 'No active vector layer' - }, [changeDetectionGeoJson, datasetContent, detectionGeoJson, segmentationGeoJson, selectedDataset]) + }, [changeDetectionGeoJson, datasetLayerActive, detectionGeoJson, segmentationGeoJson, selectedDataset]) const mapFeatureCount = mapFeatureCollection?.features.length ?? 0 const areaFeatureCount = areaFeatureCollection?.features.length ?? 0 diff --git a/frontend/src/hooks/useViewportVectorLayer.ts b/frontend/src/hooks/useViewportVectorLayer.ts new file mode 100644 index 00000000..5bf754c2 --- /dev/null +++ b/frontend/src/hooks/useViewportVectorLayer.ts @@ -0,0 +1,134 @@ +import { useEffect, useMemo, useRef, useState } from 'react' +import { + VECTOR_VIEWPORT_DEBOUNCE_MS, + VECTOR_VIEWPORT_FEATURE_LIMIT, + VECTOR_VIEWPORT_FEATURE_THRESHOLD, + VECTOR_VIEWPORT_MIN_ZOOM, +} from '../config/vectorDelivery' +import { formatError } from '../lib/formatError' +import { datasetsApi } from '../services/api/datasets' +import type { DatasetCreateResponse, MapViewportState } from '../types' + +interface ViewportVectorLayerOptions { + selectedProjectId: string | null + selectedDataset: DatasetCreateResponse | null + featureCount: number | null + isVectorDatasetType: (datasetType: string) => boolean +} + +export function useViewportVectorLayer({ + selectedProjectId, + selectedDataset, + featureCount, + isVectorDatasetType, +}: ViewportVectorLayerOptions) { + const [viewport, setViewport] = useState