From 232e8c2126216a1fcfe8afb4e02769d0d4f20ca4 Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 15 Jul 2026 12:22:37 +0200 Subject: [PATCH] fix: load complete temporal dataset inventory --- CHANGELOG.md | 3 ++ .../tests/test_sprint203_waterinfo_history.py | 9 ++++++ docs/CODEX_EXECUTION_LOG.md | 19 ++++++++++-- frontend/src/services/api/datasets.ts | 30 +++++++++++++++++-- .../provision_waterinfo_station_history.py | 9 ++++++ 5 files changed, 66 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6cdca413..afbc09c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,9 @@ historical imagery never produces fake current-state quality metrics. - Converted BWK/Natura 2000, agricultural parcels, Buildings Register, DHMV and bathymetry into an ordered acceptance-criteria backlog. +- Made the frontend dataset client exhaust canonical 200-row pages. Real + Waterinfo persistence pushed the regional project beyond the old 50-row + default and exposed that older sources otherwise disappeared from the UI. ## Sprint 202 Source intelligence, full evolution metrics and local assistant (2026-07-15) diff --git a/backend/tests/test_sprint203_waterinfo_history.py b/backend/tests/test_sprint203_waterinfo_history.py index 59c77c58..cbdec207 100644 --- a/backend/tests/test_sprint203_waterinfo_history.py +++ b/backend/tests/test_sprint203_waterinfo_history.py @@ -133,3 +133,12 @@ def test_waterinfo_operator_is_packaged_and_readiness_checked() -> None: assert "COPY scripts/provision_waterinfo_station_history.py" in dockerfile assert '"provision_waterinfo_station_history.py"' in vector_service assert '"sum", "mean", "area_weighted_sum"' in vector_service + assert '"status": "no_stations_in_area"' in (ROOT / "scripts" / "provision_waterinfo_station_history.py").read_text(encoding="utf-8") + + +def test_frontend_loads_all_dataset_pages_after_temporal_import_expansion() -> None: + client = (ROOT / "frontend" / "src" / "services" / "api" / "datasets.ts").read_text(encoding="utf-8") + + assert "DATASET_PAGE_SIZE = 200" in client + assert "offset < (total ?? 0)" in client + assert "items.length !== total" in client diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 9bd50077..d558c6d8 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -8464,7 +8464,22 @@ Implemented: Validation evidence: - Focused Waterinfo/orthophoto/semantic metric tests and frontend typecheck/build passed before the full repository gate. +- Full readiness passed 627 backend tests, 93 documented routes, one Alembic + head and the frontend production build. Tower passed PostGIS 3.6 migration, + schema, frontend proxy and icon smokes. +- Live Mol import persisted 26 annual water-level observations across two + independent stations for 2013-2025. Two stale station series contained no + observations and no in-area annual discharge station was available. +- Live temporal comparison for station `L10_089` returned 30.460 m in 2013 and + 30.455 m in 2025 with all 13 observations, `mean` aggregation and the point- + measurement/no-volume warning. +- Live 2020 bounded WMS acquisition persisted an EPSG:31370 raster with annual + temporal metadata; its constrained PNG endpoint returned HTTP 200. +- Browser validation then exposed that 26 new annual datasets pushed the + project above the frontend's implicit 50-row dataset page. The central API + client now exhausts canonical 200-row pages and fails if totals drift or a + page is incomplete, so older population/parcel sources remain visible. Remaining operational step: -- Deploy the all-in-one image, provision the real Mol Waterinfo station series - and verify one historical image acquisition/overlay through the LAN browser. +- Redeploy the pagination correction and verify complete source visibility plus + the historical overlay through the LAN browser. diff --git a/frontend/src/services/api/datasets.ts b/frontend/src/services/api/datasets.ts index e7789de6..8a8bf13e 100644 --- a/frontend/src/services/api/datasets.ts +++ b/frontend/src/services/api/datasets.ts @@ -20,9 +20,35 @@ import type { OrthophotoProductRead, } from '../../types' +const DATASET_PAGE_SIZE = 200 + +async function listProjectDatasets(projectId: string): Promise { + const items: DatasetCreateResponse[] = [] + let offset = 0 + let total: number | null = null + do { + const page = await apiGet( + `/api/v1/projects/${projectId}/datasets?limit=${DATASET_PAGE_SIZE}&offset=${offset}`, + ) + if (total === null) { + total = page.total + } else if (page.total !== total) { + throw new Error('De datasetlijst wijzigde tijdens het laden. Vernieuw de werkruimte.') + } + items.push(...page.items) + if (page.items.length === 0) { + break + } + offset += page.items.length + } while (offset < (total ?? 0)) + if (total !== null && items.length !== total) { + throw new Error(`Niet alle datasets konden worden geladen (${items.length}/${total}).`) + } + return { items, total: total ?? 0, limit: items.length, offset: 0 } +} + export const datasetsApi = { - list: (projectId: string): Promise => - apiGet(`/api/v1/projects/${projectId}/datasets`), + list: listProjectDatasets, upload: ( projectId: string, payload: { diff --git a/scripts/provision_waterinfo_station_history.py b/scripts/provision_waterinfo_station_history.py index 94200ed3..83eea9e8 100644 --- a/scripts/provision_waterinfo_station_history.py +++ b/scripts/provision_waterinfo_station_history.py @@ -446,6 +446,15 @@ def main() -> int: ) layer_path = args.output_dir / f"waterinfo_{parameter.key}_station_layer.json" layer_sha256 = write_json(layer_path, station_layer) + if not stations: + results.append( + { + "parameter": parameter.key, + "timeseries_group_id": parameter.group_id, + "status": "no_stations_in_area", + "observation_count": 0, + } + ) if len(stations) > args.max_stations: raise RuntimeError( f"Waterinfo returned {len(stations)} in-area {parameter.key} stations; safety limit is {args.max_stations}"