diff --git a/backend/tests/test_sprint188_official_landuse_timeseries.py b/backend/tests/test_sprint188_official_landuse_timeseries.py index 1eaac8bc..039b3411 100644 --- a/backend/tests/test_sprint188_official_landuse_timeseries.py +++ b/backend/tests/test_sprint188_official_landuse_timeseries.py @@ -141,6 +141,41 @@ def test_official_landuse_metadata_keeps_modern_series_separate(tmp_path: Path) assert "historical-landuse" not in module.series_key(theme, "mol") +def test_official_landuse_operator_paginates_within_api_limit() -> None: + module = load_provisioner() + + class Response: + ok = True + status_code = 200 + text = "" + + def __init__(self, payload): + self.payload = payload + + def json(self): + return {"data": self.payload} + + class Session: + def __init__(self) -> None: + self.calls = [] + + def get(self, url, *, params, timeout): + self.calls.append((url, params, timeout)) + offset = params["offset"] + page_items = [{"id": index} for index in range(offset, min(offset + 200, 405))] + return Response({"items": page_items, "total": 405, "limit": 200, "offset": offset}) + + session = Session() + items = module.list_paginated_items(session, "http://backend/api/v1/projects", timeout=30) + + assert len(items) == 405 + assert [call[1] for call in session.calls] == [ + {"limit": 200, "offset": 0}, + {"limit": 200, "offset": 200}, + {"limit": 200, "offset": 400}, + ] + + def test_official_landuse_operator_is_packaged_and_readiness_checked() -> None: readiness = (ROOT / "scripts/run_readiness_check.sh").read_text(encoding="utf-8") dockerfile = (ROOT / "deploy/unraid/Dockerfile.all-in-one").read_text(encoding="utf-8") diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index d7c9662c..f894cb23 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -7837,9 +7837,10 @@ Source validation: - Every raster cell touching the municipality is considered before exact vector clipping. The 916 NoData edge cells in each WCS subset were explicitly excluded and recorded rather than assigned a class. Validation: -- New focused backend suite passed 4 tests. -- Frontend TypeScript typecheck passed after temporal-series selection wiring. -- Full readiness and live Tower/PostGIS/browser validation remain the final steps of this pass. +- The first live import was rejected before any upload because the operator requested 500 datasets while the canonical API limit is 200. The operator now follows every 200-item page and a focused pagination regression test covers 405 records. +- New focused backend suite passed 5 tests. +- Full readiness passed 533 backend tests, one Alembic head, frontend TypeScript typecheck/build and all repository smoke gates. +- Tower rebuilt successfully and passed PostGIS 3.6, migration, container health and browser-proxy validation; live data provisioning remains the final step of this pass. Next: - Deploy and provision all five modern snapshots on Tower, then verify current forest selection, both temporal series and a drawn rectangle in the internal browser. diff --git a/scripts/provision_official_landuse_timeseries.py b/scripts/provision_official_landuse_timeseries.py index fd97664f..d98d0a1f 100644 --- a/scripts/provision_official_landuse_timeseries.py +++ b/scripts/provision_official_landuse_timeseries.py @@ -585,23 +585,47 @@ def response_data(response: requests.Response) -> Any: return payload["data"] +def list_paginated_items(session: requests.Session, url: str, *, timeout: int) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + offset = 0 + total: int | None = None + while total is None or offset < total: + page = response_data(session.get(url, params={"limit": 200, "offset": offset}, timeout=timeout)) + page_items = page.get("items") if isinstance(page, dict) else None + if not isinstance(page_items, list): + raise RuntimeError(f"GeoIntel list response for {url} has no items array") + if total is None: + total = int(page.get("total", len(page_items))) + items.extend(page_items) + if not page_items: + break + offset += len(page_items) + if total is not None and len(items) != total: + raise RuntimeError(f"GeoIntel list response for {url} returned {len(items)} of {total} items") + return items + + def locate_workspace(session: requests.Session, base_url: str, args: argparse.Namespace): - projects = response_data(session.get(f"{base_url}/api/v1/projects", params={"limit": 200}, timeout=args.import_timeout)) - project = next((item for item in projects.get("items") or [] if item.get("name") == args.project_name), None) + projects = list_paginated_items(session, f"{base_url}/api/v1/projects", timeout=args.import_timeout) + project = next((item for item in projects if item.get("name") == args.project_name), None) if not project: raise RuntimeError(f"Project {args.project_name!r} is missing") project_id = str(project["id"]) - areas = response_data( - session.get(f"{base_url}/api/v1/projects/{project_id}/areas", params={"limit": 200}, timeout=args.import_timeout) + areas = list_paginated_items( + session, + f"{base_url}/api/v1/projects/{project_id}/areas", + timeout=args.import_timeout, ) area_fragment = args.area_name.strip().casefold() - matches = [item for item in areas.get("items") or [] if area_fragment in str(item.get("name") or "").casefold()] + matches = [item for item in areas if area_fragment in str(item.get("name") or "").casefold()] if len(matches) != 1: raise RuntimeError(f"Expected one Area matching {args.area_name!r}, received {len(matches)}") - datasets = response_data( - session.get(f"{base_url}/api/v1/projects/{project_id}/datasets", params={"limit": 500}, timeout=args.import_timeout) + datasets = list_paginated_items( + session, + f"{base_url}/api/v1/projects/{project_id}/datasets", + timeout=args.import_timeout, ) - return project_id, str(matches[0]["id"]), list(datasets.get("items") or []) + return project_id, str(matches[0]["id"]), datasets def build_source_metadata(args: argparse.Namespace, snapshot: PreparedSnapshot) -> dict[str, Any]: