fix: paginate official land-use workspace lookup

scripts/provision_official_landuse_timeseries.py

@@ -585,23 +585,47 @@ def response_data(response: requests.Response) -> Any:
     return payload["data"]
 
 
+def list_paginated_items(session: requests.Session, url: str, *, timeout: int) -> list[dict[str, Any]]:
+    items: list[dict[str, Any]] = []
+    offset = 0
+    total: int | None = None
+    while total is None or offset < total:
+        page = response_data(session.get(url, params={"limit": 200, "offset": offset}, timeout=timeout))
+        page_items = page.get("items") if isinstance(page, dict) else None
+        if not isinstance(page_items, list):
+            raise RuntimeError(f"GeoIntel list response for {url} has no items array")
+        if total is None:
+            total = int(page.get("total", len(page_items)))
+        items.extend(page_items)
+        if not page_items:
+            break
+        offset += len(page_items)
+    if total is not None and len(items) != total:
+        raise RuntimeError(f"GeoIntel list response for {url} returned {len(items)} of {total} items")
+    return items
+
+
 def locate_workspace(session: requests.Session, base_url: str, args: argparse.Namespace):
-    projects = response_data(session.get(f"{base_url}/api/v1/projects", params={"limit": 200}, timeout=args.import_timeout))
-    project = next((item for item in projects.get("items") or [] if item.get("name") == args.project_name), None)
+    projects = list_paginated_items(session, f"{base_url}/api/v1/projects", timeout=args.import_timeout)
+    project = next((item for item in projects if item.get("name") == args.project_name), None)
     if not project:
         raise RuntimeError(f"Project {args.project_name!r} is missing")
     project_id = str(project["id"])
-    areas = response_data(
-        session.get(f"{base_url}/api/v1/projects/{project_id}/areas", params={"limit": 200}, timeout=args.import_timeout)
+    areas = list_paginated_items(
+        session,
+        f"{base_url}/api/v1/projects/{project_id}/areas",
+        timeout=args.import_timeout,
     )
     area_fragment = args.area_name.strip().casefold()
-    matches = [item for item in areas.get("items") or [] if area_fragment in str(item.get("name") or "").casefold()]
+    matches = [item for item in areas if area_fragment in str(item.get("name") or "").casefold()]
     if len(matches) != 1:
         raise RuntimeError(f"Expected one Area matching {args.area_name!r}, received {len(matches)}")
-    datasets = response_data(
-        session.get(f"{base_url}/api/v1/projects/{project_id}/datasets", params={"limit": 500}, timeout=args.import_timeout)
+    datasets = list_paginated_items(
+        session,
+        f"{base_url}/api/v1/projects/{project_id}/datasets",
+        timeout=args.import_timeout,
     )
-    return project_id, str(matches[0]["id"]), list(datasets.get("items") or [])
+    return project_id, str(matches[0]["id"]), datasets
 
 
 def build_source_metadata(args: argparse.Namespace, snapshot: PreparedSnapshot) -> dict[str, Any]: