Initial public release

backend/tests/test_bathymetry_pagination_integrity.py

@@ -0,0 +1,128 @@
+"""A provider that ignores ``resultOffset`` must not produce duplicated data.
+
+ArcGIS layers without ``supportsPagination`` accept ``resultOffset`` and ignore
+it, answering every page with the first one. The profile reader advanced its
+offset by the page length and stopped when it reached the announced count, so
+for a count that is a multiple of the page size it collected N copies of page
+one, matched the expected total exactly, and stored that as an official
+dataset. The watercourse-name reader had no bound at all: it looped for as long
+as the provider kept setting ``exceededTransferLimit``.
+
+The sibling reader for official vector products already refuses a repeated page
+and deduplicates on feature identity. These tests hold this reader to the same
+rule.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+from urllib.parse import parse_qs, urlparse
+
+import pytest
+
+from app.core.config import Settings, get_settings
+from app.core.errors import AppError
+from app.services.bathymetry_profile_acquisition_service import (
+    BathymetryProfileAcquisitionService,
+)
+
+BBOX = (4.30, 51.20, 4.32, 51.22)
+
+
+def _settings(**overrides: Any) -> Settings:
+    base = get_settings()
+    return base.model_copy(update={"bathymetry_profiles_page_size": 2, **overrides})
+
+
+class _Response:
+    def __init__(self, payload: dict[str, Any]) -> None:
+        self._body = json.dumps(payload).encode("utf-8")
+
+    def read(self, _limit: int | None = None) -> bytes:
+        return self._body
+
+    def __enter__(self) -> "_Response":
+        return self
+
+    def __exit__(self, *_exc: object) -> bool:
+        return False
+
+
+def _profile(object_id: int) -> dict[str, Any]:
+    return {
+        "attributes": {"OBJECTID": object_id, "vhag": 7, "opg_diepte": 1.5},
+        "geometry": {"x": 4.31, "y": 51.21},
+    }
+
+
+class StuckProvider:
+    """Answers every page with the same records, as an unpaged layer does."""
+
+    def __init__(self, *, count: int, page: list[dict[str, Any]]) -> None:
+        self.count = count
+        self.page = page
+        self.requests: list[str] = []
+
+    def __call__(self, request: Any, **_kwargs: Any) -> _Response:
+        url = request.full_url if hasattr(request, "full_url") else str(request)
+        self.requests.append(url)
+        query = parse_qs(urlparse(url).query)
+        if query.get("returnCountOnly") == ["true"]:
+            return _Response({"count": self.count})
+        return _Response({"features": list(self.page), "exceededTransferLimit": True})
+
+
+def test_a_provider_that_ignores_the_offset_is_refused_not_duplicated() -> None:
+    """Four announced records, two per page, the same two every time.
+
+    Advancing by page length reaches the announced total after two pages, so the
+    completeness check passed while every record was stored twice.
+    """
+
+    provider = StuckProvider(count=4, page=[_profile(1), _profile(2)])
+
+    with pytest.raises(AppError) as exc_info:
+        BathymetryProfileAcquisitionService._fetch_profiles(BBOX, _settings(), provider)
+
+    assert exc_info.value.code == "BATHYMETRY_PROVIDER_UNSTABLE_PAGINATION"
+    assert exc_info.value.status_code == 502
+
+
+def test_honest_pagination_still_returns_every_record() -> None:
+    """The guard must not reject a provider that pages correctly."""
+
+    pages = [[_profile(1), _profile(2)], [_profile(3), _profile(4)]]
+
+    def opener(request: Any, **_kwargs: Any) -> _Response:
+        url = request.full_url
+        query = parse_qs(urlparse(url).query)
+        if query.get("returnCountOnly") == ["true"]:
+            return _Response({"count": 4})
+        offset = int(query.get("resultOffset", ["0"])[0])
+        index = offset // 2
+        page = pages[index] if index < len(pages) else []
+        return _Response({"features": page})
+
+    features, provenance = BathymetryProfileAcquisitionService._fetch_profiles(
+        BBOX, _settings(), opener
+    )
+
+    assert [item["attributes"]["OBJECTID"] for item in features] == [1, 2, 3, 4]
+    assert provenance["candidate_count"] == 4
+
+
+def test_watercourse_names_stop_instead_of_looping_forever() -> None:
+    """``exceededTransferLimit`` forever is not a reason to request forever."""
+
+    provider = StuckProvider(
+        count=0,
+        page=[{"attributes": {"wlasvl.vhag": 7, "VHAG_TABEL.naam": "Schelde"}}],
+    )
+
+    with pytest.raises(AppError) as exc_info:
+        BathymetryProfileAcquisitionService._fetch_watercourse_names({7}, _settings(), provider)
+
+    assert exc_info.value.code == "BATHYMETRY_PROVIDER_UNSTABLE_PAGINATION"
+    # Bounded, and bounded early: it must not have hammered the provider first.
+    assert len(provider.requests) <= 3