Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
129 lines
4.6 KiB
Python
129 lines
4.6 KiB
Python
"""A provider that ignores ``resultOffset`` must not produce duplicated data.
|
|
|
|
ArcGIS layers without ``supportsPagination`` accept ``resultOffset`` and ignore
|
|
it, answering every page with the first one. The profile reader advanced its
|
|
offset by the page length and stopped when it reached the announced count, so
|
|
for a count that is a multiple of the page size it collected N copies of page
|
|
one, matched the expected total exactly, and stored that as an official
|
|
dataset. The watercourse-name reader had no bound at all: it looped for as long
|
|
as the provider kept setting ``exceededTransferLimit``.
|
|
|
|
The sibling reader for official vector products already refuses a repeated page
|
|
and deduplicates on feature identity. These tests hold this reader to the same
|
|
rule.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from typing import Any
|
|
from urllib.parse import parse_qs, urlparse
|
|
|
|
import pytest
|
|
|
|
from app.core.config import Settings, get_settings
|
|
from app.core.errors import AppError
|
|
from app.services.bathymetry_profile_acquisition_service import (
|
|
BathymetryProfileAcquisitionService,
|
|
)
|
|
|
|
BBOX = (4.30, 51.20, 4.32, 51.22)
|
|
|
|
|
|
def _settings(**overrides: Any) -> Settings:
|
|
base = get_settings()
|
|
return base.model_copy(update={"bathymetry_profiles_page_size": 2, **overrides})
|
|
|
|
|
|
class _Response:
|
|
def __init__(self, payload: dict[str, Any]) -> None:
|
|
self._body = json.dumps(payload).encode("utf-8")
|
|
|
|
def read(self, _limit: int | None = None) -> bytes:
|
|
return self._body
|
|
|
|
def __enter__(self) -> "_Response":
|
|
return self
|
|
|
|
def __exit__(self, *_exc: object) -> bool:
|
|
return False
|
|
|
|
|
|
def _profile(object_id: int) -> dict[str, Any]:
|
|
return {
|
|
"attributes": {"OBJECTID": object_id, "vhag": 7, "opg_diepte": 1.5},
|
|
"geometry": {"x": 4.31, "y": 51.21},
|
|
}
|
|
|
|
|
|
class StuckProvider:
|
|
"""Answers every page with the same records, as an unpaged layer does."""
|
|
|
|
def __init__(self, *, count: int, page: list[dict[str, Any]]) -> None:
|
|
self.count = count
|
|
self.page = page
|
|
self.requests: list[str] = []
|
|
|
|
def __call__(self, request: Any, **_kwargs: Any) -> _Response:
|
|
url = request.full_url if hasattr(request, "full_url") else str(request)
|
|
self.requests.append(url)
|
|
query = parse_qs(urlparse(url).query)
|
|
if query.get("returnCountOnly") == ["true"]:
|
|
return _Response({"count": self.count})
|
|
return _Response({"features": list(self.page), "exceededTransferLimit": True})
|
|
|
|
|
|
def test_a_provider_that_ignores_the_offset_is_refused_not_duplicated() -> None:
|
|
"""Four announced records, two per page, the same two every time.
|
|
|
|
Advancing by page length reaches the announced total after two pages, so the
|
|
completeness check passed while every record was stored twice.
|
|
"""
|
|
|
|
provider = StuckProvider(count=4, page=[_profile(1), _profile(2)])
|
|
|
|
with pytest.raises(AppError) as exc_info:
|
|
BathymetryProfileAcquisitionService._fetch_profiles(BBOX, _settings(), provider)
|
|
|
|
assert exc_info.value.code == "BATHYMETRY_PROVIDER_UNSTABLE_PAGINATION"
|
|
assert exc_info.value.status_code == 502
|
|
|
|
|
|
def test_honest_pagination_still_returns_every_record() -> None:
|
|
"""The guard must not reject a provider that pages correctly."""
|
|
|
|
pages = [[_profile(1), _profile(2)], [_profile(3), _profile(4)]]
|
|
|
|
def opener(request: Any, **_kwargs: Any) -> _Response:
|
|
url = request.full_url
|
|
query = parse_qs(urlparse(url).query)
|
|
if query.get("returnCountOnly") == ["true"]:
|
|
return _Response({"count": 4})
|
|
offset = int(query.get("resultOffset", ["0"])[0])
|
|
index = offset // 2
|
|
page = pages[index] if index < len(pages) else []
|
|
return _Response({"features": page})
|
|
|
|
features, provenance = BathymetryProfileAcquisitionService._fetch_profiles(
|
|
BBOX, _settings(), opener
|
|
)
|
|
|
|
assert [item["attributes"]["OBJECTID"] for item in features] == [1, 2, 3, 4]
|
|
assert provenance["candidate_count"] == 4
|
|
|
|
|
|
def test_watercourse_names_stop_instead_of_looping_forever() -> None:
|
|
"""``exceededTransferLimit`` forever is not a reason to request forever."""
|
|
|
|
provider = StuckProvider(
|
|
count=0,
|
|
page=[{"attributes": {"wlasvl.vhag": 7, "VHAG_TABEL.naam": "Schelde"}}],
|
|
)
|
|
|
|
with pytest.raises(AppError) as exc_info:
|
|
BathymetryProfileAcquisitionService._fetch_watercourse_names({7}, _settings(), provider)
|
|
|
|
assert exc_info.value.code == "BATHYMETRY_PROVIDER_UNSTABLE_PAGINATION"
|
|
# Bounded, and bounded early: it must not have hammered the provider first.
|
|
assert len(provider.requests) <= 3
|