JensandClaude Opus 5
7351993fee
An ArcGIS layer without supportsPagination accepts resultOffset and ignores it, answering every page with the first one. The VHA profile reader advanced its offset by the page length and stopped at the announced count, so for a count that is a multiple of the page size it collected N copies of page one — and its completeness check, len(features) == candidate_count, passed. Four announced records became four stored records, two of them duplicates, filed under an official provenance. That is the substitution bounded acquisition exists to prevent, arriving through the front door. The reader now refuses a record it already collected. It fails rather than silently dropping the duplicate: a provider that cannot page is a provider whose count proves nothing, so a smaller-but-clean result would still be unverifiable. Its watercourse-name loop was worse — a bare `while True` that ended only when the provider stopped setting exceededTransferLimit, with names deduplicated by code so a stuck provider produced no visible change while the requests continued. It now refuses a repeated page body, and both loops have the page budget the sibling readers already had. Those siblings turned out to be fine. GRB and official vector already refuse a repeated page URL, bound the page count, and deduplicate on feature identity — but none of it had a test, so none of it was known to work. Exercised now, including the case where distinct URLs defeat the loop check and the budget is the only backstop. A duplicate across two genuinely different pages is kept once rather than failing, because a cursor over a changing table produces that legitimately. Also: _bash_path fell back to the raw path whenever wslpath failed, except on timeout, which propagated and reddened the suite when starting WSL took more than ten seconds under load. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
121 lines
4.1 KiB
Python
121 lines
4.1 KiB
Python
"""The paged readers' loop protections, exercised rather than assumed.
|
|
|
|
GRB and official vector both refuse a repeated page URL and bound the page
|
|
count, and GRB deduplicates on feature identity. None of that had a test, so
|
|
none of it was known to work — the same category as the redirect handler that
|
|
turned out to be dead code while looking like protection.
|
|
|
|
A provider that answers every page with a "next" link pointing back at itself
|
|
is not hypothetical: it is what a misconfigured cursor or a caching proxy in
|
|
front of an OGC endpoint produces.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
from shapely.geometry import Polygon
|
|
|
|
from app.core.config import Settings
|
|
from app.core.errors import AppError
|
|
from app.services.grb_acquisition_service import GrbAcquisitionService
|
|
from tests.test_sprint239_bounded_grb_acquisition import JsonResponse, polygon_feature
|
|
|
|
COLLECTION_ITEMS = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items"
|
|
SCOPE = Polygon([(5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)])
|
|
|
|
|
|
def _building(feature_id: str) -> dict:
|
|
return polygon_feature(
|
|
feature_id,
|
|
[(5.155, 51.185), (5.175, 51.185), (5.175, 51.195), (5.155, 51.195), (5.155, 51.185)],
|
|
)
|
|
|
|
|
|
def _fetch(opener):
|
|
return GrbAcquisitionService._fetch_features(
|
|
GrbAcquisitionService._product("buildings"),
|
|
SCOPE,
|
|
SCOPE.bounds,
|
|
"bounded_selection",
|
|
Settings(_env_file=None),
|
|
opener,
|
|
)
|
|
|
|
|
|
def test_a_next_link_pointing_at_itself_is_refused() -> None:
|
|
requests: list[str] = []
|
|
|
|
def opener(request, timeout): # noqa: ARG001
|
|
requests.append(request.full_url)
|
|
return JsonResponse(
|
|
{
|
|
"type": "FeatureCollection",
|
|
"features": [_building("GBG.1")],
|
|
"links": [{"rel": "next", "href": f"{COLLECTION_ITEMS}?cursor=stuck"}],
|
|
}
|
|
)
|
|
|
|
with pytest.raises(AppError) as exc_info:
|
|
_fetch(opener)
|
|
|
|
assert exc_info.value.code == "GRB_PROVIDER_PAGINATION_LOOP"
|
|
# Refused on the second sighting, not after exhausting the page budget.
|
|
assert len(requests) == 2
|
|
|
|
|
|
def test_an_endless_chain_of_fresh_pages_stops_at_the_page_limit() -> None:
|
|
"""Distinct URLs defeat the loop check, so the page budget is the backstop."""
|
|
|
|
settings = Settings(_env_file=None)
|
|
requests: list[str] = []
|
|
|
|
def opener(request, timeout): # noqa: ARG001
|
|
requests.append(request.full_url)
|
|
cursor = len(requests)
|
|
return JsonResponse(
|
|
{
|
|
"type": "FeatureCollection",
|
|
"features": [_building(f"GBG.{cursor}")],
|
|
"links": [{"rel": "next", "href": f"{COLLECTION_ITEMS}?cursor=p{cursor}"}],
|
|
}
|
|
)
|
|
|
|
with pytest.raises(AppError) as exc_info:
|
|
_fetch(opener)
|
|
|
|
assert exc_info.value.code == "GRB_SELECTION_TOO_LARGE"
|
|
assert exc_info.value.status_code == 422
|
|
assert len(requests) == settings.grb_max_pages
|
|
|
|
|
|
def test_a_repeated_feature_across_pages_is_counted_once() -> None:
|
|
"""Two pages, distinct URLs, overlapping content.
|
|
|
|
Unlike a repeated URL this is not necessarily provider misbehaviour — a
|
|
cursor over a changing table can hand back a record twice — so the reader
|
|
keeps it once rather than failing the acquisition.
|
|
"""
|
|
|
|
def opener(request, timeout): # noqa: ARG001
|
|
if "cursor=next" in request.full_url:
|
|
return JsonResponse(
|
|
{
|
|
"type": "FeatureCollection",
|
|
"features": [_building("GBG.1"), _building("GBG.2")],
|
|
"links": [],
|
|
}
|
|
)
|
|
return JsonResponse(
|
|
{
|
|
"type": "FeatureCollection",
|
|
"features": [_building("GBG.1")],
|
|
"links": [{"rel": "next", "href": f"{COLLECTION_ITEMS}?cursor=next"}],
|
|
}
|
|
)
|
|
|
|
features, transfer = _fetch(opener)
|
|
|
|
assert {feature["id"] for feature in features} == {"GBG:GBG.1", "GBG:GBG.2"}
|
|
assert transfer["candidate_feature_count"] == 3
|
|
assert transfer["feature_count"] == 2
|