fix: retry transient official source failures
This commit is contained in:
@@ -12,6 +12,7 @@
|
||||
- Added an explicit operator provisioner for the official Digitaal Vlaanderen Mol municipality boundary (NIS `13025`) and the complete GRB GBG building population clipped to that boundary.
|
||||
- Added auditable source artifacts and a manifest with page count, checksums, exact WGS84 bounds, municipality area, feature totals and truncation state; incomplete pagination now fails closed.
|
||||
- Declared EPSG:4326 explicitly in both official GeoJSON artifacts so downstream QA does not downgrade known OGC provenance to an inferred CRS.
|
||||
- Added bounded exponential retries for safe official-source GET requests after a live transient GRB page failure; upload POST requests are never retried automatically.
|
||||
- Provisioning remains explicit and imports Project, Area and Dataset records through existing canonical API routes and DatasetService/VectorFeatureService persistence rather than writing directly to PostGIS.
|
||||
- Made the complete `Mol Municipality Workbench` the preferred fresh-session context and the official municipality boundary its lightweight default layer, ahead of historical Postel validation projects.
|
||||
- Replaced large coordinate arrays and spread-based bounds calculations with streaming, memoized GeoJSON bounds so municipality-scale vector layers remain safe in MapLibre.
|
||||
|
||||
@@ -85,6 +85,17 @@ def test_mol_provisioner_refuses_a_truncated_municipality_dataset() -> None:
|
||||
module.build_municipality_buildings(pages, boundary, max_features=1)
|
||||
|
||||
|
||||
def test_mol_source_session_retries_only_safe_get_requests() -> None:
|
||||
module = load_provisioner()
|
||||
|
||||
with module.build_source_session() as session:
|
||||
retry = session.get_adapter("https://").max_retries
|
||||
|
||||
assert retry.total == 5
|
||||
assert retry.allowed_methods == frozenset({"GET"})
|
||||
assert set(retry.status_forcelist) == {429, 500, 502, 503, 504}
|
||||
|
||||
|
||||
def test_large_vector_persistence_flushes_once_without_per_feature_refresh() -> None:
|
||||
class FakeSession:
|
||||
def __init__(self) -> None:
|
||||
|
||||
@@ -7408,6 +7408,9 @@ Open:
|
||||
explicit truncation flag. Pagination and identity checks fail closed.
|
||||
- Declared EPSG:4326 in both generated GeoJSON FeatureCollections so the
|
||||
importer records `crs_assumed=false` for this known official OGC source.
|
||||
- Added bounded retries for safe source GETs after the live refresh exposed a
|
||||
transient GRB HTTP 500 on page 66. Mutating GeoIntel API requests are not
|
||||
retried automatically.
|
||||
- Kept the provider boundary honest: provisioning is an operator action and
|
||||
imports through canonical Project, Area and Dataset HTTP routes. It does not
|
||||
enable the dormant live GRB provider or write directly to `vector_features`.
|
||||
|
||||
@@ -17,10 +17,12 @@ from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from pyproj import Transformer
|
||||
from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, mapping, shape
|
||||
from shapely.ops import transform, unary_union
|
||||
from shapely.validation import make_valid
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
|
||||
MUNICIPALITY_NAME = "Mol"
|
||||
@@ -40,6 +42,7 @@ DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||
DEFAULT_PAGE_LIMIT = 1000
|
||||
DEFAULT_MAX_FEATURES = 100000
|
||||
GEOJSON_CRS = {"type": "name", "properties": {"name": "EPSG:4326"}}
|
||||
RETRYABLE_SOURCE_STATUS_CODES = (429, 500, 502, 503, 504)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
@@ -116,6 +119,25 @@ def write_json(path: Path, payload: dict[str, Any], *, pretty: bool = False) ->
|
||||
)
|
||||
|
||||
|
||||
def build_source_session() -> requests.Session:
|
||||
retry = Retry(
|
||||
total=5,
|
||||
connect=5,
|
||||
read=5,
|
||||
status=5,
|
||||
backoff_factor=1.0,
|
||||
status_forcelist=RETRYABLE_SOURCE_STATUS_CODES,
|
||||
allowed_methods=frozenset({"GET"}),
|
||||
raise_on_status=True,
|
||||
)
|
||||
adapter = HTTPAdapter(max_retries=retry)
|
||||
session = requests.Session()
|
||||
session.headers.update({"User-Agent": "GeoIntel-Mol-Municipality-Operator/1.0"})
|
||||
session.mount("https://", adapter)
|
||||
session.mount("http://", adapter)
|
||||
return session
|
||||
|
||||
|
||||
def next_page_url(payload: dict[str, Any]) -> str | None:
|
||||
links = payload.get("links") or []
|
||||
for link in links:
|
||||
@@ -319,8 +341,7 @@ def prepare_artifacts(args: argparse.Namespace) -> tuple[Path, Path, dict[str, A
|
||||
if args.page_limit <= 0 or args.max_features <= 0:
|
||||
raise RuntimeError("--page-limit and --max-features must be positive integers")
|
||||
|
||||
with requests.Session() as session:
|
||||
session.headers.update({"User-Agent": "GeoIntel-Mol-Municipality-Operator/1.0"})
|
||||
with build_source_session() as session:
|
||||
boundary_feature, boundary, boundary_source_url = fetch_mol_boundary(session, args.request_timeout)
|
||||
buildings, building_summary = build_municipality_buildings(
|
||||
iter_grb_pages(
|
||||
|
||||
Reference in New Issue
Block a user