fix: retry transient official source failures
This commit is contained in:
@@ -12,6 +12,7 @@
|
|||||||
- Added an explicit operator provisioner for the official Digitaal Vlaanderen Mol municipality boundary (NIS `13025`) and the complete GRB GBG building population clipped to that boundary.
|
- Added an explicit operator provisioner for the official Digitaal Vlaanderen Mol municipality boundary (NIS `13025`) and the complete GRB GBG building population clipped to that boundary.
|
||||||
- Added auditable source artifacts and a manifest with page count, checksums, exact WGS84 bounds, municipality area, feature totals and truncation state; incomplete pagination now fails closed.
|
- Added auditable source artifacts and a manifest with page count, checksums, exact WGS84 bounds, municipality area, feature totals and truncation state; incomplete pagination now fails closed.
|
||||||
- Declared EPSG:4326 explicitly in both official GeoJSON artifacts so downstream QA does not downgrade known OGC provenance to an inferred CRS.
|
- Declared EPSG:4326 explicitly in both official GeoJSON artifacts so downstream QA does not downgrade known OGC provenance to an inferred CRS.
|
||||||
|
- Added bounded exponential retries for safe official-source GET requests after a live transient GRB page failure; upload POST requests are never retried automatically.
|
||||||
- Provisioning remains explicit and imports Project, Area and Dataset records through existing canonical API routes and DatasetService/VectorFeatureService persistence rather than writing directly to PostGIS.
|
- Provisioning remains explicit and imports Project, Area and Dataset records through existing canonical API routes and DatasetService/VectorFeatureService persistence rather than writing directly to PostGIS.
|
||||||
- Made the complete `Mol Municipality Workbench` the preferred fresh-session context and the official municipality boundary its lightweight default layer, ahead of historical Postel validation projects.
|
- Made the complete `Mol Municipality Workbench` the preferred fresh-session context and the official municipality boundary its lightweight default layer, ahead of historical Postel validation projects.
|
||||||
- Replaced large coordinate arrays and spread-based bounds calculations with streaming, memoized GeoJSON bounds so municipality-scale vector layers remain safe in MapLibre.
|
- Replaced large coordinate arrays and spread-based bounds calculations with streaming, memoized GeoJSON bounds so municipality-scale vector layers remain safe in MapLibre.
|
||||||
|
|||||||
@@ -85,6 +85,17 @@ def test_mol_provisioner_refuses_a_truncated_municipality_dataset() -> None:
|
|||||||
module.build_municipality_buildings(pages, boundary, max_features=1)
|
module.build_municipality_buildings(pages, boundary, max_features=1)
|
||||||
|
|
||||||
|
|
||||||
|
def test_mol_source_session_retries_only_safe_get_requests() -> None:
|
||||||
|
module = load_provisioner()
|
||||||
|
|
||||||
|
with module.build_source_session() as session:
|
||||||
|
retry = session.get_adapter("https://").max_retries
|
||||||
|
|
||||||
|
assert retry.total == 5
|
||||||
|
assert retry.allowed_methods == frozenset({"GET"})
|
||||||
|
assert set(retry.status_forcelist) == {429, 500, 502, 503, 504}
|
||||||
|
|
||||||
|
|
||||||
def test_large_vector_persistence_flushes_once_without_per_feature_refresh() -> None:
|
def test_large_vector_persistence_flushes_once_without_per_feature_refresh() -> None:
|
||||||
class FakeSession:
|
class FakeSession:
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
|
|||||||
@@ -7408,6 +7408,9 @@ Open:
|
|||||||
explicit truncation flag. Pagination and identity checks fail closed.
|
explicit truncation flag. Pagination and identity checks fail closed.
|
||||||
- Declared EPSG:4326 in both generated GeoJSON FeatureCollections so the
|
- Declared EPSG:4326 in both generated GeoJSON FeatureCollections so the
|
||||||
importer records `crs_assumed=false` for this known official OGC source.
|
importer records `crs_assumed=false` for this known official OGC source.
|
||||||
|
- Added bounded retries for safe source GETs after the live refresh exposed a
|
||||||
|
transient GRB HTTP 500 on page 66. Mutating GeoIntel API requests are not
|
||||||
|
retried automatically.
|
||||||
- Kept the provider boundary honest: provisioning is an operator action and
|
- Kept the provider boundary honest: provisioning is an operator action and
|
||||||
imports through canonical Project, Area and Dataset HTTP routes. It does not
|
imports through canonical Project, Area and Dataset HTTP routes. It does not
|
||||||
enable the dormant live GRB provider or write directly to `vector_features`.
|
enable the dormant live GRB provider or write directly to `vector_features`.
|
||||||
|
|||||||
@@ -17,10 +17,12 @@ from pathlib import Path
|
|||||||
from typing import Any, Iterable
|
from typing import Any, Iterable
|
||||||
|
|
||||||
import requests
|
import requests
|
||||||
|
from requests.adapters import HTTPAdapter
|
||||||
from pyproj import Transformer
|
from pyproj import Transformer
|
||||||
from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, mapping, shape
|
from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, mapping, shape
|
||||||
from shapely.ops import transform, unary_union
|
from shapely.ops import transform, unary_union
|
||||||
from shapely.validation import make_valid
|
from shapely.validation import make_valid
|
||||||
|
from urllib3.util.retry import Retry
|
||||||
|
|
||||||
|
|
||||||
MUNICIPALITY_NAME = "Mol"
|
MUNICIPALITY_NAME = "Mol"
|
||||||
@@ -40,6 +42,7 @@ DEFAULT_API_URL = "http://127.0.0.1:8000"
|
|||||||
DEFAULT_PAGE_LIMIT = 1000
|
DEFAULT_PAGE_LIMIT = 1000
|
||||||
DEFAULT_MAX_FEATURES = 100000
|
DEFAULT_MAX_FEATURES = 100000
|
||||||
GEOJSON_CRS = {"type": "name", "properties": {"name": "EPSG:4326"}}
|
GEOJSON_CRS = {"type": "name", "properties": {"name": "EPSG:4326"}}
|
||||||
|
RETRYABLE_SOURCE_STATUS_CODES = (429, 500, 502, 503, 504)
|
||||||
|
|
||||||
|
|
||||||
def parse_args() -> argparse.Namespace:
|
def parse_args() -> argparse.Namespace:
|
||||||
@@ -116,6 +119,25 @@ def write_json(path: Path, payload: dict[str, Any], *, pretty: bool = False) ->
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def build_source_session() -> requests.Session:
|
||||||
|
retry = Retry(
|
||||||
|
total=5,
|
||||||
|
connect=5,
|
||||||
|
read=5,
|
||||||
|
status=5,
|
||||||
|
backoff_factor=1.0,
|
||||||
|
status_forcelist=RETRYABLE_SOURCE_STATUS_CODES,
|
||||||
|
allowed_methods=frozenset({"GET"}),
|
||||||
|
raise_on_status=True,
|
||||||
|
)
|
||||||
|
adapter = HTTPAdapter(max_retries=retry)
|
||||||
|
session = requests.Session()
|
||||||
|
session.headers.update({"User-Agent": "GeoIntel-Mol-Municipality-Operator/1.0"})
|
||||||
|
session.mount("https://", adapter)
|
||||||
|
session.mount("http://", adapter)
|
||||||
|
return session
|
||||||
|
|
||||||
|
|
||||||
def next_page_url(payload: dict[str, Any]) -> str | None:
|
def next_page_url(payload: dict[str, Any]) -> str | None:
|
||||||
links = payload.get("links") or []
|
links = payload.get("links") or []
|
||||||
for link in links:
|
for link in links:
|
||||||
@@ -319,8 +341,7 @@ def prepare_artifacts(args: argparse.Namespace) -> tuple[Path, Path, dict[str, A
|
|||||||
if args.page_limit <= 0 or args.max_features <= 0:
|
if args.page_limit <= 0 or args.max_features <= 0:
|
||||||
raise RuntimeError("--page-limit and --max-features must be positive integers")
|
raise RuntimeError("--page-limit and --max-features must be positive integers")
|
||||||
|
|
||||||
with requests.Session() as session:
|
with build_source_session() as session:
|
||||||
session.headers.update({"User-Agent": "GeoIntel-Mol-Municipality-Operator/1.0"})
|
|
||||||
boundary_feature, boundary, boundary_source_url = fetch_mol_boundary(session, args.request_timeout)
|
boundary_feature, boundary, boundary_source_url = fetch_mol_boundary(session, args.request_timeout)
|
||||||
buildings, building_summary = build_municipality_buildings(
|
buildings, building_summary = build_municipality_buildings(
|
||||||
iter_grb_pages(
|
iter_grb_pages(
|
||||||
|
|||||||
Reference in New Issue
Block a user