fix: retry transient official source failures
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-14 05:47:06 +02:00
parent 480634c301
commit e8a35b96ad
4 changed files with 38 additions and 2 deletions
+1
View File
@@ -12,6 +12,7 @@
- Added an explicit operator provisioner for the official Digitaal Vlaanderen Mol municipality boundary (NIS `13025`) and the complete GRB GBG building population clipped to that boundary.
- Added auditable source artifacts and a manifest with page count, checksums, exact WGS84 bounds, municipality area, feature totals and truncation state; incomplete pagination now fails closed.
- Declared EPSG:4326 explicitly in both official GeoJSON artifacts so downstream QA does not downgrade known OGC provenance to an inferred CRS.
- Added bounded exponential retries for safe official-source GET requests after a live transient GRB page failure; upload POST requests are never retried automatically.
- Provisioning remains explicit and imports Project, Area and Dataset records through existing canonical API routes and DatasetService/VectorFeatureService persistence rather than writing directly to PostGIS.
- Made the complete `Mol Municipality Workbench` the preferred fresh-session context and the official municipality boundary its lightweight default layer, ahead of historical Postel validation projects.
- Replaced large coordinate arrays and spread-based bounds calculations with streaming, memoized GeoJSON bounds so municipality-scale vector layers remain safe in MapLibre.
@@ -85,6 +85,17 @@ def test_mol_provisioner_refuses_a_truncated_municipality_dataset() -> None:
module.build_municipality_buildings(pages, boundary, max_features=1)
def test_mol_source_session_retries_only_safe_get_requests() -> None:
module = load_provisioner()
with module.build_source_session() as session:
retry = session.get_adapter("https://").max_retries
assert retry.total == 5
assert retry.allowed_methods == frozenset({"GET"})
assert set(retry.status_forcelist) == {429, 500, 502, 503, 504}
def test_large_vector_persistence_flushes_once_without_per_feature_refresh() -> None:
class FakeSession:
def __init__(self) -> None:
+3
View File
@@ -7408,6 +7408,9 @@ Open:
explicit truncation flag. Pagination and identity checks fail closed.
- Declared EPSG:4326 in both generated GeoJSON FeatureCollections so the
importer records `crs_assumed=false` for this known official OGC source.
- Added bounded retries for safe source GETs after the live refresh exposed a
transient GRB HTTP 500 on page 66. Mutating GeoIntel API requests are not
retried automatically.
- Kept the provider boundary honest: provisioning is an operator action and
imports through canonical Project, Area and Dataset HTTP routes. It does not
enable the dormant live GRB provider or write directly to `vector_features`.
@@ -17,10 +17,12 @@ from pathlib import Path
from typing import Any, Iterable
import requests
from requests.adapters import HTTPAdapter
from pyproj import Transformer
from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, mapping, shape
from shapely.ops import transform, unary_union
from shapely.validation import make_valid
from urllib3.util.retry import Retry
MUNICIPALITY_NAME = "Mol"
@@ -40,6 +42,7 @@ DEFAULT_API_URL = "http://127.0.0.1:8000"
DEFAULT_PAGE_LIMIT = 1000
DEFAULT_MAX_FEATURES = 100000
GEOJSON_CRS = {"type": "name", "properties": {"name": "EPSG:4326"}}
RETRYABLE_SOURCE_STATUS_CODES = (429, 500, 502, 503, 504)
def parse_args() -> argparse.Namespace:
@@ -116,6 +119,25 @@ def write_json(path: Path, payload: dict[str, Any], *, pretty: bool = False) ->
)
def build_source_session() -> requests.Session:
retry = Retry(
total=5,
connect=5,
read=5,
status=5,
backoff_factor=1.0,
status_forcelist=RETRYABLE_SOURCE_STATUS_CODES,
allowed_methods=frozenset({"GET"}),
raise_on_status=True,
)
adapter = HTTPAdapter(max_retries=retry)
session = requests.Session()
session.headers.update({"User-Agent": "GeoIntel-Mol-Municipality-Operator/1.0"})
session.mount("https://", adapter)
session.mount("http://", adapter)
return session
def next_page_url(payload: dict[str, Any]) -> str | None:
links = payload.get("links") or []
for link in links:
@@ -319,8 +341,7 @@ def prepare_artifacts(args: argparse.Namespace) -> tuple[Path, Path, dict[str, A
if args.page_limit <= 0 or args.max_features <= 0:
raise RuntimeError("--page-limit and --max-features must be positive integers")
with requests.Session() as session:
session.headers.update({"User-Agent": "GeoIntel-Mol-Municipality-Operator/1.0"})
with build_source_session() as session:
boundary_feature, boundary, boundary_source_url = fetch_mol_boundary(session, args.request_timeout)
buildings, building_summary = build_municipality_buildings(
iter_grb_pages(