diff --git a/CHANGELOG.md b/CHANGELOG.md index e1c4e9e3..21e6ccea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Added an explicit operator provisioner for the official Digitaal Vlaanderen Mol municipality boundary (NIS `13025`) and the complete GRB GBG building population clipped to that boundary. - Added auditable source artifacts and a manifest with page count, checksums, exact WGS84 bounds, municipality area, feature totals and truncation state; incomplete pagination now fails closed. - Declared EPSG:4326 explicitly in both official GeoJSON artifacts so downstream QA does not downgrade known OGC provenance to an inferred CRS. +- Added bounded exponential retries for safe official-source GET requests after a live transient GRB page failure; upload POST requests are never retried automatically. - Provisioning remains explicit and imports Project, Area and Dataset records through existing canonical API routes and DatasetService/VectorFeatureService persistence rather than writing directly to PostGIS. - Made the complete `Mol Municipality Workbench` the preferred fresh-session context and the official municipality boundary its lightweight default layer, ahead of historical Postel validation projects. - Replaced large coordinate arrays and spread-based bounds calculations with streaming, memoized GeoJSON bounds so municipality-scale vector layers remain safe in MapLibre. diff --git a/backend/tests/test_sprint181_mol_municipality_workspace.py b/backend/tests/test_sprint181_mol_municipality_workspace.py index 82e3d01f..1220be59 100644 --- a/backend/tests/test_sprint181_mol_municipality_workspace.py +++ b/backend/tests/test_sprint181_mol_municipality_workspace.py @@ -85,6 +85,17 @@ def test_mol_provisioner_refuses_a_truncated_municipality_dataset() -> None: module.build_municipality_buildings(pages, boundary, max_features=1) +def test_mol_source_session_retries_only_safe_get_requests() -> None: + module = load_provisioner() + + with module.build_source_session() as session: + retry = session.get_adapter("https://").max_retries + + assert retry.total == 5 + assert retry.allowed_methods == frozenset({"GET"}) + assert set(retry.status_forcelist) == {429, 500, 502, 503, 504} + + def test_large_vector_persistence_flushes_once_without_per_feature_refresh() -> None: class FakeSession: def __init__(self) -> None: diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 5046b8ce..1975336a 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -7408,6 +7408,9 @@ Open: explicit truncation flag. Pagination and identity checks fail closed. - Declared EPSG:4326 in both generated GeoJSON FeatureCollections so the importer records `crs_assumed=false` for this known official OGC source. +- Added bounded retries for safe source GETs after the live refresh exposed a + transient GRB HTTP 500 on page 66. Mutating GeoIntel API requests are not + retried automatically. - Kept the provider boundary honest: provisioning is an operator action and imports through canonical Project, Area and Dataset HTTP routes. It does not enable the dormant live GRB provider or write directly to `vector_features`. diff --git a/scripts/provision_mol_municipality_workspace.py b/scripts/provision_mol_municipality_workspace.py index bb2d0be9..d365890c 100644 --- a/scripts/provision_mol_municipality_workspace.py +++ b/scripts/provision_mol_municipality_workspace.py @@ -17,10 +17,12 @@ from pathlib import Path from typing import Any, Iterable import requests +from requests.adapters import HTTPAdapter from pyproj import Transformer from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, mapping, shape from shapely.ops import transform, unary_union from shapely.validation import make_valid +from urllib3.util.retry import Retry MUNICIPALITY_NAME = "Mol" @@ -40,6 +42,7 @@ DEFAULT_API_URL = "http://127.0.0.1:8000" DEFAULT_PAGE_LIMIT = 1000 DEFAULT_MAX_FEATURES = 100000 GEOJSON_CRS = {"type": "name", "properties": {"name": "EPSG:4326"}} +RETRYABLE_SOURCE_STATUS_CODES = (429, 500, 502, 503, 504) def parse_args() -> argparse.Namespace: @@ -116,6 +119,25 @@ def write_json(path: Path, payload: dict[str, Any], *, pretty: bool = False) -> ) +def build_source_session() -> requests.Session: + retry = Retry( + total=5, + connect=5, + read=5, + status=5, + backoff_factor=1.0, + status_forcelist=RETRYABLE_SOURCE_STATUS_CODES, + allowed_methods=frozenset({"GET"}), + raise_on_status=True, + ) + adapter = HTTPAdapter(max_retries=retry) + session = requests.Session() + session.headers.update({"User-Agent": "GeoIntel-Mol-Municipality-Operator/1.0"}) + session.mount("https://", adapter) + session.mount("http://", adapter) + return session + + def next_page_url(payload: dict[str, Any]) -> str | None: links = payload.get("links") or [] for link in links: @@ -319,8 +341,7 @@ def prepare_artifacts(args: argparse.Namespace) -> tuple[Path, Path, dict[str, A if args.page_limit <= 0 or args.max_features <= 0: raise RuntimeError("--page-limit and --max-features must be positive integers") - with requests.Session() as session: - session.headers.update({"User-Agent": "GeoIntel-Mol-Municipality-Operator/1.0"}) + with build_source_session() as session: boundary_feature, boundary, boundary_source_url = fetch_mol_boundary(session, args.request_timeout) buildings, building_summary = build_municipality_buildings( iter_grb_pages(