From 791548e332c2f2bf51d29951a665e3ed5cd1bd62 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 16 Jul 2026 04:46:13 +0200 Subject: [PATCH] Retry interrupted thematic WCS tiles --- .../thematic_raster_acquisition_service.py | 59 +++++++++++-------- .../tests/test_sprint213_thematic_rasters.py | 52 ++++++++++++++++ 2 files changed, 87 insertions(+), 24 deletions(-) diff --git a/backend/app/services/thematic_raster_acquisition_service.py b/backend/app/services/thematic_raster_acquisition_service.py index 2d204083..ebcffcd2 100644 --- a/backend/app/services/thematic_raster_acquisition_service.py +++ b/backend/app/services/thematic_raster_acquisition_service.py @@ -6,6 +6,7 @@ import math import time from dataclasses import dataclass from datetime import UTC, datetime +from http.client import HTTPException from pathlib import Path from typing import Any, Callable from urllib.error import HTTPError, URLError @@ -55,6 +56,8 @@ class ThematicRasterAcquisitionService: NODATA = -9999.0 WCS_TILE_SIDE_M = 10_000.0 WCS_REQUEST_INTERVAL_SECONDS = 0.5 + WCS_FETCH_ATTEMPTS = 3 + WCS_RETRY_DELAY_SECONDS = 1.0 ATTRIBUTION = "Bron: Departement Omgeving, MercatorNet" LICENSE_NOTE = "Publieke GDI-Vlaanderen bron; bronvermelding en productspecifieke gebruiksvoorwaarden blijven van toepassing." @@ -305,30 +308,38 @@ class ThematicRasterAcquisitionService: def _fetch(request_url: str, settings: Settings, opener: Callable[..., Any] | None = None) -> tuple[bytes, str]: request = Request(request_url, headers={"Accept": "image/tiff,*/*", "User-Agent": "GeoIntel/0.1 bounded-thematic-raster"}) max_bytes = settings.thematic_raster_max_response_mb * 1024 * 1024 - try: - with (opener or urlopen)(request, timeout=settings.thematic_raster_timeout_seconds) as response: - content_type = str(response.headers.get("Content-Type", "")) - content_length = response.headers.get("Content-Length") - if content_length and int(content_length) > max_bytes: - raise AppError(code="THEMATIC_RASTER_RESPONSE_TOO_LARGE", message="Official raster response exceeds the configured size limit", status_code=502) - content = response.read(max_bytes + 1) - except AppError: - raise - except HTTPError as exc: - preview = exc.read(300).decode("utf-8", errors="replace") - raise AppError( - code="THEMATIC_RASTER_PROVIDER_UNAVAILABLE", - message="The official MercatorNet WCS could not complete the bounded request", - details={"reason": str(exc), "provider_status_code": int(exc.code), "response_preview": preview}, - status_code=502, - ) from exc - except (URLError, TimeoutError, OSError) as exc: - raise AppError( - code="THEMATIC_RASTER_PROVIDER_UNAVAILABLE", - message="The official MercatorNet WCS could not complete the bounded request", - details={"reason": str(exc)}, - status_code=502, - ) from exc + for attempt in range(1, ThematicRasterAcquisitionService.WCS_FETCH_ATTEMPTS + 1): + try: + with (opener or urlopen)(request, timeout=settings.thematic_raster_timeout_seconds) as response: + content_type = str(response.headers.get("Content-Type", "")) + content_length = response.headers.get("Content-Length") + if content_length and int(content_length) > max_bytes: + raise AppError(code="THEMATIC_RASTER_RESPONSE_TOO_LARGE", message="Official raster response exceeds the configured size limit", status_code=502) + content = response.read(max_bytes + 1) + break + except AppError: + raise + except HTTPError as exc: + preview = exc.read(300).decode("utf-8", errors="replace") + if attempt < ThematicRasterAcquisitionService.WCS_FETCH_ATTEMPTS and int(exc.code) in {429, 500, 502, 503, 504}: + time.sleep(ThematicRasterAcquisitionService.WCS_RETRY_DELAY_SECONDS * attempt) + continue + raise AppError( + code="THEMATIC_RASTER_PROVIDER_UNAVAILABLE", + message="The official MercatorNet WCS could not complete the bounded request", + details={"reason": str(exc), "provider_status_code": int(exc.code), "response_preview": preview, "attempts": attempt}, + status_code=502, + ) from exc + except (URLError, TimeoutError, OSError, HTTPException) as exc: + if attempt < ThematicRasterAcquisitionService.WCS_FETCH_ATTEMPTS: + time.sleep(ThematicRasterAcquisitionService.WCS_RETRY_DELAY_SECONDS * attempt) + continue + raise AppError( + code="THEMATIC_RASTER_PROVIDER_UNAVAILABLE", + message="The official MercatorNet WCS could not complete the bounded request", + details={"reason": str(exc), "attempts": attempt}, + status_code=502, + ) from exc if len(content) > max_bytes: raise AppError(code="THEMATIC_RASTER_RESPONSE_TOO_LARGE", message="Official raster response exceeds the configured size limit", status_code=502) if not content.startswith((b"II*\x00", b"MM\x00*")): diff --git a/backend/tests/test_sprint213_thematic_rasters.py b/backend/tests/test_sprint213_thematic_rasters.py index 83b7f400..36317588 100644 --- a/backend/tests/test_sprint213_thematic_rasters.py +++ b/backend/tests/test_sprint213_thematic_rasters.py @@ -1,5 +1,6 @@ from __future__ import annotations +from http.client import IncompleteRead from pathlib import Path from types import SimpleNamespace from uuid import uuid4 @@ -89,6 +90,11 @@ class FakeResponse: return self.content[:limit] +class IncompleteResponse(FakeResponse): + def read(self, limit: int): + raise IncompleteRead(self.content[:limit]) + + def payload(product_key: str = "space_occupation_2025", *, side_m: float = 1000.0) -> ThematicRasterAcquireRequest: transformer = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True) min_x, min_y = transformer.transform(200_000, 210_000) @@ -171,6 +177,52 @@ def test_complete_kempen_scope_fits_the_tiled_thematic_guardrails() -> None: assert exc_info.value.code == "THEMATIC_RASTER_SELECTION_TOO_LARGE" +def test_wcs_fetch_retries_an_incomplete_tile_without_accepting_partial_bytes(monkeypatch) -> None: + content = b"II*\x00complete-geotiff" + responses = [IncompleteResponse(content), FakeResponse(content)] + attempts = 0 + + def opener(*_args, **_kwargs): + nonlocal attempts + response = responses[attempts] + attempts += 1 + return response + + monkeypatch.setattr("app.services.thematic_raster_acquisition_service.time.sleep", lambda _seconds: None) + + result, content_type = ThematicRasterAcquisitionService._fetch( + "https://example.invalid/wcs", + Settings(_env_file=None), + opener, + ) + + assert attempts == 2 + assert result == content + assert content_type == "image/tiff" + + +def test_wcs_fetch_fails_closed_after_bounded_incomplete_tile_retries(monkeypatch) -> None: + attempts = 0 + + def opener(*_args, **_kwargs): + nonlocal attempts + attempts += 1 + return IncompleteResponse(b"II*\x00partial") + + monkeypatch.setattr("app.services.thematic_raster_acquisition_service.time.sleep", lambda _seconds: None) + + with pytest.raises(AppError) as exc_info: + ThematicRasterAcquisitionService._fetch( + "https://example.invalid/wcs", + Settings(_env_file=None), + opener, + ) + + assert attempts == ThematicRasterAcquisitionService.WCS_FETCH_ATTEMPTS + assert exc_info.value.code == "THEMATIC_RASTER_PROVIDER_UNAVAILABLE" + assert exc_info.value.details["attempts"] == 3 + + def test_binary_and_normalized_products_fail_closed_on_invalid_values() -> None: binary = ThematicRasterAcquisitionService._product("space_occupation_2025") score = ThematicRasterAcquisitionService._product("service_level_2022")