Rate-limit official DHMV tile acquisition

backend/app/services/dhmv_acquisition_service.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import hashlib
 import json
 import math
+import time
 from dataclasses import dataclass
 from datetime import UTC, datetime
 from email.parser import BytesParser
@@ -47,6 +48,9 @@ class DhmvAcquisitionService:
     ATTRIBUTION = "Bron: Digitaal Vlaanderen, Digitaal Hoogtemodel Vlaanderen II"
     LICENSE_NOTE = "Gebruik volgens het gebruiksrecht geografische webdiensten van Digitaal Vlaanderen."
     WCS_TILE_SIDE_M = 10_000.0
+    WCS_REQUEST_INTERVAL_SECONDS = 2.0
+    WCS_RETRY_DELAY_SECONDS = 4.0
+    WCS_TRANSIENT_STATUS_CODES = frozenset({400, 429, 502, 503, 504})
     DTM_CATALOG_URL = (
         "https://www.vlaanderen.be/datavindplaats/catalogus/"
         "digitaal-hoogtemodel-vlaanderen-ii-dtm-raster-1-m"
@@ -296,7 +300,19 @@ class DhmvAcquisitionService:
                 content = response.read(max_bytes + 1)
         except AppError:
             raise
-        except (HTTPError, URLError, TimeoutError, OSError) as exc:
+        except HTTPError as exc:
+            preview = exc.read(300).decode("utf-8", errors="replace")
+            raise AppError(
+                code="DHMV_PROVIDER_UNAVAILABLE",
+                message="The official DHMV WCS could not complete the bounded request",
+                details={
+                    "reason": str(exc),
+                    "provider_status_code": int(exc.code),
+                    "response_preview": preview,
+                },
+                status_code=502,
+            ) from exc
+        except (URLError, TimeoutError, OSError) as exc:
             raise AppError(
                 code="DHMV_PROVIDER_UNAVAILABLE",
                 message="The official DHMV WCS could not complete the bounded request",
@@ -422,8 +438,17 @@ class DhmvAcquisitionService:
         coverage_hash = hashlib.sha256()
         content_types: list[str] = []
         coverages: list[bytes] = []
-        for request_url in request_urls:
+        for index, request_url in enumerate(request_urls):
-            raw_content, content_type = DhmvAcquisitionService._fetch(request_url, settings, opener)
+            if index > 0 and opener is None:
+                time.sleep(DhmvAcquisitionService.WCS_REQUEST_INTERVAL_SECONDS)
+            try:
+                raw_content, content_type = DhmvAcquisitionService._fetch(request_url, settings, opener)
+            except AppError as exc:
+                provider_status = (exc.details or {}).get("provider_status_code")
+                if opener is not None or provider_status not in DhmvAcquisitionService.WCS_TRANSIENT_STATUS_CODES:
+                    raise
+                time.sleep(DhmvAcquisitionService.WCS_RETRY_DELAY_SECONDS)
+                raw_content, content_type = DhmvAcquisitionService._fetch(request_url, settings, opener)
             coverage_content = DhmvAcquisitionService._extract_geotiff(raw_content, content_type)
             raw_hash.update(len(raw_content).to_bytes(8, "big"))
             raw_hash.update(raw_content)