diff --git a/backend/app/services/dhmv_acquisition_service.py b/backend/app/services/dhmv_acquisition_service.py index 0871d336..818c282c 100644 --- a/backend/app/services/dhmv_acquisition_service.py +++ b/backend/app/services/dhmv_acquisition_service.py @@ -3,6 +3,7 @@ from __future__ import annotations import hashlib import json import math +import time from dataclasses import dataclass from datetime import UTC, datetime from email.parser import BytesParser @@ -47,6 +48,9 @@ class DhmvAcquisitionService: ATTRIBUTION = "Bron: Digitaal Vlaanderen, Digitaal Hoogtemodel Vlaanderen II" LICENSE_NOTE = "Gebruik volgens het gebruiksrecht geografische webdiensten van Digitaal Vlaanderen." WCS_TILE_SIDE_M = 10_000.0 + WCS_REQUEST_INTERVAL_SECONDS = 2.0 + WCS_RETRY_DELAY_SECONDS = 4.0 + WCS_TRANSIENT_STATUS_CODES = frozenset({400, 429, 502, 503, 504}) DTM_CATALOG_URL = ( "https://www.vlaanderen.be/datavindplaats/catalogus/" "digitaal-hoogtemodel-vlaanderen-ii-dtm-raster-1-m" @@ -296,7 +300,19 @@ class DhmvAcquisitionService: content = response.read(max_bytes + 1) except AppError: raise - except (HTTPError, URLError, TimeoutError, OSError) as exc: + except HTTPError as exc: + preview = exc.read(300).decode("utf-8", errors="replace") + raise AppError( + code="DHMV_PROVIDER_UNAVAILABLE", + message="The official DHMV WCS could not complete the bounded request", + details={ + "reason": str(exc), + "provider_status_code": int(exc.code), + "response_preview": preview, + }, + status_code=502, + ) from exc + except (URLError, TimeoutError, OSError) as exc: raise AppError( code="DHMV_PROVIDER_UNAVAILABLE", message="The official DHMV WCS could not complete the bounded request", @@ -422,8 +438,17 @@ class DhmvAcquisitionService: coverage_hash = hashlib.sha256() content_types: list[str] = [] coverages: list[bytes] = [] - for request_url in request_urls: - raw_content, content_type = DhmvAcquisitionService._fetch(request_url, settings, opener) + for index, request_url in enumerate(request_urls): + if index > 0 and opener is None: + time.sleep(DhmvAcquisitionService.WCS_REQUEST_INTERVAL_SECONDS) + try: + raw_content, content_type = DhmvAcquisitionService._fetch(request_url, settings, opener) + except AppError as exc: + provider_status = (exc.details or {}).get("provider_status_code") + if opener is not None or provider_status not in DhmvAcquisitionService.WCS_TRANSIENT_STATUS_CODES: + raise + time.sleep(DhmvAcquisitionService.WCS_RETRY_DELAY_SECONDS) + raw_content, content_type = DhmvAcquisitionService._fetch(request_url, settings, opener) coverage_content = DhmvAcquisitionService._extract_geotiff(raw_content, content_type) raw_hash.update(len(raw_content).to_bytes(8, "big")) raw_hash.update(raw_content)