Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
336 lines
16 KiB
Python
336 lines
16 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from datetime import UTC, datetime
|
|
from typing import Any, Callable
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
|
from urllib.request import Request
|
|
from uuid import UUID
|
|
|
|
from app.core.config import Settings, get_settings
|
|
from app.core.errors import AppError
|
|
from app.services.outbound_request_guard import guarded_opener
|
|
from app.models import Dataset
|
|
from app.schemas.bathymetry import MdkBathymetryAcquireRequest, MdkBathymetryAcquisitionResult
|
|
from app.services.dataset_service import DatasetService
|
|
from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService
|
|
|
|
|
|
class MdkBathymetryAcquisitionService:
|
|
"""Bounded, fail-closed GetCoverage acquisition for the MDK Belgian North Sea depth model.
|
|
|
|
Acquisition only runs when:
|
|
|
|
- the operator explicitly enabled acquisition and configured a coverage id,
|
|
- the live strict-TLS readiness probe reports ``reachable``,
|
|
- the configured coverage id is advertised by the live capabilities document,
|
|
- the requested EPSG:4326 bbox stays within the configured size bound.
|
|
|
|
No depth values are ever synthesized, no insecure TLS fallback exists and the
|
|
LAT vertical reference is persisted with every artifact so it can never be
|
|
silently compared with TAW or mDNG data.
|
|
"""
|
|
|
|
PROVIDER = "mdk_bcp_bathymetry"
|
|
VERTICAL_REFERENCE = "LAT"
|
|
NATIVE_RESOLUTION_M = 20.0
|
|
MAX_PIXELS_PER_SIDE = 4096
|
|
LIMITATION = (
|
|
"Dieptewaarden zijn LAT-gerefereerd en gelden voor de bemonsterde survey-periode van het officiële "
|
|
"MDK-model. LAT mag nooit zonder gedocumenteerde datumtransformatie met TAW- of mDNG-gegevens worden "
|
|
"vergeleken; watervolume blijft zonder compatibel wateroppervlak niet ondersteund."
|
|
)
|
|
ATTRIBUTION = "Agentschap Maritieme Dienstverlening en Kust (MDK)"
|
|
LICENSE_NOTE = "Consult the official MDK product license before redistribution."
|
|
|
|
@staticmethod
|
|
def acquire(
|
|
db,
|
|
project_id: UUID,
|
|
payload: MdkBathymetryAcquireRequest,
|
|
*,
|
|
settings: Settings | None = None,
|
|
opener: Callable[..., Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
resolved_settings = settings or get_settings()
|
|
if not resolved_settings.mdk_bathymetry_acquisition_enabled:
|
|
raise AppError(
|
|
code="MDK_BATHYMETRY_ACQUISITION_DISABLED",
|
|
message=(
|
|
"MDK bathymetry acquisition is disabled. Enable it explicitly with "
|
|
"MDK_BATHYMETRY_ACQUISITION_ENABLED=true after the readiness probe reports reachable."
|
|
),
|
|
status_code=409,
|
|
)
|
|
coverage_id = (resolved_settings.mdk_bathymetry_coverage_id or "").strip()
|
|
if not coverage_id:
|
|
raise AppError(
|
|
code="MDK_BATHYMETRY_COVERAGE_NOT_CONFIGURED",
|
|
message="MDK_BATHYMETRY_COVERAGE_ID is not configured; GeoIntel will not guess coverage identifiers.",
|
|
status_code=409,
|
|
)
|
|
|
|
bbox = MdkBathymetryAcquisitionService._validated_bbox(payload, resolved_settings)
|
|
|
|
probe = MdkBathymetryProbeService.probe(settings=resolved_settings, opener=opener)
|
|
if probe.get("status") != "reachable":
|
|
raise AppError(
|
|
code="MDK_BATHYMETRY_ENDPOINT_NOT_READY",
|
|
message="The live MDK readiness probe does not report a reachable, TLS-verified WCS endpoint.",
|
|
details={"probe_status": probe.get("status"), "probe_message": probe.get("message")},
|
|
status_code=502,
|
|
)
|
|
if coverage_id not in (probe.get("coverage_identifiers") or []):
|
|
raise AppError(
|
|
code="MDK_BATHYMETRY_COVERAGE_NOT_ADVERTISED",
|
|
message="The configured coverage id is not advertised by the live MDK capabilities document.",
|
|
details={
|
|
"configured_coverage_id": coverage_id,
|
|
"advertised_coverage_identifiers": probe.get("coverage_identifiers") or [],
|
|
},
|
|
status_code=502,
|
|
)
|
|
|
|
request_url = MdkBathymetryAcquisitionService._get_coverage_url(resolved_settings, coverage_id, bbox)
|
|
request_hash = hashlib.sha256(request_url.encode("utf-8")).hexdigest()
|
|
filename = f"mdk_bathymetry_{request_hash[:12]}.tif"
|
|
|
|
if not payload.force_refresh:
|
|
cached = MdkBathymetryAcquisitionService._cached_dataset(db, project_id, filename)
|
|
if cached is not None:
|
|
return MdkBathymetryAcquisitionResult(
|
|
output_dataset_id=cached.id,
|
|
reused=True,
|
|
provider=MdkBathymetryAcquisitionService.PROVIDER,
|
|
coverage_id=coverage_id,
|
|
bbox_epsg4326=bbox,
|
|
vertical_reference=MdkBathymetryAcquisitionService.VERTICAL_REFERENCE,
|
|
resolution_m=MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M,
|
|
attribution=MdkBathymetryAcquisitionService.ATTRIBUTION,
|
|
limitation_message=MdkBathymetryAcquisitionService.LIMITATION,
|
|
).model_dump(mode="json")
|
|
|
|
content, content_type = MdkBathymetryAcquisitionService._fetch(request_url, resolved_settings, opener)
|
|
validation = MdkBathymetryAcquisitionService._validate_geotiff(content)
|
|
acquired_at = datetime.now(UTC)
|
|
|
|
dataset = DatasetService.import_raster_bytes(
|
|
db,
|
|
project_id=project_id,
|
|
area_id=payload.area_id,
|
|
filename=filename,
|
|
content=content,
|
|
source=f"MDK Belgian Continental Shelf WCS {coverage_id}",
|
|
source_name=MdkBathymetryAcquisitionService.PROVIDER,
|
|
source_metadata={
|
|
"provider": MdkBathymetryAcquisitionService.PROVIDER,
|
|
"service": "WCS",
|
|
"service_version": "1.0.0",
|
|
"coverage_id": coverage_id,
|
|
"vertical_reference": MdkBathymetryAcquisitionService.VERTICAL_REFERENCE,
|
|
"native_resolution_m": MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M,
|
|
"bbox_epsg4326": bbox,
|
|
"attribution": MdkBathymetryAcquisitionService.ATTRIBUTION,
|
|
"license_note": MdkBathymetryAcquisitionService.LICENSE_NOTE,
|
|
"raster_validation": validation,
|
|
},
|
|
provenance_metadata={
|
|
"acquisition": "explicit_bounded_wcs_get_coverage",
|
|
"acquired_at": acquired_at.isoformat(),
|
|
"request_url": request_url,
|
|
"request_hash": request_hash,
|
|
"response_content_type": content_type,
|
|
"coverage_sha256": hashlib.sha256(content).hexdigest(),
|
|
"probe_status": probe.get("status"),
|
|
"probe_response_sha256": probe.get("response_sha256"),
|
|
"probe_checked_at": probe.get("checked_at"),
|
|
"limitation_message": MdkBathymetryAcquisitionService.LIMITATION,
|
|
},
|
|
)
|
|
return MdkBathymetryAcquisitionResult(
|
|
output_dataset_id=dataset.id,
|
|
reused=False,
|
|
provider=MdkBathymetryAcquisitionService.PROVIDER,
|
|
coverage_id=coverage_id,
|
|
bbox_epsg4326=bbox,
|
|
vertical_reference=MdkBathymetryAcquisitionService.VERTICAL_REFERENCE,
|
|
resolution_m=MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M,
|
|
attribution=MdkBathymetryAcquisitionService.ATTRIBUTION,
|
|
limitation_message=MdkBathymetryAcquisitionService.LIMITATION,
|
|
).model_dump(mode="json")
|
|
|
|
@staticmethod
|
|
def _validated_bbox(payload: MdkBathymetryAcquireRequest, settings: Settings) -> list[float]:
|
|
bbox = payload.bbox
|
|
min_x, min_y, max_x, max_y = (
|
|
float(bbox.min_x),
|
|
float(bbox.min_y),
|
|
float(bbox.max_x),
|
|
float(bbox.max_y),
|
|
)
|
|
if max_x <= min_x or max_y <= min_y:
|
|
raise AppError(
|
|
code="MDK_BATHYMETRY_INVALID_BBOX",
|
|
message="The requested bbox must have positive width and height in EPSG:4326.",
|
|
status_code=422,
|
|
)
|
|
area_deg2 = (max_x - min_x) * (max_y - min_y)
|
|
if area_deg2 > float(settings.mdk_bathymetry_max_bbox_deg2):
|
|
raise AppError(
|
|
code="MDK_BATHYMETRY_BBOX_TOO_LARGE",
|
|
message="The requested bbox exceeds the configured bounded acquisition size.",
|
|
details={
|
|
"bbox_area_deg2": area_deg2,
|
|
"max_bbox_deg2": float(settings.mdk_bathymetry_max_bbox_deg2),
|
|
},
|
|
status_code=422,
|
|
)
|
|
return [min_x, min_y, max_x, max_y]
|
|
|
|
@staticmethod
|
|
def _get_coverage_url(settings: Settings, coverage_id: str, bbox: list[float]) -> str:
|
|
parsed = urlsplit(settings.mdk_bathymetry_wcs_url.strip())
|
|
if parsed.scheme.lower() != "https" or not parsed.hostname:
|
|
raise AppError(
|
|
code="MDK_BATHYMETRY_INVALID_CONFIGURATION",
|
|
message="MDK bathymetry acquisition requires an absolute HTTPS WCS URL.",
|
|
status_code=409,
|
|
)
|
|
width, height = MdkBathymetryAcquisitionService._pixel_dimensions(bbox)
|
|
parameters = dict(parse_qsl(parsed.query, keep_blank_values=True))
|
|
parameters.update(
|
|
{
|
|
"service": "WCS",
|
|
"request": "GetCoverage",
|
|
"version": "1.0.0",
|
|
"coverage": coverage_id,
|
|
"crs": settings.mdk_bathymetry_request_crs,
|
|
"bbox": ",".join(f"{value:.8f}" for value in bbox),
|
|
"width": str(width),
|
|
"height": str(height),
|
|
"format": "GeoTIFF",
|
|
}
|
|
)
|
|
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(parameters), ""))
|
|
|
|
@staticmethod
|
|
def _pixel_dimensions(bbox: list[float]) -> tuple[int, int]:
|
|
min_x, min_y, max_x, max_y = bbox
|
|
# Approximate meters per degree near the Belgian North Sea (~51.5N).
|
|
meters_per_deg_lat = 111_320.0
|
|
meters_per_deg_lon = 69_400.0
|
|
width = int((max_x - min_x) * meters_per_deg_lon / MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M)
|
|
height = int((max_y - min_y) * meters_per_deg_lat / MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M)
|
|
width = max(1, min(width, MdkBathymetryAcquisitionService.MAX_PIXELS_PER_SIDE))
|
|
height = max(1, min(height, MdkBathymetryAcquisitionService.MAX_PIXELS_PER_SIDE))
|
|
return width, height
|
|
|
|
@staticmethod
|
|
def _fetch(request_url: str, settings: Settings, opener: Callable[..., Any] | None = None) -> tuple[bytes, str]:
|
|
request = Request(
|
|
request_url,
|
|
headers={
|
|
"Accept": "image/tiff,*/*;q=0.1",
|
|
"User-Agent": "GeoIntel/1.0 MDK-bathymetry-bounded-acquisition",
|
|
},
|
|
)
|
|
max_bytes = settings.mdk_bathymetry_acquisition_max_response_mb * 1024 * 1024
|
|
try:
|
|
with (opener or guarded_opener(request_url))(request, timeout=settings.mdk_bathymetry_acquisition_timeout_seconds) as response:
|
|
content_type = str(response.headers.get("Content-Type", "")) if hasattr(response, "headers") else ""
|
|
content = response.read(max_bytes + 1)
|
|
except HTTPError as exc:
|
|
preview = exc.read(300).decode("utf-8", errors="replace")
|
|
raise AppError(
|
|
code="MDK_BATHYMETRY_PROVIDER_UNAVAILABLE",
|
|
message="The MDK WCS could not complete the bounded GetCoverage request.",
|
|
details={"provider_status_code": int(exc.code), "response_preview": preview},
|
|
status_code=502,
|
|
) from exc
|
|
except (URLError, TimeoutError, OSError) as exc:
|
|
raise AppError(
|
|
code="MDK_BATHYMETRY_PROVIDER_UNAVAILABLE",
|
|
message="The MDK WCS could not be reached for the bounded GetCoverage request.",
|
|
details={"reason": str(exc)},
|
|
status_code=502,
|
|
) from exc
|
|
if len(content) > max_bytes:
|
|
raise AppError(
|
|
code="MDK_BATHYMETRY_RESPONSE_TOO_LARGE",
|
|
message="The MDK coverage response exceeds the configured size limit.",
|
|
status_code=502,
|
|
)
|
|
if not content.startswith((b"II*\x00", b"MM\x00*")):
|
|
preview = content[:300].decode("utf-8", errors="replace")
|
|
raise AppError(
|
|
code="MDK_BATHYMETRY_INVALID_RESPONSE",
|
|
message="The MDK WCS did not return a GeoTIFF coverage.",
|
|
details={"content_type": content_type, "response_preview": preview},
|
|
status_code=502,
|
|
)
|
|
return content, content_type
|
|
|
|
@staticmethod
|
|
def _validate_geotiff(content: bytes) -> dict[str, Any]:
|
|
try:
|
|
import numpy as np
|
|
from rasterio.io import MemoryFile
|
|
except ImportError as exc:
|
|
raise AppError(
|
|
code="RASTER_PROCESSING_UNAVAILABLE",
|
|
message="Rasterio is required to validate the MDK bathymetry coverage before persistence.",
|
|
status_code=503,
|
|
) from exc
|
|
try:
|
|
with MemoryFile(content) as memory, memory.open() as source:
|
|
if source.count < 1:
|
|
raise AppError(
|
|
code="MDK_BATHYMETRY_INVALID_RESPONSE",
|
|
message="The MDK coverage contains no raster bands.",
|
|
status_code=502,
|
|
)
|
|
band = source.read(1, masked=True)
|
|
valid = band.compressed()
|
|
if valid.size == 0:
|
|
raise AppError(
|
|
code="MDK_BATHYMETRY_NO_VALID_DATA",
|
|
message="The MDK coverage contains no valid depth cells in this selection.",
|
|
status_code=422,
|
|
)
|
|
return {
|
|
"crs": str(source.crs) if source.crs else None,
|
|
"width": int(source.width),
|
|
"height": int(source.height),
|
|
"nodata": None if source.nodata is None else float(source.nodata),
|
|
"valid_cell_count": int(valid.size),
|
|
"minimum_value": float(np.min(valid)),
|
|
"maximum_value": float(np.max(valid)),
|
|
}
|
|
except AppError:
|
|
raise
|
|
except Exception as exc: # rasterio raises many distinct errors for corrupt input
|
|
raise AppError(
|
|
code="MDK_BATHYMETRY_INVALID_RESPONSE",
|
|
message="The MDK coverage could not be opened as a valid GeoTIFF.",
|
|
details={"reason": str(exc)},
|
|
status_code=502,
|
|
) from exc
|
|
|
|
@staticmethod
|
|
def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None:
|
|
from pathlib import Path
|
|
|
|
candidate = (
|
|
db.query(Dataset)
|
|
.filter(
|
|
Dataset.project_id == project_id,
|
|
Dataset.name == filename,
|
|
Dataset.source_name == MdkBathymetryAcquisitionService.PROVIDER,
|
|
Dataset.status == "ready",
|
|
)
|
|
.order_by(Dataset.imported_at.desc())
|
|
.first()
|
|
)
|
|
return candidate if candidate and candidate.storage_path and Path(candidate.storage_path).is_file() else None
|