JensandClaude Opus 5
16dedeb670
Two defects of the same kind: work that is supposed to be bounded is not. The analysis worker selected queued jobs and then set them to running in a second statement. A restarted process overlapping the previous one, or a second replica, could both select the same row and both start tiled GPU inference on it — duplicate analysis runs and double the GPU load. The AOI worker beside it already claims with FOR UPDATE SKIP LOCKED; this uses a conditional update, which is the same guarantee in one statement. run_once now reports jobs it actually claimed rather than jobs it looked at. urlopen follows redirects, so although every acquisition URL is built from settings and cannot be steered by a request payload, a misconfigured or compromised upstream could send the runtime to the loopback interface, to another container on the compose network, or to a cloud metadata endpoint — and the bytes would then be persisted under an official provenance. That is exactly the substitution the product forbids. All eight fetch sites now open through a guard that refuses private, loopback and link-local destinations (resolving the host first, so a DNS name cannot hide one) and refuses a redirect that leaves the configured origin or downgrades from HTTPS. The guard is proven by calling the services' own fetch paths, not by grepping for the call: every existing acquisition test injects an opener, which bypasses it by design. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
254 lines
11 KiB
Python
254 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime
|
|
import hashlib
|
|
import ssl
|
|
from typing import Any, Callable
|
|
from urllib.error import HTTPError, URLError
|
|
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
|
from urllib.request import Request, urlopen
|
|
from xml.etree import ElementTree
|
|
|
|
from app.core.config import Settings, get_settings
|
|
from app.schemas.bathymetry import BathymetrySourceProbeRead
|
|
|
|
|
|
class MdkBathymetryProbeService:
|
|
SOURCE_KEY = "mdk_bcp_bathymetry"
|
|
LIMITATION = (
|
|
"Deze probe leest alleen WCS GetCapabilities met strikte TLS-controle. "
|
|
"GeoIntel downloadt of activeert geen Noordzee-raster totdat endpoint, coverage-id, CRS, LAT, "
|
|
"nodata, resolutie, begrenzing en responslimieten live zijn gevalideerd."
|
|
)
|
|
|
|
@staticmethod
|
|
def _capabilities_url(configured_url: str) -> str:
|
|
parsed = urlsplit(configured_url.strip())
|
|
if parsed.scheme.lower() != "https" or not parsed.hostname:
|
|
raise ValueError("The MDK WCS probe requires an absolute HTTPS URL")
|
|
parameters = dict(parse_qsl(parsed.query, keep_blank_values=True))
|
|
parameters.update({"service": "WCS", "request": "GetCapabilities", "version": "1.0.0"})
|
|
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(parameters), ""))
|
|
|
|
@staticmethod
|
|
def _read_capabilities(
|
|
capabilities_url: str,
|
|
settings: Settings,
|
|
opener: Callable[..., Any] | None,
|
|
) -> tuple[bytes, str]:
|
|
request = Request(
|
|
capabilities_url,
|
|
headers={
|
|
"Accept": "application/xml,text/xml;q=0.9,*/*;q=0.1",
|
|
"User-Agent": "GeoIntel/1.0 MDK-bathymetry-readiness-probe",
|
|
},
|
|
)
|
|
with (opener or guarded_opener(capabilities_url))(request, timeout=settings.mdk_bathymetry_probe_timeout_seconds) as response:
|
|
limit = settings.mdk_bathymetry_probe_max_response_mb * 1024 * 1024
|
|
content = response.read(limit + 1)
|
|
if len(content) > limit:
|
|
raise ValueError("MDK WCS capabilities response exceeded the configured size limit")
|
|
content_type = str(response.headers.get("Content-Type") or "") if hasattr(response, "headers") else ""
|
|
return content, content_type
|
|
|
|
@staticmethod
|
|
def _local_name(tag: str) -> str:
|
|
return tag.rsplit("}", 1)[-1].casefold()
|
|
|
|
@staticmethod
|
|
def _parse_capabilities(content: bytes) -> dict[str, Any]:
|
|
root = ElementTree.fromstring(content)
|
|
root_name = MdkBathymetryProbeService._local_name(root.tag)
|
|
if "capabilities" not in root_name:
|
|
raise ValueError("MDK endpoint did not return a WCS capabilities document")
|
|
|
|
coverage_identifiers: set[str] = set()
|
|
advertised_formats: set[str] = set()
|
|
advertised_crs: set[str] = set()
|
|
for element in root.iter():
|
|
local_name = MdkBathymetryProbeService._local_name(element.tag)
|
|
text = (element.text or "").strip()
|
|
if local_name in {"coverageofferingbrief", "coverageoffering"}:
|
|
for child in element:
|
|
if MdkBathymetryProbeService._local_name(child.tag) in {"name", "identifier"}:
|
|
identifier = (child.text or "").strip()
|
|
if identifier:
|
|
coverage_identifiers.add(identifier)
|
|
break
|
|
if local_name in {"format", "formats"} and text:
|
|
advertised_formats.add(text)
|
|
if local_name in {"requestresponsecrss", "requestcrss", "responsecrss", "nativecrss", "crs"} and text:
|
|
advertised_crs.add(text)
|
|
for attribute_value in element.attrib.values():
|
|
normalized = str(attribute_value).strip()
|
|
if "EPSG" in normalized.upper() or "CRS:" in normalized.upper():
|
|
advertised_crs.add(normalized)
|
|
|
|
return {
|
|
"wcs_version": str(root.attrib.get("version") or "") or None,
|
|
"coverage_identifiers": sorted(coverage_identifiers),
|
|
"advertised_formats": sorted(advertised_formats),
|
|
"advertised_crs": sorted(advertised_crs),
|
|
}
|
|
|
|
@staticmethod
|
|
def _result(
|
|
*,
|
|
settings: Settings,
|
|
status: str,
|
|
checked_at: datetime,
|
|
message: str,
|
|
capabilities_url: str | None = None,
|
|
tls_verified: bool = False,
|
|
capabilities_reachable: bool = False,
|
|
response_sha256: str | None = None,
|
|
parsed: dict[str, Any] | None = None,
|
|
) -> dict[str, Any]:
|
|
parsed = parsed or {}
|
|
return BathymetrySourceProbeRead(
|
|
source_key=MdkBathymetryProbeService.SOURCE_KEY,
|
|
status=status,
|
|
configured_url=settings.mdk_bathymetry_wcs_url,
|
|
capabilities_url=capabilities_url,
|
|
tls_verified=tls_verified,
|
|
capabilities_reachable=capabilities_reachable,
|
|
acquisition_supported=False,
|
|
wcs_version=parsed.get("wcs_version"),
|
|
coverage_identifiers=parsed.get("coverage_identifiers") or [],
|
|
advertised_formats=parsed.get("advertised_formats") or [],
|
|
advertised_crs=parsed.get("advertised_crs") or [],
|
|
response_sha256=response_sha256,
|
|
checked_at=checked_at,
|
|
message=message,
|
|
limitation_message=MdkBathymetryProbeService.LIMITATION,
|
|
).model_dump(mode="json")
|
|
|
|
@staticmethod
|
|
def probe(
|
|
*,
|
|
settings: Settings | None = None,
|
|
opener: Callable[..., Any] | None = None,
|
|
checked_at: datetime | None = None,
|
|
) -> dict[str, Any]:
|
|
resolved_settings = settings or get_settings()
|
|
now = checked_at or datetime.now(UTC)
|
|
if not resolved_settings.mdk_bathymetry_probe_enabled:
|
|
return MdkBathymetryProbeService._result(
|
|
settings=resolved_settings,
|
|
status="disabled",
|
|
checked_at=now,
|
|
message="MDK bathymetry readiness probing is disabled.",
|
|
)
|
|
try:
|
|
capabilities_url = MdkBathymetryProbeService._capabilities_url(
|
|
resolved_settings.mdk_bathymetry_wcs_url
|
|
)
|
|
except ValueError as exc:
|
|
return MdkBathymetryProbeService._result(
|
|
settings=resolved_settings,
|
|
status="invalid_configuration",
|
|
checked_at=now,
|
|
message=str(exc),
|
|
)
|
|
|
|
try:
|
|
content, content_type = MdkBathymetryProbeService._read_capabilities(
|
|
capabilities_url, resolved_settings, opener
|
|
)
|
|
except HTTPError as exc:
|
|
return MdkBathymetryProbeService._result(
|
|
settings=resolved_settings,
|
|
status="endpoint_unavailable",
|
|
checked_at=now,
|
|
capabilities_url=capabilities_url,
|
|
tls_verified=True,
|
|
message=f"MDK WCS GetCapabilities returned HTTP {exc.code}.",
|
|
)
|
|
except ssl.SSLCertVerificationError:
|
|
return MdkBathymetryProbeService._result(
|
|
settings=resolved_settings,
|
|
status="tls_error",
|
|
checked_at=now,
|
|
capabilities_url=capabilities_url,
|
|
message="MDK WCS TLS certificate validation failed; insecure fallback is prohibited.",
|
|
)
|
|
except URLError as exc:
|
|
reason = exc.reason
|
|
is_tls_error = isinstance(reason, (ssl.SSLError, ssl.CertificateError)) or "certificate" in str(
|
|
reason
|
|
).casefold()
|
|
return MdkBathymetryProbeService._result(
|
|
settings=resolved_settings,
|
|
status="tls_error" if is_tls_error else "endpoint_unavailable",
|
|
checked_at=now,
|
|
capabilities_url=capabilities_url,
|
|
message=(
|
|
"MDK WCS TLS certificate validation failed; insecure fallback is prohibited."
|
|
if is_tls_error
|
|
else "MDK WCS GetCapabilities could not be reached."
|
|
),
|
|
)
|
|
except (TimeoutError, OSError) as exc:
|
|
return MdkBathymetryProbeService._result(
|
|
settings=resolved_settings,
|
|
status="endpoint_unavailable",
|
|
checked_at=now,
|
|
capabilities_url=capabilities_url,
|
|
message=f"MDK WCS GetCapabilities could not be reached ({type(exc).__name__}).",
|
|
)
|
|
except ValueError as exc:
|
|
return MdkBathymetryProbeService._result(
|
|
settings=resolved_settings,
|
|
status="invalid_capabilities",
|
|
checked_at=now,
|
|
capabilities_url=capabilities_url,
|
|
tls_verified=True,
|
|
capabilities_reachable=True,
|
|
message=str(exc),
|
|
)
|
|
|
|
response_sha256 = hashlib.sha256(content).hexdigest()
|
|
try:
|
|
parsed = MdkBathymetryProbeService._parse_capabilities(content)
|
|
except (ElementTree.ParseError, ValueError) as exc:
|
|
detail = " ".join(str(exc).split())
|
|
if content_type:
|
|
detail = f"{detail} Content-Type: {content_type}."
|
|
return MdkBathymetryProbeService._result(
|
|
settings=resolved_settings,
|
|
status="invalid_capabilities",
|
|
checked_at=now,
|
|
capabilities_url=capabilities_url,
|
|
tls_verified=True,
|
|
capabilities_reachable=True,
|
|
response_sha256=response_sha256,
|
|
message=detail,
|
|
)
|
|
|
|
if not parsed["coverage_identifiers"]:
|
|
return MdkBathymetryProbeService._result(
|
|
settings=resolved_settings,
|
|
status="invalid_capabilities",
|
|
checked_at=now,
|
|
capabilities_url=capabilities_url,
|
|
tls_verified=True,
|
|
capabilities_reachable=True,
|
|
response_sha256=response_sha256,
|
|
parsed=parsed,
|
|
message="MDK WCS capabilities are reachable but advertise no coverage identifier.",
|
|
)
|
|
return MdkBathymetryProbeService._result(
|
|
settings=resolved_settings,
|
|
status="reachable",
|
|
checked_at=now,
|
|
capabilities_url=capabilities_url,
|
|
tls_verified=True,
|
|
capabilities_reachable=True,
|
|
response_sha256=response_sha256,
|
|
parsed=parsed,
|
|
message=(
|
|
"MDK WCS capabilities are reachable with verified TLS. "
|
|
"Raster acquisition remains disabled pending bounded coverage validation."
|
|
),
|
|
)
|