feat(scope): make Belgium and North Sea operational default
GeoIntel release gates / Compile, test, contracts and builds (push) Canceled after 0s
GeoIntel release gates / Python and npm vulnerability policy (push) Canceled after 0s
GeoIntel release gates / GIS image, SBOM and container scan (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-22 02:11:48 +02:00
parent 46884cbbc9
commit 0aff8e3b8c
61 changed files with 2499 additions and 194 deletions
+21
View File
@@ -50,6 +50,7 @@ from app.schemas import (
BathymetryProfileAcquireRequest,
BathymetryRasterSelectionRequest,
BathymetryRasterSelectionResponse,
MdkBathymetryAcquireRequest,
ThematicRasterAcquireRequest,
ThematicRasterProductRead,
ThematicRasterSelectionResponse,
@@ -92,6 +93,7 @@ from app.services.flood_hazard_acquisition_service import FloodHazardAcquisition
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
from app.services.bathymetry_raster_analysis_service import BathymetryRasterAnalysisService
from app.services.mdk_bathymetry_acquisition_service import MdkBathymetryAcquisitionService
from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
@@ -342,6 +344,25 @@ def probe_mdk_bathymetry_readiness(project_id: UUID, db: Session = Depends(get_d
return envelope(MdkBathymetryProbeService.probe())
@router.post(
"/datasets/bathymetry/mdk/acquire",
response_model=Envelope[JobRead],
)
def acquire_bounded_mdk_bathymetry(
project_id: UUID,
payload: MdkBathymetryAcquireRequest,
db: Session = Depends(get_db),
):
job = JobService.run_sync_job(
db=db,
project_id=project_id,
job_type="raster.mdk_bathymetry.acquire",
parameters=payload.model_dump(mode="json"),
operation=lambda: MdkBathymetryAcquisitionService.acquire(db, project_id, payload),
)
return envelope(job)
@router.post(
"/datasets/bathymetry/profiles/acquire",
response_model=Envelope[JobRead],
+6 -1
View File
@@ -6,6 +6,7 @@ from fastapi import APIRouter, Depends, Query
from fastapi.responses import FileResponse
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.db.session import get_db
from app.schemas import Envelope
from app.schemas.export import (
@@ -47,7 +48,11 @@ def export_geojson(payload: GeoJsonExportRequest, db: Session = Depends(get_db))
)
if payload.dataset_id is not None:
return envelope(ExportService.export_dataset_geojson(db, payload.dataset_id, payload.name).model_dump(mode="json"))
return envelope({})
raise AppError(
code="INVALID_EXPORT_REQUEST",
message="GeoJSON export request does not match any supported export target",
status_code=422,
)
@router.post("/metadata", response_model=Envelope[ExportCreateResponse])
+6 -1
View File
@@ -147,6 +147,11 @@ def capabilities() -> SystemCapabilitiesEnvelope:
)
yolo_configured = bool(configured_yolo and configured_yolo.configured)
yolo_status = configured_yolo.status if configured_yolo else "not_configured"
configured_sam = ModelRegistryService.get_model_capability(
settings.sam_model_id,
settings=settings,
task_type="segmentation",
)
postgis_ready = _database_checks()["postgis"].startswith("ok:")
return SystemCapabilitiesEnvelope(
data=SystemCapabilities(
@@ -155,7 +160,7 @@ def capabilities() -> SystemCapabilitiesEnvelope:
geopandas=_dependency_enabled("geopandas"),
yolo=yolo_configured,
yolo_status=yolo_status,
sam=False,
sam=bool(configured_sam and configured_sam.configured),
grb="bounded",
sentinel="planned",
version=settings.app_version,
+44
View File
@@ -247,6 +247,27 @@ class Settings(BaseSettings):
default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs",
validation_alias="THEMATIC_RASTER_WCS_URL",
)
mdk_bathymetry_acquisition_enabled: bool = Field(
default=False,
validation_alias="MDK_BATHYMETRY_ACQUISITION_ENABLED",
)
mdk_bathymetry_coverage_id: str | None = Field(default=None, validation_alias="MDK_BATHYMETRY_COVERAGE_ID")
mdk_bathymetry_request_crs: str = Field(default="EPSG:4326", validation_alias="MDK_BATHYMETRY_REQUEST_CRS")
mdk_bathymetry_max_bbox_deg2: float = Field(
default=0.25,
gt=0,
validation_alias="MDK_BATHYMETRY_MAX_BBOX_DEG2",
)
mdk_bathymetry_acquisition_timeout_seconds: int = Field(
default=120,
ge=1,
validation_alias="MDK_BATHYMETRY_ACQUISITION_TIMEOUT_SECONDS",
)
mdk_bathymetry_acquisition_max_response_mb: int = Field(
default=160,
ge=1,
validation_alias="MDK_BATHYMETRY_ACQUISITION_MAX_RESPONSE_MB",
)
thematic_raster_min_side_m: float = Field(default=100.0, gt=0, validation_alias="THEMATIC_RASTER_MIN_SIDE_M")
thematic_raster_max_side_m: float = Field(default=60_000.0, gt=0, validation_alias="THEMATIC_RASTER_MAX_SIDE_M")
thematic_raster_max_pixels: int = Field(default=30_000_000, ge=1, validation_alias="THEMATIC_RASTER_MAX_PIXELS")
@@ -272,6 +293,29 @@ class Settings(BaseSettings):
yolo_max_detections: int = Field(default=1000, validation_alias="YOLO_MAX_DETECTIONS")
yolo_duplicate_iou_threshold: float = Field(default=0.5, ge=0.0, le=1.0, validation_alias="YOLO_DUPLICATE_IOU_THRESHOLD")
yolo_batch_size: int = Field(default=1, validation_alias="YOLO_BATCH_SIZE")
yolo_seg_enabled: bool = Field(default=False, validation_alias="YOLO_SEG_ENABLED")
yolo_seg_model_path: str | None = Field(default=None, validation_alias="YOLO_SEG_MODEL_PATH")
yolo_seg_model_id: str = Field(default="yolo-seg-configured", validation_alias="YOLO_SEG_MODEL_ID")
yolo_seg_model_display_name: str = Field(
default="Configured YOLO segmentation",
validation_alias="YOLO_SEG_MODEL_DISPLAY_NAME",
)
yolo_seg_model_version: str | None = Field(default=None, validation_alias="YOLO_SEG_MODEL_VERSION")
sam_enabled: bool = Field(default=False, validation_alias="SAM_ENABLED")
sam_model_path: str | None = Field(default=None, validation_alias="SAM_MODEL_PATH")
sam_model_id: str = Field(default="sam-configured", validation_alias="SAM_MODEL_ID")
sam_model_display_name: str = Field(
default="Configured SAM segmentation",
validation_alias="SAM_MODEL_DISPLAY_NAME",
)
sam_model_version: str | None = Field(default=None, validation_alias="SAM_MODEL_VERSION")
segmentation_max_masks_per_tile: int = Field(default=300, ge=1, validation_alias="SEGMENTATION_MAX_MASKS_PER_TILE")
segmentation_duplicate_iou_threshold: float = Field(
default=0.5,
ge=0.0,
le=1.0,
validation_alias="SEGMENTATION_DUPLICATE_IOU_THRESHOLD",
)
ollama_enabled: bool = Field(default=False, validation_alias="OLLAMA_ENABLED")
ollama_base_url: str = Field(default="http://127.0.0.1:11434", validation_alias="OLLAMA_BASE_URL")
ollama_default_model: str = Field(default="qwen3.5:9b", validation_alias="OLLAMA_DEFAULT_MODEL")
+1 -1
View File
@@ -18,7 +18,7 @@ class Project(Base):
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
name: Mapped[str] = mapped_column(String(255), nullable=False)
description: Mapped[str | None] = mapped_column(Text, nullable=True)
region: Mapped[str] = mapped_column(String(120), default="Kempen")
region: Mapped[str] = mapped_column(String(120), default="Belgium and Belgian North Sea")
status: Mapped[str] = mapped_column(String(32), default="active")
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
+4
View File
@@ -99,6 +99,8 @@ from .bathymetry import (
BathymetryRasterSelectionSummary,
BathymetrySourceProbeRead,
BathymetrySourceRead,
MdkBathymetryAcquireRequest,
MdkBathymetryAcquisitionResult,
)
from .thematic_raster import (
ThematicRasterAcquireRequest,
@@ -265,6 +267,8 @@ __all__ = [
"BathymetryPartitionFinalizationResult",
"BathymetrySourceProbeRead",
"BathymetrySourceRead",
"MdkBathymetryAcquireRequest",
"MdkBathymetryAcquisitionResult",
"ThematicRasterAcquireRequest",
"ThematicRasterAcquisitionResult",
"ThematicRasterMetric",
+18
View File
@@ -111,6 +111,24 @@ class BathymetrySourceProbeRead(BaseModel):
limitation_message: str
class MdkBathymetryAcquireRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
force_refresh: bool = False
class MdkBathymetryAcquisitionResult(BaseModel):
output_dataset_id: UUID
reused: bool
provider: str
coverage_id: str
bbox_epsg4326: list[float]
vertical_reference: str
resolution_m: float = Field(gt=0)
attribution: str
limitation_message: str
class BathymetryRasterSelectionRequest(BaseModel):
bbox: VectorSelectionBBox
area_id: UUID | None = None
+1 -1
View File
@@ -10,7 +10,7 @@ from pydantic import BaseModel
class ProjectCreate(BaseModel):
name: str
description: str | None = None
region: str | None = "Kempen"
region: str | None = "Belgium and Belgian North Sea"
class ProjectUpdate(BaseModel):
@@ -134,8 +134,37 @@ class BathymetryProfileAcquisitionService:
)
@staticmethod
def list_sources() -> list[dict[str, Any]]:
return [BathymetrySourceRead(**item).model_dump() for item in BathymetryProfileAcquisitionService._SOURCES]
def list_sources(settings=None) -> list[dict[str, Any]]:
from app.core.config import get_settings
resolved_settings = settings or get_settings()
items: list[dict[str, Any]] = []
for source in BathymetryProfileAcquisitionService._SOURCES:
item = dict(source)
if item["key"] == "mdk_bcp_bathymetry":
mdk_configured = bool(
resolved_settings.mdk_bathymetry_acquisition_enabled
and (resolved_settings.mdk_bathymetry_coverage_id or "").strip()
)
item["acquisition_supported"] = True
item["configured"] = mdk_configured
if mdk_configured:
item["integration_status"] = "operational"
item["limitation_message"] = (
"Begrensde WCS-acquisitie is expliciet ingeschakeld en draait alleen wanneer de "
"live readiness-probe bereikbaar is en het geconfigureerde coverage-id door de "
"capabilities wordt geadverteerd. Dieptes blijven LAT-gerefereerd; watervolume "
"blijft zonder compatibel wateroppervlak niet ondersteund."
)
else:
item["limitation_message"] = (
"Begrensde WCS-acquisitie bestaat maar staat uit. Zet "
"MDK_BATHYMETRY_ACQUISITION_ENABLED=true en configureer MDK_BATHYMETRY_COVERAGE_ID "
"pas nadat de readiness-probe live 'reachable' rapporteert. Er wordt nooit "
"onbeveiligd of ongevalideerd gedownload."
)
items.append(item)
return [BathymetrySourceRead(**item).model_dump() for item in items]
@staticmethod
def _validate_bbox(payload: BathymetryProfileAcquireRequest) -> tuple[float, float, float, float]:
@@ -272,11 +272,11 @@ SOURCE_DEFINITIONS = (
attribution="Brussels UrbIS",
license_note="Consult the license of the selected UrbIS dataset.",
limitation_message=(
"Bounded UrbIS buildings and cadastral parcels are operational; "
"other Brussels themes remain unavailable until separately governed."
"Bounded UrbIS buildings, cadastral parcels, street axes and Land Cover blocks are operational. "
"Permanent water uses the official WB block class; no separate hydrography network is inferred."
),
materialized_source_names=("urbis",),
operational_themes=("buildings", "parcels"),
operational_themes=("buildings", "parcels", "roads", "surface_water", "land_cover_use"),
),
_contract(
source_name="rbins_marine_reporting_units",
@@ -341,7 +341,10 @@ SOURCE_DEFINITIONS = (
source_url="https://www.vlaanderen.be/datavindplaats",
attribution="Agentschap Maritieme Dienstverlening en Kust (MDK)",
license_note="Consult the official product license before acquisition.",
limitation_message="Strict-TLS acquisition and vertical datum evidence are not yet sufficient; no depths are synthesized.",
limitation_message=(
"Bounded strict-TLS WCS acquisition is implemented but stays disabled until the operator enables it "
"with a live-validated coverage id; no depths are synthesized."
),
),
)
@@ -377,6 +380,9 @@ REGIONAL_THEME_DATASETS: dict[str, dict[str, dict[str, tuple[str, ...]]]] = {
"urbis": {
"buildings": {"urbis": ("buildings",)},
"parcels": {"urbis": ("parcels",)},
"roads": {"urbis": ("roads",)},
"surface_water": {"urbis": ("water",)},
"land_cover_use": {"urbis": ("space_occupation", "forest")},
},
}
@@ -39,6 +39,75 @@ def pixel_bbox_to_epsg4326_polygon(bbox: list[float], tile: dict[str, Any], crs:
return polygon
def pixel_points_to_epsg4326_polygon(points: list[list[float]], tile: dict[str, Any], crs: str | None = None) -> Polygon:
if not isinstance(points, list) or len(points) < 3:
raise AppError(
code="SEGMENTATION_INVALID_MASK",
message="Segmentation mask polygon must contain at least three pixel points",
status_code=422,
)
try:
pixel_points = [(float(point[0]), float(point[1])) for point in points]
except (TypeError, ValueError, IndexError) as exc:
raise AppError(
code="SEGMENTATION_INVALID_MASK",
message="Segmentation mask polygon points must be numeric [x, y] pairs",
status_code=422,
) from exc
transform = tile.get("transform")
if isinstance(transform, list) and len(transform) >= 6:
coordinates = [_apply_gdal_transform(transform, x, y) for x, y in pixel_points]
else:
coordinates = [_project_pixel_with_bounds(tile, x, y) for x, y in pixel_points]
source_crs = crs or tile.get("crs") or tile.get("source_crs") or "EPSG:4326"
if str(source_crs).upper() not in {"EPSG:4326", "4326"}:
transformer = Transformer.from_crs(source_crs, "EPSG:4326", always_xy=True)
coordinates = [transformer.transform(x, y) for x, y in coordinates]
if coordinates[0] != coordinates[-1]:
coordinates.append(coordinates[0])
polygon = Polygon(coordinates)
if not polygon.is_valid:
from shapely.validation import make_valid
repaired = make_valid(polygon)
polygon = _largest_polygon(repaired)
if polygon is None or polygon.is_empty or not polygon.is_valid or polygon.area <= 0:
raise AppError(
code="SEGMENTATION_INVALID_GEOMETRY",
message="Georeferenced segmentation geometry is invalid",
status_code=422,
)
return polygon
def _largest_polygon(geometry: Any) -> Polygon | None:
if isinstance(geometry, Polygon):
return geometry
candidates = [geom for geom in getattr(geometry, "geoms", []) if isinstance(geom, Polygon) and geom.area > 0]
if not candidates:
return None
return max(candidates, key=lambda geom: geom.area)
def _project_pixel_with_bounds(tile: dict[str, Any], px: float, py: float) -> tuple[float, float]:
bounds = tile.get("bounds")
pixel_window = tile.get("pixel_window")
if not (isinstance(bounds, list) and len(bounds) == 4 and isinstance(pixel_window, list) and len(pixel_window) == 4):
raise AppError(
code="DETECTION_TILE_MANIFEST_INVALID",
message="Tile manifest entries require transform or bounds plus pixel_window for georeferencing",
status_code=422,
)
left, bottom, right, top = [float(value) for value in bounds]
_, _, width, height = [float(value) for value in pixel_window]
if width <= 0 or height <= 0:
raise AppError(code="DETECTION_TILE_MANIFEST_INVALID", message="Tile pixel_window must have positive size", status_code=422)
return (left + (px / width) * (right - left), top - (py / height) * (top - bottom))
def _apply_gdal_transform(transform: list[float], x: float, y: float) -> tuple[float, float]:
c, a, b, f, d, e = [float(value) for value in transform[:6]]
return (a * x + b * y + c, d * x + e * y + f)
+41 -12
View File
@@ -130,18 +130,23 @@ class DetectionService:
)
if model.model_id == "manual-fixture-detector":
detections = DetectionService._persist_fixture_detections(
db=db,
project_id=project_id,
dataset_id=dataset_id,
analysis_run=analysis_run,
job=job,
model_name=model.model_id,
model_version=model.version,
raw_detections=parameters.get("fixture_detections"),
confidence_threshold=confidence_threshold,
class_filter=class_filter or [],
)
try:
detections = DetectionService._persist_fixture_detections(
db=db,
project_id=project_id,
dataset_id=dataset_id,
analysis_run=analysis_run,
job=job,
model_name=model.model_id,
model_version=model.version,
raw_detections=parameters.get("fixture_detections"),
confidence_threshold=confidence_threshold,
class_filter=class_filter or [],
)
except Exception as exc:
# A rejected fixture payload must never leave the run stuck in "running".
DetectionService._fail_run_after_exception(db, analysis_run, job, exc, fallback_code="DETECTION_INTERNAL_ERROR")
raise
DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections))
return DetectionRunResponse(
analysis_run_id=analysis_run.id,
@@ -183,6 +188,10 @@ class DetectionService:
error_code=exc.code,
message=exc.message,
)
except Exception as exc:
# An unexpected inference error must never leave the run stuck in "running".
DetectionService._fail_run_after_exception(db, analysis_run, job, exc, fallback_code="DETECTION_INTERNAL_ERROR")
raise
DetectionService._mark_success(db, analysis_run, job, detection_count=len(detections), extra_result=postprocess_summary)
return DetectionRunResponse(
analysis_run_id=analysis_run.id,
@@ -195,8 +204,28 @@ class DetectionService:
message="YOLO detections persisted.",
)
DetectionService._mark_failed(
db,
analysis_run,
job,
code="DETECTION_MODEL_UNAVAILABLE",
message="Detection model is unavailable",
)
raise AppError(code="DETECTION_MODEL_UNAVAILABLE", message="Detection model is unavailable", status_code=503)
@staticmethod
def _fail_run_after_exception(db, analysis_run: AnalysisRun, job: Job, exc: Exception, fallback_code: str) -> None:
try:
db.rollback()
except Exception:
pass
code = getattr(exc, "code", None) or fallback_code
message = getattr(exc, "message", None) or "Unexpected internal error during analysis run"
try:
DetectionService._mark_failed(db, analysis_run, job, code=str(code), message=str(message))
except Exception:
pass
@staticmethod
def get_run(db, analysis_run_id: uuid.UUID) -> DetectionRunRead:
run = db.get(AnalysisRun, analysis_run_id)
+16
View File
@@ -76,6 +76,22 @@ class JobService:
result_json["output_dataset_id"] = str(result_json["output_dataset_id"])
payload["result_json"] = result_json
raise
except Exception:
# An unexpected error must never leave the job stuck in "running".
try:
db.rollback()
except Exception:
pass
try:
JobService.mark_failed(
db,
created.id,
error_message="Unexpected internal error during synchronous job execution",
details={"code": "JOB_INTERNAL_ERROR"},
)
except Exception:
pass
raise
@staticmethod
def _coerce_payload(payload: dict[str, Any] | None) -> dict[str, Any]:
@@ -0,0 +1,334 @@
from __future__ import annotations
import hashlib
from datetime import UTC, datetime
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
from urllib.request import Request, urlopen
from uuid import UUID
from app.core.config import Settings, get_settings
from app.core.errors import AppError
from app.models import Dataset
from app.schemas.bathymetry import MdkBathymetryAcquireRequest, MdkBathymetryAcquisitionResult
from app.services.dataset_service import DatasetService
from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService
class MdkBathymetryAcquisitionService:
"""Bounded, fail-closed GetCoverage acquisition for the MDK Belgian North Sea depth model.
Acquisition only runs when:
- the operator explicitly enabled acquisition and configured a coverage id,
- the live strict-TLS readiness probe reports ``reachable``,
- the configured coverage id is advertised by the live capabilities document,
- the requested EPSG:4326 bbox stays within the configured size bound.
No depth values are ever synthesized, no insecure TLS fallback exists and the
LAT vertical reference is persisted with every artifact so it can never be
silently compared with TAW or mDNG data.
"""
PROVIDER = "mdk_bcp_bathymetry"
VERTICAL_REFERENCE = "LAT"
NATIVE_RESOLUTION_M = 20.0
MAX_PIXELS_PER_SIDE = 4096
LIMITATION = (
"Dieptewaarden zijn LAT-gerefereerd en gelden voor de bemonsterde survey-periode van het officiële "
"MDK-model. LAT mag nooit zonder gedocumenteerde datumtransformatie met TAW- of mDNG-gegevens worden "
"vergeleken; watervolume blijft zonder compatibel wateroppervlak niet ondersteund."
)
ATTRIBUTION = "Agentschap Maritieme Dienstverlening en Kust (MDK)"
LICENSE_NOTE = "Consult the official MDK product license before redistribution."
@staticmethod
def acquire(
db,
project_id: UUID,
payload: MdkBathymetryAcquireRequest,
*,
settings: Settings | None = None,
opener: Callable[..., Any] | None = None,
) -> dict[str, Any]:
resolved_settings = settings or get_settings()
if not resolved_settings.mdk_bathymetry_acquisition_enabled:
raise AppError(
code="MDK_BATHYMETRY_ACQUISITION_DISABLED",
message=(
"MDK bathymetry acquisition is disabled. Enable it explicitly with "
"MDK_BATHYMETRY_ACQUISITION_ENABLED=true after the readiness probe reports reachable."
),
status_code=409,
)
coverage_id = (resolved_settings.mdk_bathymetry_coverage_id or "").strip()
if not coverage_id:
raise AppError(
code="MDK_BATHYMETRY_COVERAGE_NOT_CONFIGURED",
message="MDK_BATHYMETRY_COVERAGE_ID is not configured; GeoIntel will not guess coverage identifiers.",
status_code=409,
)
bbox = MdkBathymetryAcquisitionService._validated_bbox(payload, resolved_settings)
probe = MdkBathymetryProbeService.probe(settings=resolved_settings, opener=opener)
if probe.get("status") != "reachable":
raise AppError(
code="MDK_BATHYMETRY_ENDPOINT_NOT_READY",
message="The live MDK readiness probe does not report a reachable, TLS-verified WCS endpoint.",
details={"probe_status": probe.get("status"), "probe_message": probe.get("message")},
status_code=502,
)
if coverage_id not in (probe.get("coverage_identifiers") or []):
raise AppError(
code="MDK_BATHYMETRY_COVERAGE_NOT_ADVERTISED",
message="The configured coverage id is not advertised by the live MDK capabilities document.",
details={
"configured_coverage_id": coverage_id,
"advertised_coverage_identifiers": probe.get("coverage_identifiers") or [],
},
status_code=502,
)
request_url = MdkBathymetryAcquisitionService._get_coverage_url(resolved_settings, coverage_id, bbox)
request_hash = hashlib.sha256(request_url.encode("utf-8")).hexdigest()
filename = f"mdk_bathymetry_{request_hash[:12]}.tif"
if not payload.force_refresh:
cached = MdkBathymetryAcquisitionService._cached_dataset(db, project_id, filename)
if cached is not None:
return MdkBathymetryAcquisitionResult(
output_dataset_id=cached.id,
reused=True,
provider=MdkBathymetryAcquisitionService.PROVIDER,
coverage_id=coverage_id,
bbox_epsg4326=bbox,
vertical_reference=MdkBathymetryAcquisitionService.VERTICAL_REFERENCE,
resolution_m=MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M,
attribution=MdkBathymetryAcquisitionService.ATTRIBUTION,
limitation_message=MdkBathymetryAcquisitionService.LIMITATION,
).model_dump(mode="json")
content, content_type = MdkBathymetryAcquisitionService._fetch(request_url, resolved_settings, opener)
validation = MdkBathymetryAcquisitionService._validate_geotiff(content)
acquired_at = datetime.now(UTC)
dataset = DatasetService.import_raster_bytes(
db,
project_id=project_id,
area_id=payload.area_id,
filename=filename,
content=content,
source=f"MDK Belgian Continental Shelf WCS {coverage_id}",
source_name=MdkBathymetryAcquisitionService.PROVIDER,
source_metadata={
"provider": MdkBathymetryAcquisitionService.PROVIDER,
"service": "WCS",
"service_version": "1.0.0",
"coverage_id": coverage_id,
"vertical_reference": MdkBathymetryAcquisitionService.VERTICAL_REFERENCE,
"native_resolution_m": MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M,
"bbox_epsg4326": bbox,
"attribution": MdkBathymetryAcquisitionService.ATTRIBUTION,
"license_note": MdkBathymetryAcquisitionService.LICENSE_NOTE,
"raster_validation": validation,
},
provenance_metadata={
"acquisition": "explicit_bounded_wcs_get_coverage",
"acquired_at": acquired_at.isoformat(),
"request_url": request_url,
"request_hash": request_hash,
"response_content_type": content_type,
"coverage_sha256": hashlib.sha256(content).hexdigest(),
"probe_status": probe.get("status"),
"probe_response_sha256": probe.get("response_sha256"),
"probe_checked_at": probe.get("checked_at"),
"limitation_message": MdkBathymetryAcquisitionService.LIMITATION,
},
)
return MdkBathymetryAcquisitionResult(
output_dataset_id=dataset.id,
reused=False,
provider=MdkBathymetryAcquisitionService.PROVIDER,
coverage_id=coverage_id,
bbox_epsg4326=bbox,
vertical_reference=MdkBathymetryAcquisitionService.VERTICAL_REFERENCE,
resolution_m=MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M,
attribution=MdkBathymetryAcquisitionService.ATTRIBUTION,
limitation_message=MdkBathymetryAcquisitionService.LIMITATION,
).model_dump(mode="json")
@staticmethod
def _validated_bbox(payload: MdkBathymetryAcquireRequest, settings: Settings) -> list[float]:
bbox = payload.bbox
min_x, min_y, max_x, max_y = (
float(bbox.min_x),
float(bbox.min_y),
float(bbox.max_x),
float(bbox.max_y),
)
if max_x <= min_x or max_y <= min_y:
raise AppError(
code="MDK_BATHYMETRY_INVALID_BBOX",
message="The requested bbox must have positive width and height in EPSG:4326.",
status_code=422,
)
area_deg2 = (max_x - min_x) * (max_y - min_y)
if area_deg2 > float(settings.mdk_bathymetry_max_bbox_deg2):
raise AppError(
code="MDK_BATHYMETRY_BBOX_TOO_LARGE",
message="The requested bbox exceeds the configured bounded acquisition size.",
details={
"bbox_area_deg2": area_deg2,
"max_bbox_deg2": float(settings.mdk_bathymetry_max_bbox_deg2),
},
status_code=422,
)
return [min_x, min_y, max_x, max_y]
@staticmethod
def _get_coverage_url(settings: Settings, coverage_id: str, bbox: list[float]) -> str:
parsed = urlsplit(settings.mdk_bathymetry_wcs_url.strip())
if parsed.scheme.lower() != "https" or not parsed.hostname:
raise AppError(
code="MDK_BATHYMETRY_INVALID_CONFIGURATION",
message="MDK bathymetry acquisition requires an absolute HTTPS WCS URL.",
status_code=409,
)
width, height = MdkBathymetryAcquisitionService._pixel_dimensions(bbox)
parameters = dict(parse_qsl(parsed.query, keep_blank_values=True))
parameters.update(
{
"service": "WCS",
"request": "GetCoverage",
"version": "1.0.0",
"coverage": coverage_id,
"crs": settings.mdk_bathymetry_request_crs,
"bbox": ",".join(f"{value:.8f}" for value in bbox),
"width": str(width),
"height": str(height),
"format": "GeoTIFF",
}
)
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(parameters), ""))
@staticmethod
def _pixel_dimensions(bbox: list[float]) -> tuple[int, int]:
min_x, min_y, max_x, max_y = bbox
# Approximate meters per degree near the Belgian North Sea (~51.5N).
meters_per_deg_lat = 111_320.0
meters_per_deg_lon = 69_400.0
width = int((max_x - min_x) * meters_per_deg_lon / MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M)
height = int((max_y - min_y) * meters_per_deg_lat / MdkBathymetryAcquisitionService.NATIVE_RESOLUTION_M)
width = max(1, min(width, MdkBathymetryAcquisitionService.MAX_PIXELS_PER_SIDE))
height = max(1, min(height, MdkBathymetryAcquisitionService.MAX_PIXELS_PER_SIDE))
return width, height
@staticmethod
def _fetch(request_url: str, settings: Settings, opener: Callable[..., Any] | None = None) -> tuple[bytes, str]:
request = Request(
request_url,
headers={
"Accept": "image/tiff,*/*;q=0.1",
"User-Agent": "GeoIntel/1.0 MDK-bathymetry-bounded-acquisition",
},
)
max_bytes = settings.mdk_bathymetry_acquisition_max_response_mb * 1024 * 1024
try:
with (opener or urlopen)(request, timeout=settings.mdk_bathymetry_acquisition_timeout_seconds) as response:
content_type = str(response.headers.get("Content-Type", "")) if hasattr(response, "headers") else ""
content = response.read(max_bytes + 1)
except HTTPError as exc:
preview = exc.read(300).decode("utf-8", errors="replace")
raise AppError(
code="MDK_BATHYMETRY_PROVIDER_UNAVAILABLE",
message="The MDK WCS could not complete the bounded GetCoverage request.",
details={"provider_status_code": int(exc.code), "response_preview": preview},
status_code=502,
) from exc
except (URLError, TimeoutError, OSError) as exc:
raise AppError(
code="MDK_BATHYMETRY_PROVIDER_UNAVAILABLE",
message="The MDK WCS could not be reached for the bounded GetCoverage request.",
details={"reason": str(exc)},
status_code=502,
) from exc
if len(content) > max_bytes:
raise AppError(
code="MDK_BATHYMETRY_RESPONSE_TOO_LARGE",
message="The MDK coverage response exceeds the configured size limit.",
status_code=502,
)
if not content.startswith((b"II*\x00", b"MM\x00*")):
preview = content[:300].decode("utf-8", errors="replace")
raise AppError(
code="MDK_BATHYMETRY_INVALID_RESPONSE",
message="The MDK WCS did not return a GeoTIFF coverage.",
details={"content_type": content_type, "response_preview": preview},
status_code=502,
)
return content, content_type
@staticmethod
def _validate_geotiff(content: bytes) -> dict[str, Any]:
try:
import numpy as np
from rasterio.io import MemoryFile
except ImportError as exc:
raise AppError(
code="RASTER_PROCESSING_UNAVAILABLE",
message="Rasterio is required to validate the MDK bathymetry coverage before persistence.",
status_code=503,
) from exc
try:
with MemoryFile(content) as memory, memory.open() as source:
if source.count < 1:
raise AppError(
code="MDK_BATHYMETRY_INVALID_RESPONSE",
message="The MDK coverage contains no raster bands.",
status_code=502,
)
band = source.read(1, masked=True)
valid = band.compressed()
if valid.size == 0:
raise AppError(
code="MDK_BATHYMETRY_NO_VALID_DATA",
message="The MDK coverage contains no valid depth cells in this selection.",
status_code=422,
)
return {
"crs": str(source.crs) if source.crs else None,
"width": int(source.width),
"height": int(source.height),
"nodata": None if source.nodata is None else float(source.nodata),
"valid_cell_count": int(valid.size),
"minimum_value": float(np.min(valid)),
"maximum_value": float(np.max(valid)),
}
except AppError:
raise
except Exception as exc: # rasterio raises many distinct errors for corrupt input
raise AppError(
code="MDK_BATHYMETRY_INVALID_RESPONSE",
message="The MDK coverage could not be opened as a valid GeoTIFF.",
details={"reason": str(exc)},
status_code=502,
) from exc
@staticmethod
def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None:
from pathlib import Path
candidate = (
db.query(Dataset)
.filter(
Dataset.project_id == project_id,
Dataset.name == filename,
Dataset.source_name == MdkBathymetryAcquisitionService.PROVIDER,
Dataset.status == "ready",
)
.order_by(Dataset.imported_at.desc())
.first()
)
return candidate if candidate and candidate.storage_path and Path(candidate.storage_path).is_file() else None
@@ -24,11 +24,18 @@ class ModelAssetCatalogService:
if not model_directory.exists() or not model_directory.is_dir():
return ModelAssetListResponse(items=[], total=0, model_directory=str(model_directory))
items = [
ModelAssetCatalogService._asset_from_file(path, active_model_path=active_model_path)
candidate_paths = [
path
for path in sorted(model_directory.iterdir(), key=lambda item: item.name.lower())
if path.is_file() and path.suffix.lower() in ModelAssetCatalogService.SUPPORTED_SUFFIXES
]
if active_model_path is not None:
candidate_paths = [path for path in candidate_paths if path.resolve() == active_model_path]
items = [
ModelAssetCatalogService._asset_from_file(path, active_model_path=active_model_path)
for path in candidate_paths
]
return ModelAssetListResponse(items=items, total=len(items), model_directory=str(model_directory))
@staticmethod
@@ -72,8 +79,12 @@ class ModelAssetCatalogService:
size_bytes=path.stat().st_size,
sha256=ModelAssetCatalogService._sha256(path),
active=active_model_path == resolved_path,
status="available",
limitation_message="Local runtime model asset. GeoIntel will not download or mutate model weights.",
status="approved" if active_model_path == resolved_path else "available",
limitation_message=(
"Approved local runtime model asset. GeoIntel will not download or mutate model weights."
if active_model_path == resolved_path
else "Local development model asset. Configure it explicitly before production use."
),
will_download_models=False,
)
+102 -26
View File
@@ -5,6 +5,7 @@ from typing import Type
from app.core.config import Settings, get_settings
from app.schemas.detection import DetectionModelCapability
from app.services.segmentation_adapter import SamSegmentationAdapter, YoloSegmentationAdapter
from app.services.yolo_adapter import YoloDetectionAdapter
@@ -14,10 +15,16 @@ class ModelRegistryService:
settings: Settings | None = None,
yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter,
task_type: str = "object_detection",
yolo_seg_adapter_class: Type[YoloSegmentationAdapter] = YoloSegmentationAdapter,
sam_adapter_class: Type[SamSegmentationAdapter] = SamSegmentationAdapter,
) -> list[DetectionModelCapability]:
resolved_settings = settings or get_settings()
if task_type == "segmentation":
return ModelRegistryService.list_segmentation_model_capabilities()
return ModelRegistryService.list_segmentation_model_capabilities(
settings=resolved_settings,
yolo_seg_adapter_class=yolo_seg_adapter_class,
sam_adapter_class=sam_adapter_class,
)
if task_type != "object_detection":
return []
return [
@@ -52,15 +59,28 @@ class ModelRegistryService:
settings: Settings | None = None,
yolo_adapter_class: Type[YoloDetectionAdapter] = YoloDetectionAdapter,
task_type: str = "object_detection",
yolo_seg_adapter_class: Type[YoloSegmentationAdapter] = YoloSegmentationAdapter,
sam_adapter_class: Type[SamSegmentationAdapter] = SamSegmentationAdapter,
) -> DetectionModelCapability | None:
normalized = model_id.strip()
for model in ModelRegistryService.list_model_capabilities(settings=settings, yolo_adapter_class=yolo_adapter_class, task_type=task_type):
for model in ModelRegistryService.list_model_capabilities(
settings=settings,
yolo_adapter_class=yolo_adapter_class,
task_type=task_type,
yolo_seg_adapter_class=yolo_seg_adapter_class,
sam_adapter_class=sam_adapter_class,
):
if model.model_id == normalized:
return model
return None
@staticmethod
def list_segmentation_model_capabilities() -> list[DetectionModelCapability]:
def list_segmentation_model_capabilities(
settings: Settings | None = None,
yolo_seg_adapter_class: Type[YoloSegmentationAdapter] = YoloSegmentationAdapter,
sam_adapter_class: Type[SamSegmentationAdapter] = SamSegmentationAdapter,
) -> list[DetectionModelCapability]:
resolved_settings = settings or get_settings()
return [
DetectionModelCapability(
model_id="segmentation-placeholder",
@@ -70,7 +90,7 @@ class ModelRegistryService:
supported_classes=["building", "vegetation", "water", "landuse"],
configured=False,
status="not_configured",
limitation_message="Segmentation inference is not configured in Sprint 9; no SAM/YOLO-seg model is downloaded or executed.",
limitation_message="Segmentation inference is not configured for this placeholder; no model is downloaded or executed.",
version=None,
),
DetectionModelCapability(
@@ -84,30 +104,86 @@ class ModelRegistryService:
limitation_message="Fixture segmenter is for explicit tests/demo fixtures only and is not production inference.",
version="fixture-v1",
),
DetectionModelCapability(
model_id="yolo-seg-configured",
display_name="Configured YOLO segmentation",
framework="ultralytics/pytorch",
task_type="segmentation",
supported_classes=["building", "vegetation", "water", "landuse"],
configured=False,
status="not_configured",
limitation_message="YOLO-seg is not configured in Sprint 9. GeoIntel will not download segmentation model weights automatically.",
version=None,
),
DetectionModelCapability(
model_id="sam-configured",
display_name="Configured SAM segmentation",
framework="sam",
task_type="segmentation",
supported_classes=["building", "vegetation", "water", "landuse"],
configured=False,
status="not_configured",
limitation_message="SAM is not configured in Sprint 9 and is not installed as a backend dependency.",
version=None,
),
ModelRegistryService._configured_yolo_seg_capability(resolved_settings, yolo_seg_adapter_class),
ModelRegistryService._configured_sam_capability(resolved_settings, sam_adapter_class),
]
@staticmethod
def _configured_yolo_seg_capability(
settings: Settings,
adapter_class: Type[YoloSegmentationAdapter] = YoloSegmentationAdapter,
) -> DetectionModelCapability:
configured = False
status = "not_configured"
limitation = (
"YOLO segmentation is disabled. Set YOLO_SEG_ENABLED=true and YOLO_SEG_MODEL_PATH to a local "
"segmentation model file to enable inference. GeoIntel never downloads model weights automatically."
)
model_path = Path(settings.yolo_seg_model_path).expanduser() if settings.yolo_seg_model_path else None
if settings.yolo_seg_enabled:
if not adapter_class.dependencies_available():
status = "dependency_unavailable"
limitation = "Segmentation dependencies are not installed. Install backend optional extras with geointel-backend[ai]."
elif model_path is None:
limitation = "YOLO_SEG_MODEL_PATH is not set. GeoIntel will not download segmentation model weights automatically."
elif not model_path.exists() or not model_path.is_file():
limitation = "YOLO_SEG_MODEL_PATH does not point to an existing local model file. GeoIntel will not download segmentation model weights automatically."
else:
configured = True
status = "configured"
limitation = "Configured for local YOLO segmentation inference over an existing raster tile manifest."
return DetectionModelCapability(
model_id=settings.yolo_seg_model_id,
display_name=settings.yolo_seg_model_display_name,
framework="ultralytics/pytorch",
task_type="segmentation",
supported_classes=["building", "vegetation", "water", "landuse"],
configured=configured,
status=status,
limitation_message=limitation,
version=settings.yolo_seg_model_version,
)
@staticmethod
def _configured_sam_capability(
settings: Settings,
adapter_class: Type[SamSegmentationAdapter] = SamSegmentationAdapter,
) -> DetectionModelCapability:
configured = False
status = "not_configured"
limitation = (
"SAM is disabled. Set SAM_ENABLED=true and SAM_MODEL_PATH to a local SAM-compatible model file to "
"enable class-agnostic segmentation. GeoIntel never downloads model weights automatically."
)
model_path = Path(settings.sam_model_path).expanduser() if settings.sam_model_path else None
if settings.sam_enabled:
if not adapter_class.dependencies_available():
status = "dependency_unavailable"
limitation = "Segmentation dependencies are not installed. Install backend optional extras with geointel-backend[ai]."
elif model_path is None:
limitation = "SAM_MODEL_PATH is not set. GeoIntel will not download segmentation model weights automatically."
elif not model_path.exists() or not model_path.is_file():
limitation = "SAM_MODEL_PATH does not point to an existing local model file. GeoIntel will not download segmentation model weights automatically."
else:
configured = True
status = "configured"
limitation = "Configured for local class-agnostic SAM segmentation over an existing raster tile manifest."
return DetectionModelCapability(
model_id=settings.sam_model_id,
display_name=settings.sam_model_display_name,
framework="ultralytics/sam",
task_type="segmentation",
supported_classes=["segment"],
configured=configured,
status=status,
limitation_message=limitation,
version=settings.sam_model_version,
)
@staticmethod
def _configured_yolo_capability(
settings: Settings,
@@ -71,6 +71,7 @@ class OfficialVectorProduct:
response_crs: str = "EPSG:4326"
identity_field: str | None = None
requires_coverage_area: bool = False
property_filter: dict[str, tuple[str, ...]] | None = None
class OfficialVectorAcquisitionService:
@@ -509,6 +510,57 @@ class OfficialVectorAcquisitionService:
identity_field="INSPIRE_ID",
requires_coverage_area=True,
),
OfficialVectorProduct(
key="urbis_street_axes",
display_name="UrbIS street axes",
theme="roads",
provider="Paradigm Brussels",
source_name="urbis",
reference_layer_name="roads",
service_type="WFS 2.0",
collection="urbisvector:StreetAxes",
source_crs="EPSG:31370",
source_version="2026-06-06",
observation_label="UrbIS revision 6 June 2026",
authority_level="authoritative",
catalog_url=(
"https://datastore.brussels/web/data/dataset/"
"2cf42541-1813-11ef-8a81-00090ffe0001"
),
attribution="Paradigm Brussels - UrbIS",
license_note="UrbIS topographic layers are published under CC0.",
limitation_message=(
"UrbIS street axes describe topographic road geometry for the Brussels-Capital "
"Region and are not a routing network or a traffic measurement."
),
source="UrbIS WFS",
observed_at=datetime(2026, 6, 6, tzinfo=UTC),
valid_from=None,
valid_to=None,
primary_metric={
"metric_key": "road_length",
"method": "intersection_length",
"label": "Wegaslengte",
"unit": "km",
"geometry_dimension": 1,
"is_estimate": False,
},
selection_metrics=(
{
"metric_key": "road_segment_count",
"method": "feature_count",
"label": "Wegsegmenten",
"unit": "objecten",
"geometry_dimension": 1,
},
),
geometry_types=("LineString", "MultiLineString"),
coverage_zones=("brussels",),
endpoint_kind="urbis_wfs",
response_crs="EPSG:31370",
identity_field="INSPIRE_ID",
requires_coverage_area=True,
),
OfficialVectorProduct(
key="urbis_cadastral_parcels",
display_name="UrbIS cadastral parcels",
@@ -561,6 +613,143 @@ class OfficialVectorAcquisitionService:
identity_field="INSPIRE_ID",
requires_coverage_area=True,
),
OfficialVectorProduct(
key="urbis_land_cover_blocks",
display_name="UrbIS land cover blocks",
theme="space_occupation",
provider="Paradigm Brussels",
source_name="urbis",
reference_layer_name="space_occupation",
service_type="WFS 2.0",
collection="urbisvector:Blocks",
source_crs="EPSG:31370",
source_version="UrbIS Land Cover 1.0; live WFS checked 2026-07-22",
observation_label="Current UrbIS land-cover WFS",
authority_level="authoritative",
catalog_url="https://urbisdownload.datastore.brussels/UrbIS/TechSpec/LandCover_TechSpec_NL20240401.pdf",
attribution="Paradigm Brussels - UrbIS Land Cover",
license_note="UrbIS Land Cover is available through the official download and WFS service; retain source attribution.",
limitation_message=(
"UrbIS blocks describe physical and biological land cover. They are not zoning, ownership or legal land use. "
"The WFS does not expose a separate observation date per feature."
),
source="UrbIS WFS",
observed_at=None,
valid_from=None,
valid_to=None,
primary_metric={
"metric_key": "land_cover_area",
"method": "intersection_area",
"label": "Landbedekking",
"unit": "ha",
"geometry_dimension": 2,
"is_estimate": False,
},
selection_metrics=(
{
"metric_key": "land_cover_block_count",
"method": "feature_count",
"label": "Landbedekkingsblokken",
"unit": "objecten",
"geometry_dimension": 2,
},
),
coverage_zones=("brussels",),
endpoint_kind="urbis_wfs",
response_crs="EPSG:31370",
identity_field="INSPIRE_ID",
requires_coverage_area=True,
),
OfficialVectorProduct(
key="urbis_forest_parks",
display_name="UrbIS forests and parks",
theme="forest",
provider="Paradigm Brussels",
source_name="urbis",
reference_layer_name="forest",
service_type="WFS 2.0",
collection="urbisvector:Blocks",
source_crs="EPSG:31370",
source_version="UrbIS Land Cover 1.0; live WFS checked 2026-07-22",
observation_label="Current UrbIS land-cover WFS",
authority_level="authoritative",
catalog_url="https://urbisdownload.datastore.brussels/UrbIS/TechSpec/LandCover_TechSpec_NL20240401.pdf",
attribution="Paradigm Brussels - UrbIS Land Cover",
license_note="UrbIS Land Cover is available through the official download and WFS service; retain source attribution.",
limitation_message="Includes only UrbIS block types FO (forest/woodland) and GB (parks); street trees and smaller green elements are not inferred.",
source="UrbIS WFS",
observed_at=None,
valid_from=None,
valid_to=None,
primary_metric={
"metric_key": "forest_park_area",
"method": "intersection_area",
"label": "Bos- en parkoppervlakte",
"unit": "ha",
"geometry_dimension": 2,
"is_estimate": False,
},
selection_metrics=(
{
"metric_key": "forest_park_count",
"method": "feature_count",
"label": "Bos- en parkzones",
"unit": "objecten",
"geometry_dimension": 2,
},
),
coverage_zones=("brussels",),
endpoint_kind="urbis_wfs",
response_crs="EPSG:31370",
identity_field="INSPIRE_ID",
requires_coverage_area=True,
property_filter={"TYPE": ("FO", "GB")},
),
OfficialVectorProduct(
key="urbis_water_surfaces",
display_name="UrbIS permanent water surfaces",
theme="water",
provider="Paradigm Brussels",
source_name="urbis",
reference_layer_name="water",
service_type="WFS 2.0",
collection="urbisvector:Blocks",
source_crs="EPSG:31370",
source_version="UrbIS Land Cover 1.0; live WFS checked 2026-07-22",
observation_label="Current UrbIS land-cover WFS",
authority_level="authoritative",
catalog_url="https://urbisdownload.datastore.brussels/UrbIS/TechSpec/LandCover_TechSpec_NL20240401.pdf",
attribution="Paradigm Brussels - UrbIS Land Cover",
license_note="UrbIS Land Cover is available through the official download and WFS service; retain source attribution.",
limitation_message="Includes only UrbIS block type WB: canals, lakes and watercourses with predominantly permanent water.",
source="UrbIS WFS",
observed_at=None,
valid_from=None,
valid_to=None,
primary_metric={
"metric_key": "water_surface_area",
"method": "intersection_area",
"label": "Permanent wateroppervlak",
"unit": "ha",
"geometry_dimension": 2,
"is_estimate": False,
},
selection_metrics=(
{
"metric_key": "water_surface_count",
"method": "feature_count",
"label": "Waterzones",
"unit": "objecten",
"geometry_dimension": 2,
},
),
coverage_zones=("brussels",),
endpoint_kind="urbis_wfs",
response_crs="EPSG:31370",
identity_field="INSPIRE_ID",
requires_coverage_area=True,
property_filter={"TYPE": ("WB",)},
),
)
return {product.key: product for product in products}
@@ -1081,6 +1270,12 @@ class OfficialVectorAcquisitionService:
scope_metric: Any,
coverage_scope: str,
) -> dict[str, Any] | None:
raw = dict(feature.get("properties") or {})
if product.property_filter and any(
str(raw.get(property_name) or "") not in allowed_values
for property_name, allowed_values in product.property_filter.items()
):
return None
dimension = 2 if any("Polygon" in item for item in product.geometry_types) else 1
try:
source_geometry = shape(feature.get("geometry"))
@@ -1122,7 +1317,6 @@ class OfficialVectorAcquisitionService:
)
if clipped_wgs84 is None:
return None
raw = dict(feature.get("properties") or {})
identity = (
raw.get(product.identity_field or "")
or feature.get("id")
+5 -1
View File
@@ -31,7 +31,11 @@ class ProjectService:
@staticmethod
def create_project(db: Session, payload: ProjectCreate) -> ProjectRead:
project = Project(name=payload.name.strip(), description=(payload.description or "").strip() or None, region=payload.region or "Kempen")
project = Project(
name=payload.name.strip(),
description=(payload.description or "").strip() or None,
region=payload.region or "Belgium and Belgian North Sea",
)
db.add(project)
db.commit()
db.refresh(project)
@@ -1,8 +1,13 @@
from __future__ import annotations
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Protocol
from app.core.config import Settings
from app.core.errors import AppError
from app.services.yolo_adapter import _prediction_source, _to_list
@dataclass(frozen=True)
class SegmentationAdapterResult:
@@ -23,6 +28,159 @@ class SegmentationAdapter(Protocol):
"""Future segmentation adapters must local-import model dependencies inside execution paths."""
class _UltralyticsSegmentationAdapterBase:
"""Shared local-inference plumbing for ultralytics-backed segmentation models.
Model weights are never downloaded automatically; a missing local file or
missing dependency fails closed with an explicit error.
"""
def __init__(self, settings: Settings) -> None:
self.settings = settings
@staticmethod
def dependencies_available() -> bool:
try:
import torch # noqa: F401
import ultralytics # noqa: F401
except Exception:
return False
return True
def _require_model_file(self, model_path: Path) -> None:
if not model_path.exists() or not model_path.is_file():
raise AppError(
code="SEGMENTATION_MODEL_UNAVAILABLE",
message="Configured segmentation model file does not exist",
details={"model_path": str(model_path)},
status_code=503,
)
if not self.dependencies_available():
raise AppError(
code="SEGMENTATION_DEPENDENCY_UNAVAILABLE",
message="Segmentation dependencies are not installed. Install backend optional extras with geointel-backend[ai].",
status_code=503,
)
def _predict(self, model, tile_path: Path, confidence_threshold: float) -> list[Any]:
if not tile_path.exists() or not tile_path.is_file():
raise AppError(
code="SEGMENTATION_TILE_NOT_FOUND",
message="Tile referenced by manifest does not exist",
details={"tile_path": str(tile_path)},
status_code=422,
)
try:
with _prediction_source(tile_path) as prediction_source:
return model.predict(
source=prediction_source,
conf=float(confidence_threshold),
imgsz=int(self.settings.yolo_image_size),
device=self.settings.yolo_device,
verbose=False,
)
except AppError:
raise
except Exception as exc:
raise AppError(
code="SEGMENTATION_INFERENCE_FAILED",
message="Configured segmentation inference failed for a raster tile",
details={"tile_path": str(tile_path), "error": str(exc)},
status_code=503,
) from exc
def _extract_masks(self, results: list[Any], default_class_name: str | None = None) -> list[dict[str, Any]]:
segmentations: list[dict[str, Any]] = []
max_masks = int(self.settings.segmentation_max_masks_per_tile)
for result in results:
names = getattr(result, "names", {}) or {}
masks = getattr(result, "masks", None)
if masks is None:
continue
polygons = getattr(masks, "xy", None) or []
boxes = getattr(result, "boxes", None)
confidence_values = _to_list(getattr(boxes, "conf", [])) if boxes is not None else []
class_values = _to_list(getattr(boxes, "cls", [])) if boxes is not None else []
bbox_values = _to_list(getattr(boxes, "xyxy", [])) if boxes is not None else []
for index, polygon in enumerate(polygons):
if len(segmentations) >= max_masks:
return segmentations
points = _to_list(polygon)
if not isinstance(points, list) or len(points) < 3:
continue
class_id = int(class_values[index]) if index < len(class_values) else -1
if default_class_name is not None:
class_name = default_class_name
else:
class_name = str(names.get(class_id, class_id))
confidence = float(confidence_values[index]) if index < len(confidence_values) else None
bbox = [float(value) for value in bbox_values[index]] if index < len(bbox_values) else None
segmentations.append(
{
"class_name": class_name,
"confidence": confidence,
"points": [[float(point[0]), float(point[1])] for point in points],
"bbox": bbox,
"properties": {"class_id": class_id},
}
)
return segmentations
class YoloSegmentationAdapter(_UltralyticsSegmentationAdapterBase):
def load_model(self, model_path: Path):
self._require_model_file(model_path)
try:
from ultralytics import YOLO
except ImportError as exc:
raise AppError(
code="SEGMENTATION_DEPENDENCY_UNAVAILABLE",
message="YOLO segmentation dependencies are not importable. Install backend optional extras with geointel-backend[ai].",
status_code=503,
) from exc
try:
return YOLO(str(model_path))
except Exception as exc:
raise AppError(
code="SEGMENTATION_MODEL_LOAD_FAILED",
message="Configured YOLO segmentation model could not be loaded",
details={"model_path": str(model_path)},
status_code=503,
) from exc
def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict[str, Any]]:
results = self._predict(model, tile_path, confidence_threshold)
return self._extract_masks(results)
class SamSegmentationAdapter(_UltralyticsSegmentationAdapterBase):
"""Class-agnostic SAM segmentation through the ultralytics SAM interface."""
def load_model(self, model_path: Path):
self._require_model_file(model_path)
try:
from ultralytics import SAM
except ImportError as exc:
raise AppError(
code="SEGMENTATION_DEPENDENCY_UNAVAILABLE",
message="SAM segmentation requires the ultralytics SAM interface. Install backend optional extras with geointel-backend[ai].",
status_code=503,
) from exc
try:
return SAM(str(model_path))
except Exception as exc:
raise AppError(
code="SEGMENTATION_MODEL_LOAD_FAILED",
message="Configured SAM model could not be loaded",
details={"model_path": str(model_path)},
status_code=503,
) from exc
def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict[str, Any]]:
results = self._predict(model, tile_path, confidence_threshold)
return self._extract_masks(results, default_class_name="segment")
class FixtureSegmentationAdapter:
def segment(self, raw_segmentations: Any) -> list[SegmentationAdapterResult]:
if not isinstance(raw_segmentations, list):
+239 -16
View File
@@ -19,10 +19,16 @@ from app.schemas.segmentation import (
SegmentationRunRead,
SegmentationRunResponse,
)
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
from app.services.detection_service import DetectionService
from app.services.model_registry_service import ModelRegistryService
from app.services.qa_service import QaService
from app.services.quality_service import QualityService
from app.services.segmentation_adapter import FixtureSegmentationAdapter
from app.services.segmentation_adapter import (
FixtureSegmentationAdapter,
SamSegmentationAdapter,
YoloSegmentationAdapter,
)
class SegmentationService:
@@ -41,6 +47,8 @@ class SegmentationService:
tile_manifest_path: str | None = None,
parameters_json: dict[str, Any] | None = None,
settings: Settings | None = None,
yolo_seg_adapter_class: type[YoloSegmentationAdapter] = YoloSegmentationAdapter,
sam_adapter_class: type[SamSegmentationAdapter] = SamSegmentationAdapter,
) -> SegmentationRunResponse:
parameters = dict(parameters_json or {})
resolved_settings = settings or get_settings()
@@ -58,7 +66,13 @@ class SegmentationService:
status_code=400,
)
model = ModelRegistryService.get_model_capability(model_id, task_type="segmentation")
model = ModelRegistryService.get_model_capability(
model_id,
settings=resolved_settings,
task_type="segmentation",
yolo_seg_adapter_class=yolo_seg_adapter_class,
sam_adapter_class=sam_adapter_class,
)
if model is None:
raise AppError(code="SEGMENTATION_MODEL_NOT_FOUND", message="Segmentation model not found", status_code=404)
if model.model_id == "fixture-segmenter" and parameters.get("fixture_mode") is not True:
@@ -67,6 +81,13 @@ class SegmentationService:
message="Fixture segmenter requires explicit fixture_mode=true",
status_code=400,
)
configured_model_ids = {resolved_settings.yolo_seg_model_id, resolved_settings.sam_model_id}
if model.model_id in configured_model_ids and model.configured and not tile_manifest_path:
raise AppError(
code="SEGMENTATION_TILE_MANIFEST_REQUIRED",
message="Configured segmentation inference requires an existing raster tile manifest path",
status_code=400,
)
run_parameters = {
"model_id": model.model_id,
@@ -100,19 +121,24 @@ class SegmentationService:
)
if model.model_id == "fixture-segmenter":
segmentations = SegmentationService._persist_fixture_segmentations(
db=db,
project_id=project_id,
dataset_id=dataset_id,
analysis_run=analysis_run,
job=job,
model_name=model.model_id,
model_version=model.version,
raw_segmentations=parameters.get("fixture_segmentations"),
confidence_threshold=confidence_threshold,
class_filter=class_filter or [],
settings=resolved_settings,
)
try:
segmentations = SegmentationService._persist_fixture_segmentations(
db=db,
project_id=project_id,
dataset_id=dataset_id,
analysis_run=analysis_run,
job=job,
model_name=model.model_id,
model_version=model.version,
raw_segmentations=parameters.get("fixture_segmentations"),
confidence_threshold=confidence_threshold,
class_filter=class_filter or [],
settings=resolved_settings,
)
except Exception as exc:
# A rejected fixture payload must never leave the run stuck in "running".
SegmentationService._fail_run_after_exception(db, analysis_run, job, exc, fallback_code="SEGMENTATION_INTERNAL_ERROR")
raise
SegmentationService._mark_success(db, analysis_run, job, segmentation_count=len(segmentations))
return SegmentationRunResponse(
analysis_run_id=analysis_run.id,
@@ -125,8 +151,80 @@ class SegmentationService:
message="Fixture segmentations persisted.",
)
if model.model_id in configured_model_ids:
try:
segmentations, postprocess_summary = SegmentationService._run_configured_segmentation(
db=db,
project_id=project_id,
dataset_id=dataset_id,
analysis_run=analysis_run,
job=job,
model_name=model.model_id,
model_version=model.version,
tile_manifest_path=tile_manifest_path,
confidence_threshold=confidence_threshold,
class_filter=class_filter or [],
settings=resolved_settings,
yolo_seg_adapter_class=yolo_seg_adapter_class,
sam_adapter_class=sam_adapter_class,
)
except AppError as exc:
SegmentationService._mark_failed(db, analysis_run, job, code=exc.code, message=exc.message)
return SegmentationRunResponse(
analysis_run_id=analysis_run.id,
job_id=job.id,
project_id=project_id,
dataset_id=dataset_id,
model_id=model.model_id,
status="failed",
segmentation_count=0,
error_code=exc.code,
message=exc.message,
)
except Exception as exc:
# An unexpected inference error must never leave the run stuck in "running".
SegmentationService._fail_run_after_exception(db, analysis_run, job, exc, fallback_code="SEGMENTATION_INTERNAL_ERROR")
raise
SegmentationService._mark_success(
db,
analysis_run,
job,
segmentation_count=len(segmentations),
extra_result=postprocess_summary,
)
return SegmentationRunResponse(
analysis_run_id=analysis_run.id,
job_id=job.id,
project_id=project_id,
dataset_id=dataset_id,
model_id=model.model_id,
status="success",
segmentation_count=len(segmentations),
message="Configured segmentation inference persisted georeferenced masks.",
)
SegmentationService._mark_failed(
db,
analysis_run,
job,
code="SEGMENTATION_MODEL_UNAVAILABLE",
message="Segmentation model is unavailable",
)
raise AppError(code="SEGMENTATION_MODEL_UNAVAILABLE", message="Segmentation model is unavailable", status_code=503)
@staticmethod
def _fail_run_after_exception(db, analysis_run: AnalysisRun, job: Job, exc: Exception, fallback_code: str) -> None:
try:
db.rollback()
except Exception:
pass
code = getattr(exc, "code", None) or fallback_code
message = getattr(exc, "message", None) or "Unexpected internal error during analysis run"
try:
SegmentationService._mark_failed(db, analysis_run, job, code=str(code), message=str(message))
except Exception:
pass
@staticmethod
def get_run(db, analysis_run_id: uuid.UUID) -> SegmentationRunRead:
run = db.get(AnalysisRun, analysis_run_id)
@@ -378,8 +476,10 @@ class SegmentationService:
db.refresh(job)
@staticmethod
def _mark_success(db, analysis_run: AnalysisRun, job: Job, segmentation_count: int) -> None:
def _mark_success(db, analysis_run: AnalysisRun, job: Job, segmentation_count: int, extra_result: dict[str, Any] | None = None) -> None:
result = {"segmentation_count": segmentation_count}
if extra_result:
result.update(extra_result)
analysis_run.status = "success"
analysis_run.finished_at = SegmentationService._now()
analysis_run.result_json = result
@@ -392,6 +492,129 @@ class SegmentationService:
db.refresh(analysis_run)
db.refresh(job)
@staticmethod
def _run_configured_segmentation(
db,
project_id: uuid.UUID,
dataset_id: uuid.UUID,
analysis_run: AnalysisRun,
job: Job,
model_name: str,
model_version: str | None,
tile_manifest_path: str | None,
confidence_threshold: float,
class_filter: list[str],
settings: Settings,
yolo_seg_adapter_class: type[YoloSegmentationAdapter],
sam_adapter_class: type[SamSegmentationAdapter],
) -> tuple[list[Segmentation], dict[str, Any]]:
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles)
if model_name == settings.sam_model_id:
adapter = sam_adapter_class(settings)
model_path = Path(settings.sam_model_path or "").expanduser()
else:
adapter = yolo_seg_adapter_class(settings)
model_path = Path(settings.yolo_seg_model_path or "").expanduser()
model = adapter.load_model(model_path)
allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)}
manifest_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs") or "EPSG:4326"
candidates: list[dict[str, Any]] = []
for tile in manifest["tiles"]:
tile_path = DetectionService._resolve_tile_path(tile, Path(tile_manifest_path or "").expanduser())
for raw in adapter.predict_tile(model, tile_path, confidence_threshold):
model_class_name = str(raw.get("class_name") or "").strip()
class_name = DetectionService._canonical_class_name(model_class_name)
confidence = raw.get("confidence")
confidence = float(confidence) if confidence is not None else None
if allowed_classes and class_name not in allowed_classes:
continue
if confidence is not None and confidence < confidence_threshold:
continue
points = raw.get("points")
if not isinstance(points, list) or len(points) < 3:
continue
geometry = pixel_points_to_epsg4326_polygon(points=points, tile=tile, crs=tile.get("crs") or manifest_crs)
properties = dict(raw.get("properties") or {})
if model_class_name and model_class_name != class_name:
properties.setdefault("model_class_name", model_class_name)
candidates.append(
{
"class_name": class_name,
"confidence": confidence if confidence is not None else 0.0,
"reported_confidence": confidence,
"geometry": geometry,
"bbox": raw.get("bbox"),
"source_tile_path": str(tile_path),
"tile_index": tile.get("index"),
"properties": {**properties, "tile_index": tile.get("index")},
}
)
filtered_candidates = DetectionService._suppress_duplicate_candidates(
candidates,
iou_threshold=float(settings.segmentation_duplicate_iou_threshold),
)
persisted: list[Segmentation] = []
for candidate in filtered_candidates:
geometry = candidate["geometry"]
if isinstance(geometry, Polygon):
geometry = MultiPolygon([geometry])
bbox = candidate.get("bbox")
bbox_json = None
if isinstance(bbox, list) and len(bbox) == 4:
bbox_json = {
"x_min": float(bbox[0]),
"y_min": float(bbox[1]),
"x_max": float(bbox[2]),
"y_max": float(bbox[3]),
}
segmentation = Segmentation(
id=uuid.uuid4(),
project_id=project_id,
dataset_id=dataset_id,
analysis_run_id=analysis_run.id,
job_id=job.id,
model_name=model_name,
model_version=model_version,
class_name=candidate["class_name"],
confidence=candidate["reported_confidence"],
geometry=from_shape(geometry, srid=4326),
bbox_json=bbox_json,
area_m2=SegmentationService._geodesic_area_m2(geometry),
mask_path=None,
source_tile_path=candidate["source_tile_path"],
tile_index=candidate["tile_index"] if isinstance(candidate["tile_index"], int) else None,
properties_json=candidate["properties"],
provenance_json={
"inference": "local",
"model_id": model_name,
"tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()),
"tile_index": candidate["tile_index"],
"device": settings.yolo_device,
},
)
db.add(segmentation)
persisted.append(segmentation)
db.commit()
for segmentation in persisted:
db.refresh(segmentation)
return persisted, {
"raw_segmentation_count": len(candidates),
"suppressed_segmentation_count": len(candidates) - len(filtered_candidates),
"duplicate_iou_threshold": float(settings.segmentation_duplicate_iou_threshold),
"tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()),
}
@staticmethod
def _geodesic_area_m2(geometry: MultiPolygon | Polygon) -> float | None:
try:
from pyproj import Geod
area, _ = Geod(ellps="WGS84").geometry_area_perimeter(geometry)
return abs(float(area))
except Exception:
return None
@staticmethod
def _persist_fixture_segmentations(
db,