feat: add map-driven orthophoto analysis
This commit is contained in:
Codex
@@ -1057,3 +1057,19 @@ PostGIS intersection path.
|
||||
- `bash scripts/backend_test.sh`
|
||||
- `bash scripts/backend_dev.sh`
|
||||
- `bash scripts/smoke_backend_import.sh`
|
||||
|
||||
## Bounded official orthophoto acquisition
|
||||
|
||||
`POST /api/v1/projects/{project_id}/datasets/orthophoto/acquire` accepts an
|
||||
explicit EPSG:4326 map rectangle and stores the official Digitaal Vlaanderen
|
||||
`OMWRGBMRVL`/`Ortho` response as a canonical EPSG:31370 raster Dataset. The
|
||||
default safety envelope is 128-1,024 m per side, 1 m/pixel, 32 MiB and a
|
||||
24-hour exact-request cache. It runs synchronously behind the existing Job
|
||||
abstraction and never during startup.
|
||||
|
||||
Settings: `ORTHOPHOTO_ENABLED`, `ORTHOPHOTO_WMS_URL`,
|
||||
`ORTHOPHOTO_WMS_LAYER`, `ORTHOPHOTO_RESOLUTION_M`,
|
||||
`ORTHOPHOTO_MIN_SIDE_M`, `ORTHOPHOTO_MAX_SIDE_M`,
|
||||
`ORTHOPHOTO_TIMEOUT_SECONDS`, `ORTHOPHOTO_MAX_RESPONSE_MB` and
|
||||
`ORTHOPHOTO_CACHE_TTL_HOURS`. Keep the official HTTPS URL and 1 m profile
|
||||
unless a separately verified deployment/model profile requires a change.
|
||||
|
||||
@@ -21,6 +21,7 @@ from app.schemas import (
|
||||
RasterNdviRequest,
|
||||
RasterNdwiRequest,
|
||||
RasterNdbiRequest,
|
||||
OrthophotoAcquireRequest,
|
||||
VectorBBoxResponse,
|
||||
VectorBufferRequest,
|
||||
VectorClipRequest,
|
||||
@@ -38,6 +39,7 @@ from app.services.raster_operations_service import RasterOperationsService
|
||||
from app.services.vector_operations_service import VectorOperationsService
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
from app.services.dataset_service import DatasetService
|
||||
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
|
||||
from app.utils.response import envelope
|
||||
|
||||
router = APIRouter(prefix="/projects/{project_id}", tags=["datasets"])
|
||||
@@ -125,6 +127,22 @@ async def upload_dataset(
|
||||
return envelope(created.model_dump())
|
||||
|
||||
|
||||
@router.post("/datasets/orthophoto/acquire", response_model=dict)
|
||||
def acquire_bounded_orthophoto(
|
||||
project_id: UUID,
|
||||
payload: OrthophotoAcquireRequest,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
job = JobService.run_sync_job(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
job_type="raster.orthophoto.acquire",
|
||||
parameters=payload.model_dump(mode="json"),
|
||||
operation=lambda: OrthophotoAcquisitionService.acquire(db, project_id, payload),
|
||||
)
|
||||
return envelope(job)
|
||||
|
||||
|
||||
@router.get("/datasets", response_model=dict)
|
||||
def list_datasets(
|
||||
project_id: UUID,
|
||||
|
||||
@@ -19,6 +19,18 @@ class Settings(BaseSettings):
|
||||
)
|
||||
storage_root: str = Field(default="./storage", validation_alias="STORAGE_ROOT")
|
||||
max_upload_mb: int = Field(default=500, validation_alias="MAX_UPLOAD_MB")
|
||||
orthophoto_enabled: bool = Field(default=True, validation_alias="ORTHOPHOTO_ENABLED")
|
||||
orthophoto_wms_url: str = Field(
|
||||
default="https://geo.api.vlaanderen.be/OMWRGBMRVL/wms",
|
||||
validation_alias="ORTHOPHOTO_WMS_URL",
|
||||
)
|
||||
orthophoto_wms_layer: str = Field(default="Ortho", validation_alias="ORTHOPHOTO_WMS_LAYER")
|
||||
orthophoto_resolution_m: float = Field(default=1.0, gt=0, validation_alias="ORTHOPHOTO_RESOLUTION_M")
|
||||
orthophoto_min_side_m: float = Field(default=128.0, gt=0, validation_alias="ORTHOPHOTO_MIN_SIDE_M")
|
||||
orthophoto_max_side_m: float = Field(default=1024.0, gt=0, validation_alias="ORTHOPHOTO_MAX_SIDE_M")
|
||||
orthophoto_timeout_seconds: int = Field(default=120, ge=1, validation_alias="ORTHOPHOTO_TIMEOUT_SECONDS")
|
||||
orthophoto_max_response_mb: int = Field(default=32, ge=1, validation_alias="ORTHOPHOTO_MAX_RESPONSE_MB")
|
||||
orthophoto_cache_ttl_hours: int = Field(default=24, ge=0, validation_alias="ORTHOPHOTO_CACHE_TTL_HOURS")
|
||||
redis_url: str | None = Field(default=None, validation_alias="REDIS_URL")
|
||||
log_level: str = Field(default="INFO", validation_alias="GEOINTEL_LOG_LEVEL")
|
||||
database_statement_timeout_ms: int = Field(default=5_000, validation_alias="DATABASE_STATEMENT_TIMEOUT_MS")
|
||||
|
||||
@@ -31,6 +31,7 @@ from .segmentation import (
|
||||
)
|
||||
from .health import HealthResponse, SystemCapabilities
|
||||
from .job import JobCreate, JobList, JobRead, JobStatus
|
||||
from .orthophoto import OrthophotoAcquireRequest, OrthophotoAcquisitionResult
|
||||
from .external import (
|
||||
ExternalFetchRequest,
|
||||
ExternalFetchResponse,
|
||||
@@ -125,6 +126,8 @@ __all__ = [
|
||||
"JobList",
|
||||
"JobRead",
|
||||
"JobStatus",
|
||||
"OrthophotoAcquireRequest",
|
||||
"OrthophotoAcquisitionResult",
|
||||
"VectorBBoxResponse",
|
||||
"VectorClipRequest",
|
||||
"VectorBufferRequest",
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from .operations import VectorSelectionBBox
|
||||
|
||||
|
||||
class OrthophotoAcquireRequest(BaseModel):
|
||||
bbox: VectorSelectionBBox
|
||||
area_id: UUID | None = None
|
||||
force_refresh: bool = False
|
||||
|
||||
|
||||
class OrthophotoAcquisitionResult(BaseModel):
|
||||
output_dataset_id: UUID
|
||||
reused: bool
|
||||
provider: str
|
||||
layer: str
|
||||
width: int
|
||||
height: int
|
||||
resolution_m: float
|
||||
bbox_epsg4326: list[float]
|
||||
bbox_epsg31370: list[float]
|
||||
attribution: str
|
||||
limitation_message: str
|
||||
@@ -410,6 +410,92 @@ class DatasetService:
|
||||
|
||||
return DatasetService._to_response(dataset)
|
||||
|
||||
@staticmethod
|
||||
def import_raster_bytes(
|
||||
db: Session,
|
||||
*,
|
||||
project_id: UUID,
|
||||
filename: str,
|
||||
content: bytes,
|
||||
source: str,
|
||||
source_name: str,
|
||||
source_metadata: dict[str, Any],
|
||||
provenance_metadata: dict[str, Any],
|
||||
area_id: UUID | None = None,
|
||||
source_version: str | None = None,
|
||||
content_type: str = "image/tiff",
|
||||
) -> DatasetCreateResponse:
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
if area_id is not None:
|
||||
area = db.get(Area, area_id)
|
||||
if not area:
|
||||
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||
if area.project_id != project_id:
|
||||
raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400)
|
||||
if not content:
|
||||
raise AppError(code="INVALID_UPLOAD", message="Raster artifact is empty", status_code=400)
|
||||
safe_filename = DatasetService._validate_upload_filename(filename)
|
||||
if DatasetService._extension_for_path(safe_filename) not in DatasetService.RASTER_EXTENSIONS:
|
||||
raise AppError(code="INVALID_UPLOAD", message="Raster artifacts require a GeoTIFF filename", status_code=415)
|
||||
|
||||
dataset_id = uuid.uuid4()
|
||||
storage_info = StorageService.persist_dataset_file(
|
||||
project_id=str(project_id),
|
||||
dataset_id=str(dataset_id),
|
||||
dataset_type="raster",
|
||||
original_filename=safe_filename,
|
||||
content=content,
|
||||
content_type=content_type,
|
||||
)
|
||||
try:
|
||||
metadata = extract_raster_metadata(storage_info["storage_path"])
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
area_id=area_id,
|
||||
name=safe_filename,
|
||||
dataset_type="raster",
|
||||
source=source,
|
||||
dataset_role="source",
|
||||
source_name=source_name,
|
||||
source_metadata=source_metadata,
|
||||
provenance_metadata=provenance_metadata,
|
||||
imported_at=datetime.now(timezone.utc),
|
||||
source_version=source_version,
|
||||
storage_path=storage_info["storage_path"],
|
||||
original_filename=storage_info["original_filename"],
|
||||
stored_filename=storage_info["stored_filename"],
|
||||
content_type=storage_info["content_type"],
|
||||
size_bytes=storage_info["size_bytes"],
|
||||
checksum_sha256=storage_info["checksum_sha256"],
|
||||
crs=metadata.get("crs"),
|
||||
bounds_json=DatasetService._extract_raster_bounds_json(metadata),
|
||||
resolution_json=DatasetService._extract_raster_resolution_json(metadata),
|
||||
bands_json=DatasetService._extract_raster_bands_json(metadata),
|
||||
metadata_json=metadata,
|
||||
status="ready",
|
||||
)
|
||||
db.add(dataset)
|
||||
db.add(
|
||||
DatasetVersion(
|
||||
dataset_id=dataset.id,
|
||||
version=1,
|
||||
storage_path=dataset.storage_path,
|
||||
source_version=dataset.source_version,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
return DatasetService._to_response(dataset)
|
||||
except Exception:
|
||||
db.rollback()
|
||||
StorageService.remove_dataset_file(storage_info["storage_path"])
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def import_partitioned_vector_artifact(
|
||||
db: Session,
|
||||
|
||||
@@ -0,0 +1,315 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import warnings
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import Request, urlopen
|
||||
from uuid import UUID
|
||||
|
||||
from geoalchemy2.shape import to_shape
|
||||
from pyproj import Transformer
|
||||
from shapely.geometry import box
|
||||
from shapely.ops import transform as shapely_transform
|
||||
|
||||
from app.core.config import Settings, get_settings
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset, Project
|
||||
from app.schemas.orthophoto import OrthophotoAcquireRequest, OrthophotoAcquisitionResult
|
||||
from app.services.dataset_service import DatasetService
|
||||
|
||||
|
||||
class OrthophotoAcquisitionService:
|
||||
PROVIDER = "digitaal_vlaanderen_orthophoto"
|
||||
ATTRIBUTION = "Bron: Orthofotomozaiek Vlaanderen, Digitaal Vlaanderen"
|
||||
CATALOG_URL = "https://www.vlaanderen.be/datavindplaats/catalogus/orthofotomozaiek-middenschalig-winteropnamen-kleur-meest-recent-vlaanderen"
|
||||
LIMITATION = "Meest recente samengestelde winterorthofoto op het moment van de aanvraag; geen historische opnamedatum per pixel."
|
||||
|
||||
@staticmethod
|
||||
def _prepared_request(
|
||||
payload: OrthophotoAcquireRequest,
|
||||
settings: Settings,
|
||||
) -> dict[str, Any]:
|
||||
if payload.bbox.crs.upper() != "EPSG:4326":
|
||||
raise AppError(code="INVALID_CRS", message="Orthophoto selection bbox must use EPSG:4326", status_code=400)
|
||||
min_x = float(payload.bbox.min_x)
|
||||
min_y = float(payload.bbox.min_y)
|
||||
max_x = float(payload.bbox.max_x)
|
||||
max_y = float(payload.bbox.max_y)
|
||||
if not all(math.isfinite(value) for value in (min_x, min_y, max_x, max_y)) or min_x >= max_x or min_y >= max_y:
|
||||
raise AppError(code="INVALID_BBOX", message="Orthophoto selection must be a finite non-empty rectangle", status_code=400)
|
||||
|
||||
transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
||||
lambert_bounds = transformer.transform_bounds(min_x, min_y, max_x, max_y, densify_pts=21)
|
||||
width_m = lambert_bounds[2] - lambert_bounds[0]
|
||||
height_m = lambert_bounds[3] - lambert_bounds[1]
|
||||
if width_m < settings.orthophoto_min_side_m or height_m < settings.orthophoto_min_side_m:
|
||||
raise AppError(
|
||||
code="ORTHOPHOTO_SELECTION_TOO_SMALL",
|
||||
message=f"Select an area of at least {settings.orthophoto_min_side_m:.0f} by {settings.orthophoto_min_side_m:.0f} metres",
|
||||
status_code=422,
|
||||
)
|
||||
if width_m > settings.orthophoto_max_side_m or height_m > settings.orthophoto_max_side_m:
|
||||
raise AppError(
|
||||
code="ORTHOPHOTO_SELECTION_TOO_LARGE",
|
||||
message=f"Select an area no larger than {settings.orthophoto_max_side_m:.0f} by {settings.orthophoto_max_side_m:.0f} metres",
|
||||
details={"width_m": width_m, "height_m": height_m},
|
||||
status_code=422,
|
||||
)
|
||||
|
||||
width = max(1, math.ceil(width_m / settings.orthophoto_resolution_m))
|
||||
height = max(1, math.ceil(height_m / settings.orthophoto_resolution_m))
|
||||
bbox_4326 = [min_x, min_y, max_x, max_y]
|
||||
bbox_31370 = [float(value) for value in lambert_bounds]
|
||||
request_identity = {
|
||||
"provider": OrthophotoAcquisitionService.PROVIDER,
|
||||
"wms_url": settings.orthophoto_wms_url,
|
||||
"layer": settings.orthophoto_wms_layer,
|
||||
"bbox_epsg4326": [round(value, 8) for value in bbox_4326],
|
||||
"bbox_epsg31370": [round(value, 3) for value in bbox_31370],
|
||||
"width": width,
|
||||
"height": height,
|
||||
"resolution_m": settings.orthophoto_resolution_m,
|
||||
}
|
||||
request_hash = hashlib.sha256(json.dumps(request_identity, sort_keys=True).encode("utf-8")).hexdigest()
|
||||
params = {
|
||||
"SERVICE": "WMS",
|
||||
"VERSION": "1.3.0",
|
||||
"REQUEST": "GetMap",
|
||||
"LAYERS": settings.orthophoto_wms_layer,
|
||||
"STYLES": "",
|
||||
"FORMAT": "image/tiff",
|
||||
"CRS": "EPSG:31370",
|
||||
"BBOX": ",".join(f"{value:.3f}" for value in bbox_31370),
|
||||
"WIDTH": str(width),
|
||||
"HEIGHT": str(height),
|
||||
}
|
||||
return {
|
||||
**request_identity,
|
||||
"request_hash": request_hash,
|
||||
"request_url": f"{settings.orthophoto_wms_url}?{urlencode(params)}",
|
||||
"params": params,
|
||||
"bbox_epsg4326": bbox_4326,
|
||||
"bbox_epsg31370": bbox_31370,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _validate_area_scope(db, project_id: UUID, area_id: UUID | None, bbox_epsg4326: list[float]) -> None:
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
if area_id is None:
|
||||
return
|
||||
area = db.get(Area, area_id)
|
||||
if not area:
|
||||
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||
if area.project_id != project_id:
|
||||
raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400)
|
||||
selection = box(*bbox_epsg4326)
|
||||
area_geometry = to_shape(area.geometry)
|
||||
transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
||||
selection_metric = shapely_transform(transformer.transform, selection)
|
||||
area_metric = shapely_transform(transformer.transform, area_geometry)
|
||||
overlap_ratio = area_metric.intersection(selection_metric).area / selection_metric.area
|
||||
if overlap_ratio < 0.99:
|
||||
raise AppError(
|
||||
code="ORTHOPHOTO_SELECTION_OUTSIDE_AREA",
|
||||
message="Keep the orthophoto rectangle inside the selected work area",
|
||||
details={"coverage_ratio": overlap_ratio},
|
||||
status_code=422,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _cached_dataset(db, project_id: UUID, filename: str, settings: Settings) -> Dataset | None:
|
||||
if settings.orthophoto_cache_ttl_hours <= 0:
|
||||
return None
|
||||
candidate = (
|
||||
db.query(Dataset)
|
||||
.filter(
|
||||
Dataset.project_id == project_id,
|
||||
Dataset.name == filename,
|
||||
Dataset.source_name == OrthophotoAcquisitionService.PROVIDER,
|
||||
Dataset.status == "ready",
|
||||
)
|
||||
.order_by(Dataset.imported_at.desc())
|
||||
.first()
|
||||
)
|
||||
if not candidate or not candidate.storage_path or not Path(candidate.storage_path).is_file():
|
||||
return None
|
||||
imported_at = candidate.imported_at
|
||||
if imported_at is None:
|
||||
return None
|
||||
if imported_at.tzinfo is None:
|
||||
imported_at = imported_at.replace(tzinfo=UTC)
|
||||
if datetime.now(UTC) - imported_at > timedelta(hours=settings.orthophoto_cache_ttl_hours):
|
||||
return None
|
||||
return candidate
|
||||
|
||||
@staticmethod
|
||||
def _fetch(request_url: str, settings: Settings, opener: Callable[..., Any] | None = None) -> tuple[bytes, str]:
|
||||
request = Request(request_url, headers={"User-Agent": "GeoIntel/0.1 bounded-orthophoto-acquisition"})
|
||||
open_request = opener or urlopen
|
||||
try:
|
||||
with open_request(request, timeout=settings.orthophoto_timeout_seconds) as response:
|
||||
content_type = str(response.headers.get("Content-Type", ""))
|
||||
content_length = response.headers.get("Content-Length")
|
||||
max_bytes = settings.orthophoto_max_response_mb * 1024 * 1024
|
||||
if content_length and int(content_length) > max_bytes:
|
||||
raise AppError(code="ORTHOPHOTO_RESPONSE_TOO_LARGE", message="Official orthophoto response exceeds the configured size limit", status_code=502)
|
||||
content = response.read(max_bytes + 1)
|
||||
except AppError:
|
||||
raise
|
||||
except (HTTPError, URLError, TimeoutError, OSError) as exc:
|
||||
raise AppError(
|
||||
code="ORTHOPHOTO_PROVIDER_UNAVAILABLE",
|
||||
message="The official orthophoto service could not complete the bounded request",
|
||||
details={"reason": str(exc)},
|
||||
status_code=502,
|
||||
) from exc
|
||||
if len(content) > settings.orthophoto_max_response_mb * 1024 * 1024:
|
||||
raise AppError(code="ORTHOPHOTO_RESPONSE_TOO_LARGE", message="Official orthophoto response exceeds the configured size limit", status_code=502)
|
||||
if "image" not in content_type.lower() and "tiff" not in content_type.lower():
|
||||
preview = content[:300].decode("utf-8", errors="replace")
|
||||
raise AppError(
|
||||
code="ORTHOPHOTO_PROVIDER_INVALID_RESPONSE",
|
||||
message="The official orthophoto service did not return an image",
|
||||
details={"content_type": content_type, "response_preview": preview},
|
||||
status_code=502,
|
||||
)
|
||||
return content, content_type
|
||||
|
||||
@staticmethod
|
||||
def _georeference_tiff(content: bytes, prepared: dict[str, Any]) -> bytes:
|
||||
try:
|
||||
from rasterio.io import MemoryFile
|
||||
from rasterio.errors import NotGeoreferencedWarning
|
||||
from rasterio.transform import from_bounds
|
||||
except ImportError as exc:
|
||||
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio is required for orthophoto acquisition", status_code=503) from exc
|
||||
|
||||
try:
|
||||
with MemoryFile(content) as source_memory:
|
||||
with warnings.catch_warnings():
|
||||
warnings.simplefilter("ignore", NotGeoreferencedWarning)
|
||||
with source_memory.open() as source:
|
||||
if source.width != prepared["width"] or source.height != prepared["height"] or source.count < 3:
|
||||
raise AppError(
|
||||
code="ORTHOPHOTO_PROVIDER_INVALID_RESPONSE",
|
||||
message="Official orthophoto dimensions or RGB bands do not match the bounded request",
|
||||
details={"width": source.width, "height": source.height, "bands": source.count},
|
||||
status_code=502,
|
||||
)
|
||||
image = source.read()
|
||||
profile = source.profile.copy()
|
||||
profile.update(
|
||||
driver="GTiff",
|
||||
crs="EPSG:31370",
|
||||
transform=from_bounds(*prepared["bbox_epsg31370"], source.width, source.height),
|
||||
compress="deflate",
|
||||
tiled=False,
|
||||
)
|
||||
with MemoryFile() as output_memory:
|
||||
with output_memory.open(**profile) as output:
|
||||
output.write(image)
|
||||
output.update_tags(
|
||||
source="Digitaal Vlaanderen OMWRGBMRVL WMS Ortho layer",
|
||||
source_url=prepared["request_url"],
|
||||
attribution=OrthophotoAcquisitionService.ATTRIBUTION,
|
||||
acquisition="explicit_bounded_map_selection",
|
||||
)
|
||||
return output_memory.read()
|
||||
except AppError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise AppError(
|
||||
code="ORTHOPHOTO_PROVIDER_INVALID_RESPONSE",
|
||||
message="The official orthophoto response is not a readable GeoTIFF",
|
||||
details={"reason": str(exc)},
|
||||
status_code=502,
|
||||
) from exc
|
||||
|
||||
@staticmethod
|
||||
def acquire(
|
||||
db,
|
||||
project_id: UUID,
|
||||
payload: OrthophotoAcquireRequest,
|
||||
*,
|
||||
settings: Settings | None = None,
|
||||
opener: Callable[..., Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
resolved_settings = settings or get_settings()
|
||||
if not resolved_settings.orthophoto_enabled:
|
||||
raise AppError(code="ORTHOPHOTO_NOT_CONFIGURED", message="Official orthophoto acquisition is disabled", status_code=503)
|
||||
prepared = OrthophotoAcquisitionService._prepared_request(payload, resolved_settings)
|
||||
OrthophotoAcquisitionService._validate_area_scope(db, project_id, payload.area_id, prepared["bbox_epsg4326"])
|
||||
filename = f"orthofoto_selectie_{prepared['request_hash'][:12]}.tif"
|
||||
|
||||
cached = None if payload.force_refresh else OrthophotoAcquisitionService._cached_dataset(db, project_id, filename, resolved_settings)
|
||||
if cached is not None:
|
||||
return OrthophotoAcquisitionResult(
|
||||
output_dataset_id=cached.id,
|
||||
reused=True,
|
||||
provider=OrthophotoAcquisitionService.PROVIDER,
|
||||
layer=resolved_settings.orthophoto_wms_layer,
|
||||
width=prepared["width"],
|
||||
height=prepared["height"],
|
||||
resolution_m=resolved_settings.orthophoto_resolution_m,
|
||||
bbox_epsg4326=prepared["bbox_epsg4326"],
|
||||
bbox_epsg31370=prepared["bbox_epsg31370"],
|
||||
attribution=OrthophotoAcquisitionService.ATTRIBUTION,
|
||||
limitation_message=OrthophotoAcquisitionService.LIMITATION,
|
||||
).model_dump(mode="json")
|
||||
|
||||
raw_content, response_content_type = OrthophotoAcquisitionService._fetch(prepared["request_url"], resolved_settings, opener)
|
||||
geotiff_content = OrthophotoAcquisitionService._georeference_tiff(raw_content, prepared)
|
||||
acquired_at = datetime.now(UTC)
|
||||
dataset = DatasetService.import_raster_bytes(
|
||||
db,
|
||||
project_id=project_id,
|
||||
area_id=payload.area_id,
|
||||
filename=filename,
|
||||
content=geotiff_content,
|
||||
source="Digitaal Vlaanderen OMWRGBMRVL WMS",
|
||||
source_name=OrthophotoAcquisitionService.PROVIDER,
|
||||
source_version=f"most_recent_at_{acquired_at.date().isoformat()}",
|
||||
content_type="image/tiff",
|
||||
source_metadata={
|
||||
"provider": OrthophotoAcquisitionService.PROVIDER,
|
||||
"service": "WMS",
|
||||
"service_version": "1.3.0",
|
||||
"layer": resolved_settings.orthophoto_wms_layer,
|
||||
"catalog_url": OrthophotoAcquisitionService.CATALOG_URL,
|
||||
"attribution": OrthophotoAcquisitionService.ATTRIBUTION,
|
||||
"license_note": "Gebruik volgens het gebruiksrecht geografische webdiensten van Digitaal Vlaanderen.",
|
||||
},
|
||||
provenance_metadata={
|
||||
"acquisition": "explicit_bounded_map_selection",
|
||||
"acquired_at": acquired_at.isoformat(),
|
||||
"request_hash": prepared["request_hash"],
|
||||
"request_url": prepared["request_url"],
|
||||
"response_content_type": response_content_type,
|
||||
"bbox_epsg4326": prepared["bbox_epsg4326"],
|
||||
"bbox_epsg31370": prepared["bbox_epsg31370"],
|
||||
"width": prepared["width"],
|
||||
"height": prepared["height"],
|
||||
"resolution_m": resolved_settings.orthophoto_resolution_m,
|
||||
"limitation_message": OrthophotoAcquisitionService.LIMITATION,
|
||||
},
|
||||
)
|
||||
return OrthophotoAcquisitionResult(
|
||||
output_dataset_id=dataset.id,
|
||||
reused=False,
|
||||
provider=OrthophotoAcquisitionService.PROVIDER,
|
||||
layer=resolved_settings.orthophoto_wms_layer,
|
||||
width=prepared["width"],
|
||||
height=prepared["height"],
|
||||
resolution_m=resolved_settings.orthophoto_resolution_m,
|
||||
bbox_epsg4326=prepared["bbox_epsg4326"],
|
||||
bbox_epsg31370=prepared["bbox_epsg31370"],
|
||||
attribution=OrthophotoAcquisitionService.ATTRIBUTION,
|
||||
limitation_message=OrthophotoAcquisitionService.LIMITATION,
|
||||
).model_dump(mode="json")
|
||||
@@ -21,7 +21,9 @@ def test_guided_detection_reuses_canonical_raster_and_detection_apis() -> None:
|
||||
assert "tile_size: 512" in hook
|
||||
assert "overlap: 64" in hook
|
||||
assert "detectionApi.getYoloPreflight" in hook
|
||||
assert "await executeDetection(selectedProjectId, datasetId, manifestPath)" in hook
|
||||
assert "const result = await executeDetection(" in hook
|
||||
assert "effectiveModelId" in hook
|
||||
assert "effectiveModelAssetId" in hook
|
||||
assert "await loadDetectionResults(result.analysis_run_id)" in hook
|
||||
assert "model_id: selectedDetectionModelId" in hook
|
||||
assert "model_asset_id: selectedModelAssetId || null" in hook
|
||||
|
||||
@@ -0,0 +1,333 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import rasterio
|
||||
from fastapi.testclient import TestClient
|
||||
from geoalchemy2.shape import from_shape
|
||||
from pyproj import Transformer
|
||||
from rasterio.io import MemoryFile
|
||||
from rasterio.transform import from_origin
|
||||
from shapely.geometry import MultiPolygon, box
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import AppError
|
||||
from app.db.session import get_db
|
||||
from app.main import app
|
||||
from app.models import Area, Dataset, DatasetVersion, Job, Project
|
||||
from app.schemas.orthophoto import OrthophotoAcquireRequest
|
||||
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, rows: dict[tuple[type, object], object] | None = None, query_result=None):
|
||||
self.rows = rows or {}
|
||||
self.query_result = query_result
|
||||
self.added: list[object] = []
|
||||
|
||||
def get(self, model, row_id):
|
||||
row = self.rows.get((model, row_id))
|
||||
if row is not None:
|
||||
return row
|
||||
return next((item for item in self.added if isinstance(item, model) and item.id == row_id), None)
|
||||
|
||||
def add(self, row):
|
||||
self.added.append(row)
|
||||
|
||||
def commit(self):
|
||||
return None
|
||||
|
||||
def rollback(self):
|
||||
return None
|
||||
|
||||
def refresh(self, row):
|
||||
return row
|
||||
|
||||
def query(self, _model):
|
||||
return FakeQuery(self.query_result)
|
||||
|
||||
|
||||
class FakeQuery:
|
||||
def __init__(self, result):
|
||||
self.result = result
|
||||
|
||||
def filter(self, *_args):
|
||||
return self
|
||||
|
||||
def order_by(self, *_args):
|
||||
return self
|
||||
|
||||
def first(self):
|
||||
return self.result
|
||||
|
||||
|
||||
class FakeImageResponse:
|
||||
def __init__(self, content: bytes):
|
||||
self.content = content
|
||||
self.headers = {
|
||||
"Content-Type": "image/tiff",
|
||||
"Content-Length": str(len(content)),
|
||||
}
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return None
|
||||
|
||||
def read(self, limit: int) -> bytes:
|
||||
return self.content[:limit]
|
||||
|
||||
|
||||
def _selection_payload(*, side_m: float = 512.0, force_refresh: bool = True, area_id=None) -> OrthophotoAcquireRequest:
|
||||
west, south = 199_000.0, 210_000.0
|
||||
transformer = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
|
||||
min_lon, min_lat = transformer.transform(west, south)
|
||||
max_lon, max_lat = transformer.transform(west + side_m, south + side_m)
|
||||
return OrthophotoAcquireRequest(
|
||||
bbox={
|
||||
"min_x": min_lon,
|
||||
"min_y": min_lat,
|
||||
"max_x": max_lon,
|
||||
"max_y": max_lat,
|
||||
"crs": "EPSG:4326",
|
||||
},
|
||||
area_id=area_id,
|
||||
force_refresh=force_refresh,
|
||||
)
|
||||
|
||||
|
||||
def _source_tiff(width: int, height: int) -> bytes:
|
||||
pixels = np.zeros((3, height, width), dtype=np.uint8)
|
||||
pixels[0, :, :] = 92
|
||||
pixels[1, :, :] = 126
|
||||
pixels[2, :, :] = 84
|
||||
with MemoryFile() as memory:
|
||||
with memory.open(
|
||||
driver="GTiff",
|
||||
width=width,
|
||||
height=height,
|
||||
count=3,
|
||||
dtype="uint8",
|
||||
transform=from_origin(0, height, 1, 1),
|
||||
) as output:
|
||||
output.write(pixels)
|
||||
return memory.read()
|
||||
|
||||
|
||||
def test_orthophoto_request_is_bounded_and_uses_official_wms_contract() -> None:
|
||||
settings = Settings(_env_file=None)
|
||||
prepared = OrthophotoAcquisitionService._prepared_request(_selection_payload(), settings)
|
||||
|
||||
# A north-up WGS84 rectangle becomes slightly wider after the bounded
|
||||
# EPSG:31370 transform; the service must still keep it near the requested scale.
|
||||
assert 500 <= prepared["width"] <= 540
|
||||
assert 500 <= prepared["height"] <= 540
|
||||
assert prepared["params"]["CRS"] == "EPSG:31370"
|
||||
assert prepared["params"]["LAYERS"] == "Ortho"
|
||||
assert "geo.api.vlaanderen.be/OMWRGBMRVL/wms" in prepared["request_url"]
|
||||
assert len(prepared["request_hash"]) == 64
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("side_m", "expected_code"),
|
||||
[(64.0, "ORTHOPHOTO_SELECTION_TOO_SMALL"), (1_200.0, "ORTHOPHOTO_SELECTION_TOO_LARGE")],
|
||||
)
|
||||
def test_orthophoto_request_rejects_unsafe_selection_sizes(side_m: float, expected_code: str) -> None:
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
OrthophotoAcquisitionService._prepared_request(_selection_payload(side_m=side_m), Settings(_env_file=None))
|
||||
|
||||
assert exc_info.value.code == expected_code
|
||||
|
||||
|
||||
def test_orthophoto_acquisition_persists_georeferenced_raster_and_provenance(tmp_path) -> None:
|
||||
project_id = uuid4()
|
||||
area_id = uuid4()
|
||||
payload = _selection_payload(area_id=area_id)
|
||||
area_geometry = MultiPolygon(
|
||||
[
|
||||
box(
|
||||
payload.bbox.min_x - 0.01,
|
||||
payload.bbox.min_y - 0.01,
|
||||
payload.bbox.max_x + 0.01,
|
||||
payload.bbox.max_y + 0.01,
|
||||
)
|
||||
]
|
||||
)
|
||||
db = FakeSession(
|
||||
{
|
||||
(Project, project_id): Project(id=project_id, name="Mol operationele werkruimte"),
|
||||
(Area, area_id): Area(
|
||||
id=area_id,
|
||||
project_id=project_id,
|
||||
name="Gemeente Mol",
|
||||
geometry=from_shape(area_geometry, srid=4326),
|
||||
),
|
||||
}
|
||||
)
|
||||
settings = Settings(_env_file=None, storage_root=str(tmp_path), orthophoto_resolution_m=1.0)
|
||||
prepared = OrthophotoAcquisitionService._prepared_request(payload, settings)
|
||||
response = FakeImageResponse(_source_tiff(prepared["width"], prepared["height"]))
|
||||
|
||||
result = OrthophotoAcquisitionService.acquire(
|
||||
db,
|
||||
project_id,
|
||||
payload,
|
||||
settings=settings,
|
||||
opener=lambda *_args, **_kwargs: response,
|
||||
)
|
||||
|
||||
datasets = [row for row in db.added if isinstance(row, Dataset)]
|
||||
versions = [row for row in db.added if isinstance(row, DatasetVersion)]
|
||||
assert len(datasets) == 1
|
||||
assert len(versions) == 1
|
||||
dataset = datasets[0]
|
||||
assert result["output_dataset_id"] == str(dataset.id)
|
||||
assert result["reused"] is False
|
||||
assert dataset.project_id == project_id
|
||||
assert dataset.area_id == area_id
|
||||
assert dataset.dataset_type == "raster"
|
||||
assert dataset.dataset_role == "source"
|
||||
assert dataset.source_name == "digitaal_vlaanderen_orthophoto"
|
||||
assert dataset.crs == "EPSG:31370"
|
||||
assert dataset.provenance_metadata["acquisition"] == "explicit_bounded_map_selection"
|
||||
assert dataset.provenance_metadata["request_hash"] == prepared["request_hash"]
|
||||
assert dataset.source_metadata["attribution"].startswith("Bron: Orthofotomozaiek Vlaanderen")
|
||||
assert dataset.storage_path is not None
|
||||
with rasterio.open(dataset.storage_path) as stored:
|
||||
assert stored.crs.to_epsg() == 31370
|
||||
assert stored.count == 3
|
||||
assert stored.width == prepared["width"]
|
||||
assert stored.height == prepared["height"]
|
||||
assert list(stored.bounds) == pytest.approx(prepared["bbox_epsg31370"], abs=0.01)
|
||||
|
||||
|
||||
def test_orthophoto_acquisition_rejects_selection_outside_persisted_area() -> None:
|
||||
project_id = uuid4()
|
||||
area_id = uuid4()
|
||||
payload = _selection_payload(area_id=area_id)
|
||||
db = FakeSession(
|
||||
{
|
||||
(Project, project_id): Project(id=project_id, name="Mol"),
|
||||
(Area, area_id): Area(
|
||||
id=area_id,
|
||||
project_id=project_id,
|
||||
name="Unrelated area",
|
||||
geometry=from_shape(MultiPolygon([box(3.0, 50.0, 3.1, 50.1)]), srid=4326),
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
OrthophotoAcquisitionService.acquire(db, project_id, payload, settings=Settings(_env_file=None))
|
||||
|
||||
assert exc_info.value.code == "ORTHOPHOTO_SELECTION_OUTSIDE_AREA"
|
||||
|
||||
|
||||
def test_orthophoto_acquisition_reuses_fresh_exact_request_without_provider_call(tmp_path) -> None:
|
||||
project_id = uuid4()
|
||||
payload = _selection_payload(force_refresh=False)
|
||||
prepared = OrthophotoAcquisitionService._prepared_request(payload, Settings(_env_file=None))
|
||||
stored_path = tmp_path / "cached.tif"
|
||||
stored_path.write_bytes(b"persisted")
|
||||
cached = Dataset(
|
||||
id=uuid4(),
|
||||
project_id=project_id,
|
||||
name=f"orthofoto_selectie_{prepared['request_hash'][:12]}.tif",
|
||||
dataset_type="raster",
|
||||
source="Digitaal Vlaanderen",
|
||||
source_name="digitaal_vlaanderen_orthophoto",
|
||||
status="ready",
|
||||
storage_path=str(stored_path),
|
||||
imported_at=datetime.now(UTC),
|
||||
)
|
||||
db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")}, query_result=cached)
|
||||
|
||||
result = OrthophotoAcquisitionService.acquire(
|
||||
db,
|
||||
project_id,
|
||||
payload,
|
||||
settings=Settings(_env_file=None),
|
||||
opener=lambda *_args, **_kwargs: pytest.fail("fresh cached request must not call the provider"),
|
||||
)
|
||||
|
||||
assert result["output_dataset_id"] == str(cached.id)
|
||||
assert result["reused"] is True
|
||||
assert db.added == []
|
||||
|
||||
|
||||
def test_orthophoto_provider_rejects_non_image_response() -> None:
|
||||
response = FakeImageResponse(b"<ServiceException>invalid layer</ServiceException>")
|
||||
response.headers["Content-Type"] = "text/xml"
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
OrthophotoAcquisitionService._fetch(
|
||||
"https://geo.api.vlaanderen.be/OMWRGBMRVL/wms",
|
||||
Settings(_env_file=None),
|
||||
opener=lambda *_args, **_kwargs: response,
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "ORTHOPHOTO_PROVIDER_INVALID_RESPONSE"
|
||||
|
||||
|
||||
def test_orthophoto_endpoint_returns_canonical_job_envelope(monkeypatch) -> None:
|
||||
project_id = uuid4()
|
||||
output_dataset_id = uuid4()
|
||||
db = FakeSession()
|
||||
monkeypatch.setattr(
|
||||
OrthophotoAcquisitionService,
|
||||
"acquire",
|
||||
lambda *_args, **_kwargs: {
|
||||
"output_dataset_id": str(output_dataset_id),
|
||||
"reused": False,
|
||||
"provider": "digitaal_vlaanderen_orthophoto",
|
||||
},
|
||||
)
|
||||
payload = _selection_payload(force_refresh=False).model_dump(mode="json")
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
try:
|
||||
response = TestClient(app).post(f"/api/v1/projects/{project_id}/datasets/orthophoto/acquire", json=payload)
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
assert response.status_code == 200
|
||||
body = response.json()
|
||||
assert set(body) == {"data"}
|
||||
assert body["data"]["status"] == "success"
|
||||
assert body["data"]["job_type"] == "raster.orthophoto.acquire"
|
||||
assert body["data"]["output_dataset_id"] == str(output_dataset_id)
|
||||
assert body["data"]["result_json"]["provider"] == "digitaal_vlaanderen_orthophoto"
|
||||
assert any(isinstance(row, Job) for row in db.added)
|
||||
|
||||
|
||||
def test_frontend_connects_map_selection_to_existing_detection_and_qa_flows() -> None:
|
||||
app_source = (ROOT / "frontend" / "src" / "App.tsx").read_text(encoding="utf-8")
|
||||
hook_source = (ROOT / "frontend" / "src" / "hooks" / "useMapOrthophotoAnalysis.ts").read_text(encoding="utf-8")
|
||||
map_source = (ROOT / "frontend" / "src" / "components" / "map" / "MapWorkspace.tsx").read_text(encoding="utf-8")
|
||||
|
||||
assert "useMapOrthophotoAnalysis" in app_source
|
||||
assert "onRunOrthophotoAnalysis={mapOrthophotoAnalysis.run}" in app_source
|
||||
assert "datasetsApi.acquireOrthophoto" in hook_source
|
||||
assert "prepareAndRunDetection(datasetId)" in hook_source
|
||||
assert "compareDetectionRunWithReference" in hook_source
|
||||
assert "compareDetectionRunWithReference(analysisRunId, referenceDatasetId, false)" in app_source
|
||||
assert "Herken gebouwen" in map_source
|
||||
assert "Officieel luchtbeeld, lokaal AI-model" in map_source
|
||||
|
||||
|
||||
def test_unraid_runtime_exposes_bounded_orthophoto_settings() -> None:
|
||||
compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8")
|
||||
runner = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
|
||||
template = (ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml").read_text(encoding="utf-8")
|
||||
|
||||
for name in ("ORTHOPHOTO_ENABLED", "ORTHOPHOTO_WMS_URL", "ORTHOPHOTO_RESOLUTION_M", "ORTHOPHOTO_MAX_SIDE_M"):
|
||||
assert name in compose
|
||||
assert name in runner
|
||||
assert name in template
|
||||
Reference in New Issue
Block a user