Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(platform): govern geospatial analysis and raster handoffs

Browse Source
This commit is contained in:
Jens
2026-08-30 06:00:15 +02:00
parent 96f90373dc
commit 80a2d1654d
63 changed files with 2335 additions and 312 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/api/routes/analysis.py
+5 -1
View File
@@ -1,8 +1,9 @@
from __future__ import annotations
from fastapi import APIRouter, Depends
from fastapi import APIRouter, Depends, Request
from sqlalchemy.orm import Session
from app.api.guest_scope import assert_guest_project_scope
from app.core.errors import AppError
from app.db.session import get_db
from app.models import Dataset
@@ -18,12 +19,15 @@ router = APIRouter(prefix="/analysis", tags=["analysis"])
@router.post("/change-detection", response_model=Envelope[JobRead])
def run_change_detection(
payload: ChangeDetectionRequest,
request: Request,
db: Session = Depends(get_db),
) -> dict:
source_dataset = db.get(Dataset, payload.source_dataset_id)
if not source_dataset:
raise AppError(code="DATASET_NOT_FOUND", message="Source dataset not found", status_code=404)
assert_guest_project_scope(request, source_dataset.project_id)
ChangeDetectionService._get_project_vector_dataset(db, payload.source_dataset_id, source_dataset.project_id, "Source")
ChangeDetectionService._get_project_vector_dataset(db, payload.target_dataset_id, source_dataset.project_id, "Target")
job = JobService.run_sync_job(
db=db,
project_id=source_dataset.project_id,
backend/app/api/routes/datasets.py
+7 -3
View File
@@ -5,13 +5,13 @@ from datetime import datetime
from typing import Any
from uuid import UUID
from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Response
from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, Response
from fastapi import UploadFile
from sqlalchemy.orm import Session
from app.models import Area, Project
from app.core.config import get_settings
from app.core.errors import AppError
from app.db.session import get_db
from app.models import Area, Project
from app.schemas import (
BathymetryPartitionFinalizationResult,
BathymetrySourceProbeRead,
@@ -1172,11 +1172,14 @@ def raster_tile_dataset(
project_id: UUID,
dataset_id: UUID,
payload: RasterTileRequest,
request: Request,
db: Session = Depends(get_db),
):
dataset = DatasetService.get_dataset(db, dataset_id)
if dataset.project_id != project_id:
raise HTTPException(status_code=404, detail="Dataset not found")
principal = getattr(request.state, "auth_principal", None)
guest_max_tiles = get_settings().yolo_max_tiles if getattr(principal, "role", None) == "guest" else None
job = _run_job_sync(
db=db,
project_id=project_id,
@@ -1189,6 +1192,7 @@ def raster_tile_dataset(
tile_size=payload.tile_size,
overlap=payload.overlap,
output_name=payload.output_name,
max_tiles=guest_max_tiles,
),
)
return envelope(job)
backend/app/models.py
+1 -1
View File
@@ -1 +1 @@
from app.models import *
from app.models import * # noqa: F403 - legacy compatibility shim re-exports the package API
backend/app/schemas/area.py
+1
View File
@@ -14,6 +14,7 @@ class AreaCreate(BaseModel):
class AreaUpdate(BaseModel):
name: str | None = None
geometry: dict | None = None
crs: str | None = None
backend/app/schemas/detection.py
+4
View File
@@ -42,6 +42,10 @@ class ModelAssetRead(BaseModel):
size_bytes: int
sha256: str
active: bool
runtime_available: bool
runtime_status: str
governed_validation_status: str
promotion_status: str
status: str
limitation_message: str
will_download_models: bool = False
backend/app/services/aoi_operation_executor.py
+188 -40
View File
@@ -12,7 +12,10 @@ from app.schemas.spw_terrain import SpwTerrainAcquireRequest
from app.schemas.official_vector import OfficialVectorAcquireRequest
from app.schemas.flood_hazard import FloodHazardAcquireRequest
from app.schemas.thematic_raster import ThematicRasterAcquireRequest
from app.schemas.bathymetry import BathymetryProfileAcquireRequest, MdkBathymetryAcquireRequest
from app.schemas.bathymetry import (
BathymetryProfileAcquireRequest,
MdkBathymetryAcquireRequest,
)
from app.schemas.job import JobCreate
from app.schemas.operations import VectorSelectionBBox
from app.schemas.orthophoto import OrthophotoAcquireRequest
@@ -20,12 +23,20 @@ from app.services.aoi_operation_service import AoiOperationService
from app.services.grb_acquisition_service import GrbAcquisitionService
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
from app.services.spw_terrain_service import SpwTerrainService
from app.services.official_vector_acquisition_service import OfficialVectorAcquisitionService
from app.services.official_vector_acquisition_service import (
OfficialVectorAcquisitionService,
)
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
from app.services.thematic_raster_acquisition_service import (
ThematicRasterAcquisitionService,
)
from app.services.walous_land_cover_service import WalousLandCoverService
from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
from app.services.mdk_bathymetry_acquisition_service import MdkBathymetryAcquisitionService
from app.services.bathymetry_profile_acquisition_service import (
BathymetryProfileAcquisitionService,
)
from app.services.mdk_bathymetry_acquisition_service import (
MdkBathymetryAcquisitionService,
)
from app.services.job_service import JobService
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
@@ -40,65 +51,202 @@ class AoiOperationExecutor:
AoiOperationService._refresh_parent(db, operation_id)
return AoiOperationService.read(db, project_id, operation_id)
operation = db.get(AoiOperation, operation_id)
child = JobService.create_job(db, JobCreate(
job_type=f"aoi.{operation.operation_type}.partition",
project_id=project_id,
parameters_json={
"aoi_operation_id": str(operation_id),
"partition_id": str(partition.id),
"partition_key": partition.partition_key,
"provider_key": partition.provider_key,
"product_key": partition.product_key,
},
))
child = JobService.create_job(
db,
JobCreate(
job_type=f"aoi.{operation.operation_type}.partition",
project_id=project_id,
parameters_json={
"aoi_operation_id": str(operation_id),
"partition_id": str(partition.id),
"partition_key": partition.partition_key,
"provider_key": partition.provider_key,
"product_key": partition.product_key,
},
),
)
partition = db.get(AoiOperationPartition, partition.id)
partition.child_job_id = child.id
db.add(partition); db.commit()
db.add(partition)
db.commit()
JobService.mark_running(db, child.id)
try:
result = AoiOperationExecutor._dispatch(db, project_id, operation, partition)
output_id = result.get("output_dataset_id") if isinstance(result, dict) else None
JobService.mark_success(db, child.id, result=result, output_dataset_id=UUID(str(output_id)) if output_id else None)
return AoiOperationService.complete(db, project_id, operation_id, partition.id, result)
result = AoiOperationExecutor._dispatch(
db, project_id, operation, partition
)
output_id = (
result.get("output_dataset_id") if isinstance(result, dict) else None
)
JobService.mark_success(
db,
child.id,
result=result,
output_dataset_id=UUID(str(output_id)) if output_id else None,
)
return AoiOperationService.complete(
db, project_id, operation_id, partition.id, result
)
except AppError as exc:
JobService.mark_failed(db, child.id, exc.message, {"code": exc.code, "details": exc.details})
return AoiOperationService.fail(db, project_id, operation_id, partition.id, exc.message, AoiOperationExecutor._retryable(exc), {"code": exc.code, "details": exc.details})
JobService.mark_failed(
db, child.id, exc.message, {"code": exc.code, "details": exc.details}
)
return AoiOperationService.fail(
db,
project_id,
operation_id,
partition.id,
exc.message,
AoiOperationExecutor._retryable(exc),
{"code": exc.code, "details": exc.details},
)
except Exception:
try:
db.rollback()
JobService.mark_failed(db, child.id, "Unexpected partition execution error", {"code": "AOI_PARTITION_INTERNAL_ERROR"})
JobService.mark_failed(
db,
child.id,
"Unexpected partition execution error",
{"code": "AOI_PARTITION_INTERNAL_ERROR"},
)
finally:
AoiOperationService.fail(db, project_id, operation_id, partition.id, "Unexpected partition execution error", True, {"code": "AOI_PARTITION_INTERNAL_ERROR"})
AoiOperationService.fail(
db,
project_id,
operation_id,
partition.id,
"Unexpected partition execution error",
True,
{"code": "AOI_PARTITION_INTERNAL_ERROR"},
)
raise
@staticmethod
def _dispatch(db, project_id: UUID, operation: AoiOperation, partition: AoiOperationPartition) -> dict:
def _dispatch(
db, project_id: UUID, operation: AoiOperation, partition: AoiOperationPartition
) -> dict:
geometry = to_shape(partition.geometry)
min_x, min_y, max_x, max_y = geometry.bounds
bbox = VectorSelectionBBox(min_x=min_x, min_y=min_y, max_x=max_x, max_y=max_y, crs="EPSG:4326")
force_refresh = bool((operation.request_json or {}).get("parameters_json", {}).get("force_refresh", False))
bbox = VectorSelectionBBox(
min_x=min_x, min_y=min_y, max_x=max_x, max_y=max_y, crs="EPSG:4326"
)
force_refresh = bool(
(operation.request_json or {})
.get("parameters_json", {})
.get("force_refresh", False)
)
if partition.provider_key == "grb":
return GrbAcquisitionService.acquire(db, project_id, GrbAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
return GrbAcquisitionService.acquire(
db,
project_id,
GrbAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "orthophoto":
return OrthophotoAcquisitionService.acquire(db, project_id, OrthophotoAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
return OrthophotoAcquisitionService.acquire(
db,
project_id,
OrthophotoAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "dhmv":
return DhmvAcquisitionService.acquire(db, project_id, DhmvAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
return DhmvAcquisitionService.acquire(
db,
project_id,
DhmvAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "spw_terrain":
return SpwTerrainService.acquire(db, project_id, SpwTerrainAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
return SpwTerrainService.acquire(
db,
project_id,
SpwTerrainAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "official_vector":
return OfficialVectorAcquisitionService.acquire(db, project_id, OfficialVectorAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
return OfficialVectorAcquisitionService.acquire(
db,
project_id,
OfficialVectorAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "flood_hazard":
return FloodHazardAcquisitionService.acquire(db, project_id, FloodHazardAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
return FloodHazardAcquisitionService.acquire(
db,
project_id,
FloodHazardAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "thematic_raster":
return ThematicRasterAcquisitionService.acquire(db, project_id, ThematicRasterAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
return ThematicRasterAcquisitionService.acquire(
db,
project_id,
ThematicRasterAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "walous":
return WalousLandCoverService.acquire(db, project_id, ThematicRasterAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
return WalousLandCoverService.acquire(
db,
project_id,
ThematicRasterAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "bathymetry_profiles":
return BathymetryProfileAcquisitionService.acquire(db, project_id, BathymetryProfileAcquireRequest(bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh))
return BathymetryProfileAcquisitionService.acquire(
db,
project_id,
BathymetryProfileAcquireRequest(
bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh
),
)
if partition.provider_key == "mdk_bathymetry":
return MdkBathymetryAcquisitionService.acquire(db, project_id, MdkBathymetryAcquireRequest(bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh))
raise AppError(code="AOI_PROVIDER_UNSUPPORTED", message="No governed AOI executor is registered for this provider", details={"provider_key": partition.provider_key}, status_code=422)
return MdkBathymetryAcquisitionService.acquire(
db,
project_id,
MdkBathymetryAcquireRequest(
bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh
),
)
raise AppError(
code="AOI_PROVIDER_UNSUPPORTED",
message="No governed AOI executor is registered for this provider",
details={"provider_key": partition.provider_key},
status_code=422,
)
@staticmethod
def _retryable(error: AppError) -> bool:
return error.status_code >= 500 or error.code.endswith(("TIMEOUT", "UNAVAILABLE", "TLS_ERROR"))
return error.status_code >= 500 or error.code.endswith(
("TIMEOUT", "UNAVAILABLE", "TLS_ERROR")
)
backend/app/services/aoi_operation_service.py
+273 -68
View File
@@ -22,8 +22,11 @@ class AoiOperationService:
_to_metric = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
_to_wgs84 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
SCOPE_AREA_NAMES = {
"belgium": "Belgium land", "flanders": "Flanders", "wallonia": "Wallonia",
"brussels": "Brussels-Capital Region", "belgian_north_sea": "Belgian part of the North Sea",
"belgium": "Belgium land",
"flanders": "Flanders",
"wallonia": "Wallonia",
"brussels": "Brussels-Capital Region",
"belgian_north_sea": "Belgian part of the North Sea",
"territorial_sea": "Belgian territorial sea (0-12 nautical miles)",
"exclusive_economic_zone": "Belgian exclusive economic zone beyond territorial sea",
"continental_shelf": "Belgian continental shelf beyond territorial sea",
@@ -32,13 +35,19 @@ class AoiOperationService:
@staticmethod
def create(db, project_id: UUID, payload: AoiOperationCreate) -> dict:
if db.get(Project, project_id) is None:
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
raise AppError(
code="PROJECT_NOT_FOUND", message="Project not found", status_code=404
)
geometry = AoiOperationService._resolve_geometry(db, project_id, payload)
if payload.coverage_zone:
geometry = AoiOperationService._clip_to_zone(db, project_id, geometry, payload.coverage_zone)
geometry = AoiOperationService._clip_to_zone(
db, project_id, geometry, payload.coverage_zone
)
geometry = AoiOperationService._as_multipolygon(geometry)
metric_geometry = transform(AoiOperationService._to_metric.transform, geometry)
partition_side_m = AoiOperationService._partition_side(payload.provider_key, payload.max_partition_side_m)
partition_side_m = AoiOperationService._partition_side(
payload.provider_key, payload.max_partition_side_m
)
cells = AoiOperationService._partition(metric_geometry, partition_side_m)
operation_id = uuid4()
now = datetime.now(timezone.utc)
@@ -53,7 +62,9 @@ class AoiOperationService:
plan_json={
"partition_strategy": "epsg31370_square_grid_intersection_v1",
"max_partition_side_m": partition_side_m,
"budget_source": "governed_provider_registry" if payload.max_partition_side_m is None else "stricter_operator_override",
"budget_source": "governed_provider_registry"
if payload.max_partition_side_m is None
else "stricter_operator_override",
"partition_count": len(cells),
"provider_key": payload.provider_key,
"product_key": payload.product_key,
@@ -65,13 +76,21 @@ class AoiOperationService:
wgs84 = transform(AoiOperationService._to_wgs84.transform, cell)
wgs84 = AoiOperationService._as_multipolygon(wgs84)
digest = sha256(wgs84.wkb).hexdigest()[:20]
db.add(AoiOperationPartition(
id=uuid4(), operation_id=operation_id,
partition_key=f"{payload.provider_key}:{payload.product_key}:{ordinal:05d}:{digest}",
provider_key=payload.provider_key, product_key=payload.product_key,
ordinal=ordinal, status="queued", geometry=from_shape(wgs84, srid=4326),
attempt_count=0, max_attempts=payload.max_attempts, created_at=now,
))
db.add(
AoiOperationPartition(
id=uuid4(),
operation_id=operation_id,
partition_key=f"{payload.provider_key}:{payload.product_key}:{ordinal:05d}:{digest}",
provider_key=payload.provider_key,
product_key=payload.product_key,
ordinal=ordinal,
status="queued",
geometry=from_shape(wgs84, srid=4326),
attempt_count=0,
max_attempts=payload.max_attempts,
created_at=now,
)
)
db.commit()
return AoiOperationService.read(db, project_id, operation_id)
@@ -79,13 +98,32 @@ class AoiOperationService:
def _clip_to_zone(db, project_id: UUID, geometry, zone: str):
area_name = AoiOperationService.SCOPE_AREA_NAMES.get(zone)
if area_name is None:
raise AppError(code="AOI_COVERAGE_ZONE_UNSUPPORTED", message="Unknown governed coverage zone", details={"coverage_zone": zone}, status_code=422)
scope = db.query(Area).filter(Area.project_id == project_id, Area.name == area_name).first()
raise AppError(
code="AOI_COVERAGE_ZONE_UNSUPPORTED",
message="Unknown governed coverage zone",
details={"coverage_zone": zone},
status_code=422,
)
scope = (
db.query(Area)
.filter(Area.project_id == project_id, Area.name == area_name)
.first()
)
if scope is None:
raise AppError(code="AOI_COVERAGE_ZONE_NOT_MATERIALIZED", message="The governed coverage-zone geometry is not persisted in this project", details={"coverage_zone": zone}, status_code=409)
raise AppError(
code="AOI_COVERAGE_ZONE_NOT_MATERIALIZED",
message="The governed coverage-zone geometry is not persisted in this project",
details={"coverage_zone": zone},
status_code=409,
)
clipped = geometry.intersection(to_shape(scope.geometry))
if clipped.is_empty:
raise AppError(code="AOI_OUTSIDE_PROVIDER_ZONE", message="The AOI does not intersect the provider coverage zone", details={"coverage_zone": zone}, status_code=422)
raise AppError(
code="AOI_OUTSIDE_PROVIDER_ZONE",
message="The AOI does not intersect the provider coverage zone",
details={"coverage_zone": zone},
status_code=422,
)
return clipped
@staticmethod
@@ -94,19 +132,30 @@ class AoiOperationService:
return MultiPolygon([geometry])
if isinstance(geometry, MultiPolygon):
return geometry
polygons = [part for part in getattr(geometry, "geoms", []) if isinstance(part, Polygon)]
polygons = [
part for part in getattr(geometry, "geoms", []) if isinstance(part, Polygon)
]
if not polygons:
raise AppError(code="AOI_GEOMETRY_EMPTY", message="AOI contains no polygonal area after clipping", status_code=422)
raise AppError(
code="AOI_GEOMETRY_EMPTY",
message="AOI contains no polygonal area after clipping",
status_code=422,
)
return MultiPolygon(polygons)
@staticmethod
def _partition_side(provider_key: str, requested: float | None) -> float:
settings = get_settings()
def raster_side(max_side_m: float, max_pixels: int, resolution_m: float) -> float:
def raster_side(
max_side_m: float, max_pixels: int, resolution_m: float
) -> float:
# Keep every square grid cell within both the provider's spatial
# extent limit and its decoded-pixel budget. The small safety
# margin absorbs ceil/edge rounding in the acquisition services.
pixel_limited_side = math.sqrt(float(max_pixels)) * float(resolution_m) * 0.99
pixel_limited_side = (
math.sqrt(float(max_pixels)) * float(resolution_m) * 0.99
)
return min(float(max_side_m), pixel_limited_side)
budgets = {
@@ -142,22 +191,37 @@ class AoiOperationService:
"mdk_bathymetry": 20_000.0,
}
if provider_key not in budgets:
raise AppError(code="AOI_PROVIDER_UNSUPPORTED", message="No governed partition budget is registered for this provider", details={"provider_key": provider_key}, status_code=422)
raise AppError(
code="AOI_PROVIDER_UNSUPPORTED",
message="No governed partition budget is registered for this provider",
details={"provider_key": provider_key},
status_code=422,
)
governed = budgets[provider_key]
return min(governed, float(requested)) if requested is not None else governed
@staticmethod
def _resolve_geometry(db, project_id: UUID, payload: AoiOperationCreate):
if (payload.area_id is None) == (payload.bbox is None):
raise AppError(code="AOI_SELECTION_REQUIRED", message="Provide exactly one area_id or bbox", status_code=422)
raise AppError(
code="AOI_SELECTION_REQUIRED",
message="Provide exactly one area_id or bbox",
status_code=422,
)
if payload.area_id is not None:
area = db.get(Area, payload.area_id)
if area is None or area.project_id != project_id:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
raise AppError(
code="AREA_NOT_FOUND", message="Area not found", status_code=404
)
return to_shape(area.geometry)
bbox = payload.bbox
if bbox is None or bbox.crs != "EPSG:4326":
raise AppError(code="INVALID_AOI_CRS", message="AOI bbox must use EPSG:4326", status_code=422)
raise AppError(
code="INVALID_AOI_CRS",
message="AOI bbox must use EPSG:4326",
status_code=422,
)
return box(bbox.min_x, bbox.min_y, bbox.max_x, bbox.max_y)
@staticmethod
@@ -166,11 +230,26 @@ class AoiOperationService:
columns = max(1, math.ceil((max_x - min_x) / side_m))
rows = max(1, math.ceil((max_y - min_y) / side_m))
if columns * rows > AoiOperationService.MAX_PARTITIONS:
raise AppError(code="AOI_PARTITION_LIMIT_EXCEEDED", message="AOI requires too many bounded partitions", details={"candidate_count": columns * rows, "max_partitions": AoiOperationService.MAX_PARTITIONS}, status_code=422)
raise AppError(
code="AOI_PARTITION_LIMIT_EXCEEDED",
message="AOI requires too many bounded partitions",
details={
"candidate_count": columns * rows,
"max_partitions": AoiOperationService.MAX_PARTITIONS,
},
status_code=422,
)
partitions = []
for row in range(rows):
for column in range(columns):
clipped = geometry.intersection(box(min_x + column * side_m, min_y + row * side_m, min(min_x + (column + 1) * side_m, max_x), min(min_y + (row + 1) * side_m, max_y)))
clipped = geometry.intersection(
box(
min_x + column * side_m,
min_y + row * side_m,
min(min_x + (column + 1) * side_m, max_x),
min(min_y + (row + 1) * side_m, max_y),
)
)
if not clipped.is_empty and clipped.area > 0:
partitions.append(clipped)
return partitions
@@ -179,98 +258,224 @@ class AoiOperationService:
def read(db, project_id: UUID, operation_id: UUID) -> dict:
operation = db.get(AoiOperation, operation_id)
if operation is None or operation.project_id != project_id:
raise AppError(code="AOI_OPERATION_NOT_FOUND", message="AOI operation not found", status_code=404)
partitions = db.query(AoiOperationPartition).filter(AoiOperationPartition.operation_id == operation_id).order_by(AoiOperationPartition.ordinal).all()
raise AppError(
code="AOI_OPERATION_NOT_FOUND",
message="AOI operation not found",
status_code=404,
)
partitions = (
db.query(AoiOperationPartition)
.filter(AoiOperationPartition.operation_id == operation_id)
.order_by(AoiOperationPartition.ordinal)
.all()
)
counts = Counter(partition.status for partition in partitions)
complete = counts["success"] + counts["skipped"]
return {
"id": operation.id, "project_id": operation.project_id, "area_id": operation.area_id,
"parent_job_id": operation.parent_job_id, "operation_type": operation.operation_type,
"status": operation.status, "request_json": operation.request_json, "plan_json": operation.plan_json,
"result_json": operation.result_json, "error_message": operation.error_message,
"id": operation.id,
"project_id": operation.project_id,
"area_id": operation.area_id,
"parent_job_id": operation.parent_job_id,
"operation_type": operation.operation_type,
"status": operation.status,
"request_json": operation.request_json,
"plan_json": operation.plan_json,
"result_json": operation.result_json,
"error_message": operation.error_message,
"progress": round(complete / len(partitions), 6) if partitions else 0.0,
"partition_counts": dict(counts), "partitions": partitions,
"created_at": operation.created_at, "started_at": operation.started_at, "finished_at": operation.finished_at,
"partition_counts": dict(counts),
"partitions": partitions,
"created_at": operation.created_at,
"started_at": operation.started_at,
"finished_at": operation.finished_at,
}
@staticmethod
def list(db, project_id: UUID, limit: int = 50) -> dict:
rows = db.query(AoiOperation).filter(AoiOperation.project_id == project_id).order_by(AoiOperation.created_at.desc()).limit(limit).all()
return {"items": [AoiOperationService.read(db, project_id, row.id) for row in rows], "total": len(rows)}
rows = (
db.query(AoiOperation)
.filter(AoiOperation.project_id == project_id)
.order_by(AoiOperation.created_at.desc())
.limit(limit)
.all()
)
return {
"items": [AoiOperationService.read(db, project_id, row.id) for row in rows],
"total": len(rows),
}
@staticmethod
def claim_next(db, project_id: UUID, operation_id: UUID):
operation = db.get(AoiOperation, operation_id)
if operation is None or operation.project_id != project_id:
raise AppError(code="AOI_OPERATION_NOT_FOUND", message="AOI operation not found", status_code=404)
partition = db.query(AoiOperationPartition).filter(AoiOperationPartition.operation_id == operation_id, AoiOperationPartition.status == "queued").order_by(AoiOperationPartition.ordinal).with_for_update(skip_locked=True).first()
raise AppError(
code="AOI_OPERATION_NOT_FOUND",
message="AOI operation not found",
status_code=404,
)
partition = (
db.query(AoiOperationPartition)
.filter(
AoiOperationPartition.operation_id == operation_id,
AoiOperationPartition.status == "queued",
)
.order_by(AoiOperationPartition.ordinal)
.with_for_update(skip_locked=True)
.first()
)
if partition is None:
return None
now = datetime.now(timezone.utc)
partition.status = "running"; partition.started_at = now; partition.attempt_count += 1; partition.error_message = None
operation.status = "running"; operation.started_at = operation.started_at or now
db.add(partition); db.add(operation); db.commit(); db.refresh(partition)
partition.status = "running"
partition.started_at = now
partition.attempt_count += 1
partition.error_message = None
operation.status = "running"
operation.started_at = operation.started_at or now
db.add(partition)
db.add(operation)
db.commit()
db.refresh(partition)
return partition
@staticmethod
def checkpoint(db, project_id: UUID, operation_id: UUID, partition_id: UUID, checkpoint: dict):
partition = AoiOperationService._partition_row(db, project_id, operation_id, partition_id)
def checkpoint(
db, project_id: UUID, operation_id: UUID, partition_id: UUID, checkpoint: dict
):
partition = AoiOperationService._partition_row(
db, project_id, operation_id, partition_id
)
if partition.status != "running":
raise AppError(code="AOI_PARTITION_NOT_RUNNING", message="Only a running partition can be checkpointed", status_code=409)
partition.checkpoint_json = checkpoint; db.add(partition); db.commit(); db.refresh(partition)
raise AppError(
code="AOI_PARTITION_NOT_RUNNING",
message="Only a running partition can be checkpointed",
status_code=409,
)
partition.checkpoint_json = checkpoint
db.add(partition)
db.commit()
db.refresh(partition)
return partition
@staticmethod
def complete(db, project_id: UUID, operation_id: UUID, partition_id: UUID, result: dict, skipped: bool = False):
partition = AoiOperationService._partition_row(db, project_id, operation_id, partition_id)
def complete(
db,
project_id: UUID,
operation_id: UUID,
partition_id: UUID,
result: dict,
skipped: bool = False,
):
partition = AoiOperationService._partition_row(
db, project_id, operation_id, partition_id
)
if partition.status == "success" or partition.status == "skipped":
return AoiOperationService.read(db, project_id, operation_id)
if partition.status != "running":
raise AppError(code="AOI_PARTITION_NOT_RUNNING", message="Only a running partition can complete", status_code=409)
partition.status = "skipped" if skipped else "success"; partition.result_json = result; partition.finished_at = datetime.now(timezone.utc)
db.add(partition); db.commit(); AoiOperationService._refresh_parent(db, operation_id)
raise AppError(
code="AOI_PARTITION_NOT_RUNNING",
message="Only a running partition can complete",
status_code=409,
)
partition.status = "skipped" if skipped else "success"
partition.result_json = result
partition.finished_at = datetime.now(timezone.utc)
db.add(partition)
db.commit()
AoiOperationService._refresh_parent(db, operation_id)
return AoiOperationService.read(db, project_id, operation_id)
@staticmethod
def fail(db, project_id: UUID, operation_id: UUID, partition_id: UUID, message: str, retryable: bool, details: dict):
partition = AoiOperationService._partition_row(db, project_id, operation_id, partition_id)
partition.error_message = message; partition.result_json = {"details": details}
partition.status = "queued" if retryable and partition.attempt_count < partition.max_attempts else "failed"
partition.finished_at = None if partition.status == "queued" else datetime.now(timezone.utc)
db.add(partition); db.commit(); AoiOperationService._refresh_parent(db, operation_id)
def fail(
db,
project_id: UUID,
operation_id: UUID,
partition_id: UUID,
message: str,
retryable: bool,
details: dict,
):
partition = AoiOperationService._partition_row(
db, project_id, operation_id, partition_id
)
partition.error_message = message
partition.result_json = {"details": details}
partition.status = (
"queued"
if retryable and partition.attempt_count < partition.max_attempts
else "failed"
)
partition.finished_at = (
None if partition.status == "queued" else datetime.now(timezone.utc)
)
db.add(partition)
db.commit()
AoiOperationService._refresh_parent(db, operation_id)
return AoiOperationService.read(db, project_id, operation_id)
@staticmethod
def _partition_row(db, project_id, operation_id, partition_id):
operation = db.get(AoiOperation, operation_id); partition = db.get(AoiOperationPartition, partition_id)
if operation is None or operation.project_id != project_id or partition is None or partition.operation_id != operation_id:
raise AppError(code="AOI_PARTITION_NOT_FOUND", message="AOI partition not found", status_code=404)
operation = db.get(AoiOperation, operation_id)
partition = db.get(AoiOperationPartition, partition_id)
if (
operation is None
or operation.project_id != project_id
or partition is None
or partition.operation_id != operation_id
):
raise AppError(
code="AOI_PARTITION_NOT_FOUND",
message="AOI partition not found",
status_code=404,
)
return partition
@staticmethod
def _refresh_parent(db, operation_id):
operation = db.get(AoiOperation, operation_id)
partitions = db.query(AoiOperationPartition).filter(AoiOperationPartition.operation_id == operation_id).order_by(AoiOperationPartition.ordinal).all()
partitions = (
db.query(AoiOperationPartition)
.filter(AoiOperationPartition.operation_id == operation_id)
.order_by(AoiOperationPartition.ordinal)
.all()
)
statuses = [partition.status for partition in partitions]
output_dataset_ids = []
for partition in partitions:
output_id = (partition.result_json or {}).get("output_dataset_id") if isinstance(partition.result_json, dict) else None
output_id = (
(partition.result_json or {}).get("output_dataset_id")
if isinstance(partition.result_json, dict)
else None
)
if output_id and str(output_id) not in output_dataset_ids:
output_dataset_ids.append(str(output_id))
operation.result_json = {
"partition_count": len(partitions),
"completed_partition_count": sum(status in {"success", "skipped"} for status in statuses),
"completed_partition_count": sum(
status in {"success", "skipped"} for status in statuses
),
"failed_partition_count": statuses.count("failed"),
"output_dataset_ids": output_dataset_ids,
"merge_contract": "source_aware_spatial_union",
"vector_deduplication": "source_feature_id_then_geometry",
"raster_deduplication": "governed_mosaic_grid",
"complete_coverage": bool(statuses) and all(status in {"success", "skipped"} for status in statuses),
"complete_coverage": bool(statuses)
and all(status in {"success", "skipped"} for status in statuses),
}
now = datetime.now(timezone.utc)
if statuses and all(status in {"success", "skipped"} for status in statuses):
operation.status = "success"; operation.finished_at = now; operation.error_message = None
elif "failed" in statuses and not any(status in {"queued", "running"} for status in statuses):
operation.status = "partial" if any(status in {"success", "skipped"} for status in statuses) else "failed"; operation.finished_at = now
operation.status = "success"
operation.finished_at = now
operation.error_message = None
elif "failed" in statuses and not any(
status in {"queued", "running"} for status in statuses
):
operation.status = (
"partial"
if any(status in {"success", "skipped"} for status in statuses)
else "failed"
)
operation.finished_at = now
operation.error_message = "One or more bounded source partitions failed; inspect partition evidence."
db.add(operation); db.commit()
db.add(operation)
db.commit()
backend/app/services/area_service.py
+28 -7
View File
@@ -9,7 +9,7 @@ from shapely.geometry import mapping
from app.core.errors import AppError
from app.models import Area, Dataset, Project, VectorFeature
from app.schemas.area import AreaCreate, AreaRead, AreaUpdate
from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_to_multipolygon
from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_area_to_epsg4326
class AreaService:
@@ -126,7 +126,11 @@ class AreaService:
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
try:
multipolygon = normalize_to_multipolygon(payload.geometry)
multipolygon, original_crs = normalize_area_to_epsg4326(
payload.geometry,
payload.crs or "EPSG:4326",
)
metric_area = area_m2(multipolygon)
except ValueError as exc:
raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc
@@ -134,8 +138,8 @@ class AreaService:
project_id=project_id,
name=payload.name.strip() or "Unnamed area",
geometry=from_shape(multipolygon, srid=4326),
original_crs=payload.crs or "EPSG:4326",
area_m2=area_m2(multipolygon),
original_crs=original_crs,
area_m2=metric_area,
bbox=from_shape(geometry_bbox_polygon(multipolygon), srid=4326),
)
db.add(area)
@@ -157,11 +161,28 @@ class AreaService:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
changed = False
if payload.name:
if payload.name is not None and payload.name.strip():
area.name = payload.name.strip() or area.name
changed = True
if payload.crs:
area.original_crs = payload.crs
if payload.crs is not None and payload.geometry is None:
raise AppError(
code="INVALID_AREA_CRS_UPDATE",
message="crs can only be supplied together with replacement geometry",
status_code=422,
)
if payload.geometry is not None:
try:
multipolygon, original_crs = normalize_area_to_epsg4326(
payload.geometry,
payload.crs or "EPSG:4326",
)
metric_area = area_m2(multipolygon)
except ValueError as exc:
raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc
area.geometry = from_shape(multipolygon, srid=4326)
area.original_crs = original_crs
area.area_m2 = metric_area
area.bbox = from_shape(geometry_bbox_polygon(multipolygon), srid=4326)
changed = True
if not changed:
raise AppError(code="NO_CHANGES", message="No updatable fields provided", status_code=422)
backend/app/services/bathymetry_profile_acquisition_service.py
+1 -1
View File
@@ -7,7 +7,7 @@ import math
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
from urllib.request import Request
from uuid import UUID
from geoalchemy2.shape import to_shape
backend/app/services/bathymetry_raster_analysis_service.py
-1
View File
@@ -112,7 +112,6 @@ class BathymetryRasterAnalysisService:
try:
import numpy as np
import rasterio
from rasterio.features import geometry_mask
from rasterio.mask import mask
except ImportError as exc:
raise AppError(
backend/app/services/dataset_service.py
+104 -34
View File
@@ -17,6 +17,7 @@ from shapely.geometry import MultiPoint, shape
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.core.config import get_settings
from app.models import Area, Dataset, DatasetVersion, Project
from app.services.data_contract_validation import (
ContractKind,
@@ -476,6 +477,29 @@ class DatasetService:
content_type=content_type,
)
@classmethod
def _persist_vector_source_evidence_from_path(
cls,
*,
project_id: UUID,
dataset_id: UUID,
original_filename: str,
source_path: str | Path,
content_type: str | None,
) -> dict[str, Any]:
safe_filename = StorageService._safe_filename(original_filename)
evidence_path = (
StorageService.dataset_root(str(project_id), str(dataset_id), "vector")
/ "provenance"
/ f"{dataset_id}_source_{safe_filename}"
)
return StorageService.persist_file_from_path(
str(evidence_path),
source_path,
original_filename=safe_filename,
content_type=content_type,
)
@classmethod
def _record_vector_source_evidence(
cls,
@@ -853,6 +877,52 @@ class DatasetService:
raise AppError(code="INVALID_UPLOAD", message="Missing file name", status_code=400)
return filename
@staticmethod
async def _stage_upload(
*,
project_id: UUID,
dataset_id: uuid.UUID,
dataset_type: str,
filename: str,
file: UploadFile,
) -> dict[str, Any]:
settings = get_settings()
max_upload_mb = int(settings.max_upload_mb)
if DatasetService._canonical_dataset_type(dataset_type) == "vector":
max_upload_mb = min(max_upload_mb, int(settings.max_in_memory_vector_mb))
return await StorageService.persist_upload_file(
project_id=str(project_id),
dataset_id=str(dataset_id),
dataset_type=dataset_type,
original_filename=filename,
upload=file,
content_type=file.content_type,
max_bytes=max_upload_mb * 1024 * 1024,
)
@staticmethod
def _read_staged_vector_bytes(storage_info: dict[str, Any]) -> bytes:
settings = get_settings()
max_bytes = min(
int(settings.max_upload_mb),
int(settings.max_in_memory_vector_mb),
) * 1024 * 1024
path = Path(str(storage_info["storage_path"]))
with path.open("rb") as stream:
content = stream.read(max_bytes + 1)
if len(content) > max_bytes:
StorageService.remove_dataset_file(str(path))
raise AppError(
code="UPLOAD_TOO_LARGE",
message="Vector upload exceeds the bounded in-memory parsing limit.",
details={
"max_bytes": max_bytes,
"max_in_memory_vector_mb": max_bytes // (1024 * 1024),
},
status_code=413,
)
return content
@staticmethod
def list_datasets(db: Session, project_id: UUID, limit: int = 50, offset: int = 0) -> tuple[list[DatasetCreateResponse], int]:
total = db.query(Dataset).filter(Dataset.project_id == project_id).count()
@@ -1135,15 +1205,15 @@ class DatasetService:
status_code=415,
)
raw = await file.read()
storage_info = StorageService.persist_dataset_file(
project_id=str(project_id),
dataset_id=str(dataset_id := uuid.uuid4()),
dataset_id = uuid.uuid4()
storage_info = await DatasetService._stage_upload(
project_id=project_id,
dataset_id=dataset_id,
dataset_type=canonical_type,
original_filename=filename,
content=raw,
content_type=file.content_type,
filename=filename,
file=file,
)
raw = DatasetService._read_staged_vector_bytes(storage_info) if canonical_type == "vector" else None
metadata: dict[str, Any] = {}
vector_payload: dict[str, Any] | None = None
@@ -1151,6 +1221,7 @@ class DatasetService:
try:
status = "validating"
if canonical_type == "vector":
assert raw is not None
try:
text = raw.decode("utf-8")
except UnicodeDecodeError as exc:
@@ -1319,8 +1390,15 @@ class DatasetService:
temporal_granularity=temporal_granularity,
source_version=source_version,
)
raw = await file.read()
checksum_sha256 = StorageService.calculate_checksum_sha256(raw)
dataset_id = uuid.uuid4()
storage_info = await DatasetService._stage_upload(
project_id=project_id,
dataset_id=dataset_id,
dataset_type=canonical_type,
filename=filename,
file=file,
)
checksum_sha256 = storage_info["checksum_sha256"]
ingest_key = DatasetService._ingest_key(
project_id=project_id,
source_key="manual",
@@ -1333,6 +1411,7 @@ class DatasetService:
)
existing = DatasetService._find_existing_ingest(db, project_id, ingest_key)
if existing is not None:
StorageService.remove_dataset_file(storage_info["storage_path"])
return DatasetService._to_response(existing)
raw_source_metadata = dict(source_metadata or {})
@@ -1361,8 +1440,7 @@ class DatasetService:
}
)
dataset_id = uuid.uuid4()
storage_info: dict[str, Any] | None = None
raw = DatasetService._read_staged_vector_bytes(storage_info) if canonical_type == "vector" else None
storage_content = raw
source_evidence: dict[str, Any] | None = None
imported_at = datetime.now(timezone.utc)
@@ -1372,6 +1450,7 @@ class DatasetService:
parser_error: tuple[str, str] | None = None
try:
if canonical_type == "vector":
assert raw is not None
try:
payload = json.loads(raw.decode("utf-8"))
except UnicodeDecodeError as exc:
@@ -1394,22 +1473,22 @@ class DatasetService:
)
if DatasetService._vector_storage_requires_canonicalization(source_crs):
storage_content = DatasetService._canonical_vector_storage_bytes(canonical_vector_payload)
source_evidence = DatasetService._persist_vector_source_evidence(
source_evidence = DatasetService._persist_vector_source_evidence_from_path(
project_id=project_id,
dataset_id=dataset_id,
original_filename=filename,
content=raw,
source_path=storage_info["storage_path"],
content_type=file.content_type,
)
storage_info = StorageService.persist_dataset_file(
project_id=str(project_id),
dataset_id=str(dataset_id),
dataset_type=canonical_type,
original_filename=filename,
content=storage_content,
content_type=file.content_type,
)
else:
storage_info = StorageService.persist_dataset_file(
project_id=str(project_id),
dataset_id=str(dataset_id),
dataset_type=canonical_type,
original_filename=filename,
content=raw,
content_type=file.content_type,
)
metadata = extract_raster_metadata(storage_info["storage_path"])
metadata["dataset_type"] = "raster"
source_crs = metadata.get("crs")
@@ -1422,16 +1501,7 @@ class DatasetService:
"processing_code": code,
}
if storage_info is None:
storage_info = StorageService.persist_dataset_file(
project_id=str(project_id),
dataset_id=str(dataset_id),
dataset_type=canonical_type,
original_filename=filename,
content=storage_content,
content_type=file.content_type,
)
computed_storage_checksum_sha256 = StorageService.calculate_checksum_sha256(storage_content)
computed_storage_checksum_sha256 = storage_info["checksum_sha256"]
if source_evidence is not None:
resolved_source_crs = source_crs or DatasetService.CANONICAL_VECTOR_CRS
DatasetService._record_vector_source_evidence(
@@ -1502,7 +1572,7 @@ class DatasetService:
feature_collection=canonical_vector_payload or {"type": "FeatureCollection", "features": []},
checksum_sha256=storage_info["checksum_sha256"],
computed_checksum_sha256=computed_storage_checksum_sha256,
content=storage_content,
content=None,
source_registry_id=str(source_registry.id),
source_snapshot_id=str(source_snapshot.id),
imported_at=imported_at,
@@ -1541,8 +1611,8 @@ class DatasetService:
bounds=DatasetService._extract_raster_bounds_json(metadata),
resolution=resolution,
checksum_sha256=storage_info["checksum_sha256"],
computed_checksum_sha256=checksum_sha256,
content=raw,
computed_checksum_sha256=storage_info["checksum_sha256"],
content=None,
source_registry_id=str(source_registry.id),
source_snapshot_id=str(source_snapshot.id),
imported_at=imported_at,
backend/app/services/detection_review_service.py
-1
View File
@@ -1,7 +1,6 @@
from __future__ import annotations
from collections import Counter
from typing import Any
from uuid import UUID
from sqlalchemy.orm import Session
backend/app/services/detection_service.py
+27
View File
@@ -33,6 +33,7 @@ from app.services.storage_service import StorageService
from app.services.quality_service import QualityService
from app.services.runtime_model_provenance_service import RuntimeModelProvenance, RuntimeModelProvenanceService
from app.services.temporal_compatibility_service import TemporalCompatibilityService
from app.services.tile_manifest_service import TileManifestService
from app.services.yolo_adapter import YoloDetectionAdapter
@@ -968,6 +969,18 @@ class DetectionService:
yolo_adapter_class: Type[YoloDetectionAdapter],
) -> tuple[list[Detection], dict[str, Any]]:
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles, settings)
dataset = db.get(Dataset, dataset_id)
if dataset is None:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
manifest_binding = TileManifestService.validate_for_inference(
db,
dataset,
manifest,
manifest_path=tile_manifest_path or "",
settings=settings,
error_prefix="DETECTION",
)
DetectionService._attach_tile_manifest_binding(analysis_run, job, manifest_binding)
model_path = Path(settings.yolo_model_path or "").expanduser()
runtime_model_provenance = RuntimeModelProvenanceService.validate_for_production_runtime(
db=db,
@@ -1075,9 +1088,23 @@ class DetectionService:
"tile_edge_truncated_count": len(candidates) - len(edge_filtered_candidates),
"duplicate_iou_threshold": float(settings.yolo_duplicate_iou_threshold),
"containment_suppression_threshold": float(settings.yolo_containment_nms_threshold),
"tile_manifest_binding": manifest_binding,
"runtime_model_provenance": runtime_model_provenance.as_dict(),
}
@staticmethod
def _attach_tile_manifest_binding(
analysis_run: AnalysisRun,
job: Job,
binding: dict[str, Any],
) -> None:
analysis_parameters = dict(analysis_run.parameters_json or {})
analysis_parameters["tile_manifest_binding"] = dict(binding)
analysis_run.parameters_json = analysis_parameters
job_parameters = dict(job.parameters_json or {})
job_parameters["tile_manifest_binding"] = dict(binding)
job.parameters_json = job_parameters
@staticmethod
def _attach_runtime_model_provenance(
analysis_run: AnalysisRun,
backend/app/services/dhmv_acquisition_service.py
+1 -1
View File
@@ -12,7 +12,7 @@ from pathlib import Path
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
from urllib.request import Request
from uuid import UUID
from geoalchemy2.shape import to_shape
backend/app/services/flood_hazard_acquisition_service.py
+1 -1
View File
@@ -12,7 +12,7 @@ from pathlib import Path
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
from urllib.request import Request
from uuid import UUID
from xml.etree import ElementTree
backend/app/services/flood_hazard_analysis_service.py
-1
View File
@@ -232,7 +232,6 @@ class FloodHazardAnalysisService:
try:
import numpy as np
import rasterio
from rasterio.features import geometry_mask
from rasterio.mask import mask
except ImportError as exc:
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for flood-hazard analysis", status_code=503) from exc
backend/app/services/mdk_bathymetry_acquisition_service.py
+1 -1
View File
@@ -5,7 +5,7 @@ from datetime import UTC, datetime
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
from urllib.request import Request, urlopen
from urllib.request import Request
from uuid import UUID
from app.core.config import Settings, get_settings
backend/app/services/mdk_bathymetry_probe_service.py
+2 -1
View File
@@ -6,11 +6,12 @@ import ssl
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
from urllib.request import Request, urlopen
from urllib.request import Request
from xml.etree import ElementTree
from app.core.config import Settings, get_settings
from app.schemas.bathymetry import BathymetrySourceProbeRead
from app.services.outbound_request_guard import guarded_opener
class MdkBathymetryProbeService:
backend/app/services/model_asset_catalog_service.py
+7 -3
View File
@@ -79,11 +79,15 @@ class ModelAssetCatalogService:
size_bytes=path.stat().st_size,
sha256=ModelAssetCatalogService._sha256(path),
active=active_model_path == resolved_path,
status="approved" if active_model_path == resolved_path else "available",
runtime_available=True,
runtime_status="active" if active_model_path == resolved_path else "available",
governed_validation_status="not_verified_by_catalog",
promotion_status="not_verified_by_catalog",
status="runtime_active" if active_model_path == resolved_path else "runtime_available",
limitation_message=(
"Approved local runtime model asset. GeoIntel will not download or mutate model weights."
"Active local runtime model asset. Runtime selection is not evidence of governed validation or promotion."
if active_model_path == resolved_path
else "Local development model asset. Configure it explicitly before production use."
else "Local runtime model asset. Governed validation and promotion are not established by this catalog."
),
will_download_models=False,
)
backend/app/services/orthophoto_acquisition_service.py
+1 -1
View File
@@ -11,7 +11,7 @@ from pathlib import Path
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
from urllib.request import Request
from uuid import UUID
from geoalchemy2.shape import to_shape
backend/app/services/raster_operations_service.py
+73 -7
View File
@@ -1,8 +1,8 @@
from __future__ import annotations
import json
import uuid
from datetime import datetime, timezone
from hashlib import sha256
from pathlib import Path
from typing import Any
@@ -16,6 +16,7 @@ from app.models import Area, Dataset, DatasetVersion
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
from app.services.raster_service import extract_raster_metadata
from app.services.storage_service import StorageService
from app.services.tile_manifest_service import TileManifestService, canonical_manifest_json
def _import_rasterio():
@@ -70,6 +71,39 @@ class RasterOperationsService:
raise AppError(code="DATASET_FILE_MISSING", message="Stored raster file missing", status_code=404)
return dataset
@staticmethod
def _validate_storage_checksum(dataset: Dataset) -> None:
"""Refuse tiling pixels that no longer match the governed Dataset row."""
expected_checksum = str(dataset.checksum_sha256 or "").strip().lower()
governed_artifact = bool(getattr(dataset, "data_contract_key", None))
valid_checksum = len(expected_checksum) == 64 and all(character in "0123456789abcdef" for character in expected_checksum)
if not valid_checksum:
if expected_checksum or governed_artifact:
raise AppError(
code="DATASET_STORAGE_CHECKSUM_UNVERIFIABLE",
message="Governed raster storage requires a valid SHA-256 checksum before tiling.",
details={"checksum_sha256": dataset.checksum_sha256},
status_code=409,
)
return
digest = sha256()
with Path(str(dataset.storage_path)).open("rb") as stream:
for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""):
digest.update(chunk)
actual_checksum = digest.hexdigest()
if actual_checksum != expected_checksum:
raise AppError(
code="DATASET_STORAGE_CHECKSUM_MISMATCH",
message="Raster dataset storage no longer matches its validated checksum.",
details={
"expected_checksum_sha256": expected_checksum,
"actual_checksum_sha256": actual_checksum,
},
status_code=409,
)
@staticmethod
def _raster_dependencies() -> tuple[Any, Any]:
try:
@@ -970,14 +1004,17 @@ class RasterOperationsService:
tile_size: int = 512,
overlap: int = 64,
output_name: str | None = None,
max_tiles: int | None = None,
) -> dict[str, Any]:
RasterOperationsService._validate_tile_request(tile_size=tile_size, overlap=overlap)
if max_tiles is not None and max_tiles <= 0:
raise AppError(code="INVALID_PARAMETERS", message="max_tiles must be positive", status_code=400)
dataset = RasterOperationsService._load_dataset(db, dataset_id)
RasterOperationsService._validate_storage_checksum(dataset)
rasterio, _ = RasterOperationsService._raster_dependencies()
tile_set_id = str(uuid.uuid4())
tile_root = StorageService.raster_tiles_root(str(dataset.project_id), str(dataset.id), tile_set_id)
tile_root.mkdir(parents=True, exist_ok=True)
manifest_tiles: list[dict[str, Any]] = []
tile_paths: list[str] = []
@@ -997,9 +1034,23 @@ class RasterOperationsService:
source_width = int(source.width)
source_height = int(source.height)
step = max(1, tile_size - overlap)
x_offsets = RasterOperationsService._tile_offsets(source_width, tile_size, step)
y_offsets = RasterOperationsService._tile_offsets(source_height, tile_size, step)
expected_tile_count = len(x_offsets) * len(y_offsets)
if max_tiles is not None and expected_tile_count > max_tiles:
raise AppError(
code="RASTER_TILE_LIMIT_EXCEEDED",
message="Raster tile generation exceeds the guest analysis limit.",
details={
"expected_tile_count": expected_tile_count,
"max_tiles": max_tiles,
},
status_code=422,
)
tile_root.mkdir(parents=True, exist_ok=True)
tile_index = 0
for yoff in range(0, source_height, step):
for xoff in range(0, source_width, step):
for yoff in y_offsets:
for xoff in x_offsets:
tile_width = min(tile_size, source_width - xoff)
tile_height = min(tile_size, source_height - yoff)
if tile_width <= 0 or tile_height <= 0:
@@ -1022,6 +1073,7 @@ class RasterOperationsService:
tile_dest.write(tile_data)
tile_paths.append(str(tile_path))
tile_integrity = TileManifestService.tile_integrity(tile_path)
manifest_tiles.append(
{
"path": str(tile_path),
@@ -1030,6 +1082,7 @@ class RasterOperationsService:
"transform": [float(item) for item in transform.to_gdal()],
"crs": source_crs,
"index": tile_index,
**tile_integrity,
},
)
tile_index += 1
@@ -1044,9 +1097,8 @@ class RasterOperationsService:
bounds = source_metadata.get("bounds", [0.0, 0.0, 0.0, 0.0])
manifest_crs = source_crs or source_metadata.get("crs") or dataset.crs
manifest_payload = {
**TileManifestService.dataset_binding(db, dataset),
"tile_set_id": tile_set_id,
"source_dataset_id": str(dataset.id),
"source_raster_id": str(dataset.id),
"crs": manifest_crs,
"source_crs": manifest_crs,
"dataset_crs": dataset.crs,
@@ -1066,7 +1118,7 @@ class RasterOperationsService:
"tile_server": None,
}
manifest_path = tile_root / "manifest.json"
manifest_path.write_text(json.dumps(manifest_payload), encoding="utf-8")
manifest_path.write_text(canonical_manifest_json(manifest_payload), encoding="utf-8")
return {
"dataset_id": str(dataset.id),
@@ -1079,3 +1131,17 @@ class RasterOperationsService:
"count": len(manifest_tiles),
"manifest": manifest_payload,
}
@staticmethod
def _tile_offsets(dimension: int, tile_size: int, step: int) -> list[int]:
"""Return full-tile starts plus one unique edge-aligned final start."""
if dimension <= 0 or tile_size <= 0 or step <= 0:
raise AppError(code="INVALID_PARAMETERS", message="Raster tile dimensions must be positive", status_code=400)
if dimension <= tile_size:
return [0]
final_start = dimension - tile_size
offsets = list(range(0, final_start + 1, step))
if offsets[-1] != final_start:
offsets.append(final_start)
return offsets
backend/app/services/raster_partition_analysis_service.py
-2
View File
@@ -8,7 +8,6 @@ from typing import Any
from uuid import UUID
from pyproj import Transformer
from shapely.geometry import mapping
from shapely.ops import transform as shapely_transform
from app.core.errors import AppError
@@ -116,7 +115,6 @@ class RasterPartitionAnalysisService:
try:
import numpy as np
import rasterio
from rasterio.features import geometry_mask
from rasterio.merge import merge
except ImportError as exc:
raise AppError(
backend/app/services/segmentation_service.py
+14
View File
@@ -34,6 +34,7 @@ from app.services.segmentation_adapter import (
SamSegmentationAdapter,
YoloSegmentationAdapter,
)
from app.services.tile_manifest_service import TileManifestService
class SegmentationService:
@@ -749,6 +750,18 @@ class SegmentationService:
sam_adapter_class: type[SamSegmentationAdapter],
) -> tuple[list[Segmentation], dict[str, Any]]:
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles, settings)
dataset = db.get(Dataset, dataset_id)
if dataset is None:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
manifest_binding = TileManifestService.validate_for_inference(
db,
dataset,
manifest,
manifest_path=tile_manifest_path or "",
settings=settings,
error_prefix="SEGMENTATION",
)
DetectionService._attach_tile_manifest_binding(analysis_run, job, manifest_binding)
if model_name == settings.sam_model_id:
model_path = Path(settings.sam_model_path or "").expanduser()
allowed_frameworks = ("ultralytics/sam", "sam", "ultralytics", "pytorch")
@@ -861,6 +874,7 @@ class SegmentationService:
"duplicate_iou_threshold": float(settings.segmentation_duplicate_iou_threshold),
"containment_suppression_threshold": float(settings.segmentation_containment_nms_threshold),
"tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()),
"tile_manifest_binding": manifest_binding,
"runtime_model_provenance": runtime_model_provenance.as_dict(),
}
backend/app/services/source_catalog_probe_service.py
+1 -1
View File
@@ -10,7 +10,7 @@ from threading import Lock
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
from urllib.request import Request, urlopen
from urllib.request import Request
from uuid import UUID
from xml.etree import ElementTree
backend/app/services/storage_service.py
+99
View File
@@ -10,6 +10,8 @@ from app.core.errors import AppError
class StorageService:
UPLOAD_CHUNK_SIZE = 8 * 1024 * 1024
@staticmethod
def _base_dir() -> Path:
return Path(get_settings().storage_root).resolve()
@@ -143,6 +145,75 @@ class StorageService:
}
return metadata
@staticmethod
async def persist_upload_file(
*,
project_id: str,
dataset_id: str,
dataset_type: str,
original_filename: str,
upload: Any,
content_type: str | None,
max_bytes: int,
chunk_size: int | None = None,
) -> dict[str, Any]:
"""Stream an UploadFile to governed storage with a hard byte limit.
The reverse proxy limit is defense in depth. This backend boundary is
authoritative as direct/loopback requests can bypass that proxy.
"""
if max_bytes <= 0:
raise ValueError("max_bytes must be positive")
resolved_chunk_size = chunk_size or StorageService.UPLOAD_CHUNK_SIZE
if resolved_chunk_size <= 0:
raise ValueError("chunk_size must be positive")
normalized_type = StorageService.normalize_dataset_type(dataset_type)
file_path = Path(
StorageService.dataset_file_path(
project_id,
dataset_id,
normalized_type,
original_filename,
)
)
file_path.parent.mkdir(parents=True, exist_ok=True)
digest = hashlib.sha256()
size_bytes = 0
try:
with file_path.open("wb") as stream:
while True:
chunk = await upload.read(resolved_chunk_size)
if not chunk:
break
size_bytes += len(chunk)
if size_bytes > max_bytes:
raise AppError(
code="UPLOAD_TOO_LARGE",
message="Upload exceeds the configured backend size limit.",
details={
"max_bytes": max_bytes,
"max_upload_mb": max_bytes // (1024 * 1024),
},
status_code=413,
)
stream.write(chunk)
digest.update(chunk)
except Exception:
file_path.unlink(missing_ok=True)
parent = file_path.parent
if parent.exists() and parent.is_dir() and not any(parent.iterdir()):
parent.rmdir()
raise
return {
"original_filename": StorageService._safe_filename(original_filename),
"stored_filename": file_path.name,
"content_type": content_type or "application/octet-stream",
"size_bytes": size_bytes,
"checksum_sha256": digest.hexdigest(),
"storage_path": str(file_path),
}
@staticmethod
def persist_dataset_file_from_path(
project_id: str,
@@ -198,6 +269,34 @@ class StorageService:
}
return metadata
@staticmethod
def persist_file_from_path(
storage_path: str,
source_path: str | Path,
original_filename: str,
content_type: str | None,
) -> dict[str, Any]:
source = Path(source_path).resolve()
if not source.is_file():
raise FileNotFoundError(f"Source artifact does not exist: {source}")
target = Path(storage_path)
target.parent.mkdir(parents=True, exist_ok=True)
digest = hashlib.sha256()
size_bytes = 0
with source.open("rb") as input_stream, target.open("wb") as output_stream:
for chunk in iter(lambda: input_stream.read(StorageService.UPLOAD_CHUNK_SIZE), b""):
output_stream.write(chunk)
digest.update(chunk)
size_bytes += len(chunk)
return {
"original_filename": StorageService._safe_filename(original_filename),
"stored_filename": target.name,
"content_type": content_type or "application/octet-stream",
"size_bytes": size_bytes,
"checksum_sha256": digest.hexdigest(),
"storage_path": str(target),
}
@staticmethod
def remove_dataset_file(path: str) -> None:
target = Path(path)
backend/app/services/terrain_analysis_service.py
-1
View File
@@ -111,7 +111,6 @@ class TerrainAnalysisService:
try:
import numpy as np
import rasterio
from rasterio.features import geometry_mask
from rasterio.mask import mask
except ImportError as exc:
raise AppError(
backend/app/services/thematic_raster_acquisition_service.py
+1 -2
View File
@@ -11,7 +11,7 @@ from pathlib import Path
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
from urllib.request import Request
from uuid import UUID
from geoalchemy2.shape import to_shape
@@ -407,7 +407,6 @@ class ThematicRasterAcquisitionService:
if len(coverages) == 1:
return coverages[0]
try:
import rasterio
from rasterio.io import MemoryFile
from rasterio.merge import merge
except ImportError as exc:
backend/app/services/thematic_raster_analysis_service.py
-1
View File
@@ -96,7 +96,6 @@ class ThematicRasterAnalysisService:
try:
import numpy as np
import rasterio
from rasterio.features import geometry_mask
from rasterio.mask import mask
except ImportError as exc:
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for thematic raster analysis", status_code=503) from exc
backend/app/services/tile_manifest_service.py
+485
View File
@@ -0,0 +1,485 @@
from __future__ import annotations
from hashlib import sha256
import json
from math import isfinite
from pathlib import Path
from typing import Any
from geoalchemy2.shape import to_shape
from pyproj import CRS, Transformer
from shapely.geometry import box
from shapely.ops import transform as shapely_transform
from shapely.ops import unary_union
from app.core.errors import AppError
from app.models import Area, Dataset, DatasetVersion
from app.services.storage_service import StorageService
class TileManifestService:
"""Versioned provenance and integrity contract for inference tile sets."""
CONTRACT_KEY = "geointel.raster.tile-manifest"
CONTRACT_VERSION = "2.0.0"
_BINDING_FIELDS = (
"source_dataset_id",
"source_dataset_checksum_sha256",
"source_dataset_size_bytes",
"source_registry_id",
"source_snapshot_id",
"source_snapshot_checksum_sha256",
"data_contract_key",
"data_contract_version",
"source_version",
"dataset_version_id",
"dataset_version",
"dataset_version_checksum_sha256",
"source_area_id",
"source_area_geometry_sha256",
)
_REQUIRED_INFERENCE_BINDING_FIELDS = (
"source_dataset_checksum_sha256",
"source_registry_id",
"source_snapshot_id",
"source_snapshot_checksum_sha256",
"data_contract_key",
"data_contract_version",
"dataset_version_id",
"dataset_version",
"dataset_version_checksum_sha256",
)
_CHECKSUM_FIELDS = (
"source_dataset_checksum_sha256",
"source_snapshot_checksum_sha256",
"dataset_version_checksum_sha256",
)
@staticmethod
def file_sha256(path: str | Path) -> str:
digest = sha256()
with Path(path).open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
@staticmethod
def _latest_dataset_version(dataset: Dataset) -> DatasetVersion | None:
versions = list(dataset.versions or [])
if not versions:
return None
return max(versions, key=lambda item: (int(item.version or 0), str(item.id or "")))
@staticmethod
def _source_snapshot_checksum(dataset: Dataset) -> str | None:
snapshot = dataset.source_snapshot
checksum = getattr(snapshot, "checksum_sha256", None) if snapshot is not None else None
return str(checksum).lower() if checksum else None
@staticmethod
def _area_geometry_binding(db, dataset: Dataset) -> tuple[str | None, str | None]:
if dataset.area_id is None:
return None, None
area = db.get(Area, dataset.area_id)
if area is None or area.geometry is None:
return str(dataset.area_id), None
geometry = to_shape(area.geometry)
return str(dataset.area_id), sha256(geometry.wkb).hexdigest()
@classmethod
def dataset_binding(cls, db, dataset: Dataset) -> dict[str, Any]:
version = cls._latest_dataset_version(dataset)
area_id, area_geometry_sha256 = cls._area_geometry_binding(db, dataset)
return {
"manifest_contract_key": cls.CONTRACT_KEY,
"manifest_contract_version": cls.CONTRACT_VERSION,
"source_dataset_id": str(dataset.id),
"source_raster_id": str(dataset.id),
"source_dataset_checksum_sha256": (
str(dataset.checksum_sha256).lower() if dataset.checksum_sha256 else None
),
"source_dataset_size_bytes": dataset.size_bytes,
"source_registry_id": str(dataset.source_registry_id) if dataset.source_registry_id else None,
"source_snapshot_id": str(dataset.source_snapshot_id) if dataset.source_snapshot_id else None,
"source_snapshot_checksum_sha256": cls._source_snapshot_checksum(dataset),
"data_contract_key": dataset.data_contract_key,
"data_contract_version": dataset.data_contract_version,
"source_version": dataset.source_version,
"dataset_version_id": str(version.id) if version is not None and version.id else None,
"dataset_version": int(version.version) if version is not None and version.version is not None else None,
"dataset_version_checksum_sha256": (
str(version.checksum_sha256).lower()
if version is not None and version.checksum_sha256
else None
),
"source_area_id": area_id,
"source_area_geometry_sha256": area_geometry_sha256,
}
@staticmethod
def tile_integrity(path: str | Path) -> dict[str, Any]:
resolved = Path(path)
return {
"size_bytes": resolved.stat().st_size,
"sha256": TileManifestService.file_sha256(resolved),
}
@staticmethod
def _error(
error_prefix: str,
suffix: str,
message: str,
*,
details: dict[str, Any] | None = None,
) -> AppError:
return AppError(
code=f"{error_prefix}_TILE_MANIFEST_{suffix}",
message=message,
details=details,
status_code=422,
)
@staticmethod
def _bounds_values(value: Any) -> tuple[float, float, float, float] | None:
if isinstance(value, dict):
aliases = (
("min_x", "min_y", "max_x", "max_y"),
("minx", "miny", "maxx", "maxy"),
("left", "bottom", "right", "top"),
)
selected = next(
([value.get(key) for key in keys] for keys in aliases if all(key in value for key in keys)),
None,
)
elif isinstance(value, (list, tuple)) and len(value) == 4:
selected = list(value)
else:
return None
try:
bounds = tuple(float(item) for item in selected) if selected is not None else None
except (TypeError, ValueError):
return None
if bounds is None or not all(isfinite(item) for item in bounds):
return None
if bounds[0] >= bounds[2] or bounds[1] >= bounds[3]:
return None
return bounds
@staticmethod
def _to_epsg4326(bounds: tuple[float, float, float, float], raw_crs: Any):
source_crs = CRS.from_user_input(raw_crs)
geometry = box(*bounds)
if not source_crs.equals(CRS.from_epsg(4326)):
transformer = Transformer.from_crs(source_crs, "EPSG:4326", always_xy=True)
geometry = shapely_transform(transformer.transform, geometry)
if geometry.is_empty or not geometry.is_valid:
raise ValueError("Bounds do not form a valid transformed geometry")
if not all(isfinite(float(value)) for value in geometry.bounds):
raise ValueError("Bounds transform to non-finite coordinates")
return geometry
@classmethod
def _manifest_coverage(cls, manifest: dict[str, Any], *, error_prefix: str):
tiles = manifest.get("tiles")
default_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs")
parts = []
for index, tile in enumerate(tiles if isinstance(tiles, list) else []):
if not isinstance(tile, dict):
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Tile manifest entries must be objects with explicit spatial metadata.",
details={"tile_index": index},
)
bounds = cls._bounds_values(tile.get("bounds"))
raw_crs = tile.get("crs") or default_crs
if bounds is None or not raw_crs:
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Every inference tile requires finite bounds and an explicit CRS.",
details={"tile_index": index},
)
try:
parts.append(cls._to_epsg4326(bounds, raw_crs))
except Exception as exc:
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Inference tile bounds or CRS could not be normalized to EPSG:4326.",
details={"tile_index": index, "reason": str(exc)},
) from exc
coverage = unary_union(parts)
if coverage.is_empty or not coverage.is_valid:
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Inference tile union is empty or invalid.",
)
min_x, min_y, max_x, max_y = coverage.bounds
if min_x < -180 or min_y < -90 or max_x > 180 or max_y > 90:
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Inference tile union falls outside EPSG:4326 bounds.",
details={"bounds": list(coverage.bounds)},
)
return coverage
@classmethod
def _validate_binding(cls, db, dataset: Dataset, manifest: dict[str, Any], *, error_prefix: str) -> dict[str, Any]:
if (
manifest.get("manifest_contract_key") != cls.CONTRACT_KEY
or manifest.get("manifest_contract_version") != cls.CONTRACT_VERSION
):
raise cls._error(
error_prefix,
"PROVENANCE_MISMATCH",
"Inference requires a versioned GeoIntel tile-manifest contract.",
details={
"required_contract": f"{cls.CONTRACT_KEY}@{cls.CONTRACT_VERSION}",
"manifest_contract": (
f"{manifest.get('manifest_contract_key')}@{manifest.get('manifest_contract_version')}"
),
},
)
expected = cls.dataset_binding(db, dataset)
missing = [
field
for field in cls._REQUIRED_INFERENCE_BINDING_FIELDS
if expected.get(field) in {None, ""}
]
invalid_checksums = [
field
for field in cls._CHECKSUM_FIELDS
if len(str(expected.get(field) or "")) != 64
or any(character not in "0123456789abcdef" for character in str(expected.get(field) or "").lower())
]
if missing or invalid_checksums:
raise cls._error(
error_prefix,
"PROVENANCE_MISMATCH",
"The requested Dataset lacks complete immutable provenance for inference tiling.",
details={
"missing_fields": missing,
"invalid_checksum_fields": invalid_checksums,
},
)
manifest_dataset_id = manifest.get("source_dataset_id") or manifest.get("source_raster_id")
if str(manifest_dataset_id or "") != expected["source_dataset_id"]:
raise cls._error(
error_prefix,
"DATASET_MISMATCH",
"Tile manifest belongs to a different raster Dataset.",
details={
"requested_dataset_id": expected["source_dataset_id"],
"manifest_dataset_id": manifest_dataset_id,
},
)
mismatches = {}
for field in cls._BINDING_FIELDS:
expected_value = expected.get(field)
if expected_value is None or field == "source_dataset_id":
continue
observed_value = manifest.get(field)
if str(observed_value) != str(expected_value):
mismatches[field] = {"expected": expected_value, "observed": observed_value}
if mismatches:
raise cls._error(
error_prefix,
"PROVENANCE_MISMATCH",
"Tile manifest provenance no longer matches the requested Dataset snapshot.",
details={"mismatches": mismatches},
)
return expected
@classmethod
def _validate_tile_files(
cls,
manifest: dict[str, Any],
manifest_path: Path,
*,
settings,
error_prefix: str,
) -> list[str]:
resolved_paths: list[str] = []
seen_paths: set[Path] = set()
for index, tile in enumerate(manifest["tiles"]):
raw_path = tile.get("path") if isinstance(tile, dict) else None
if not isinstance(raw_path, str) or not raw_path.strip():
raise cls._error(
error_prefix,
"TILE_INTEGRITY_MISMATCH",
"Every inference tile requires a path and immutable integrity evidence.",
details={"tile_index": index},
)
candidate = Path(raw_path).expanduser()
if not candidate.is_absolute():
candidate = manifest_path.parent / candidate
candidate = StorageService.assert_within_storage_root(
candidate,
label="raster tile",
settings=settings,
)
if not candidate.is_file():
raise cls._error(
error_prefix,
"TILE_INTEGRITY_MISMATCH",
"An inference tile referenced by the manifest does not exist.",
details={"tile_index": index, "tile_path": str(candidate)},
)
if candidate in seen_paths:
raise cls._error(
error_prefix,
"TILE_INTEGRITY_MISMATCH",
"A tile path occurs more than once in the inference manifest.",
details={"tile_index": index, "tile_path": str(candidate)},
)
seen_paths.add(candidate)
observed_size = candidate.stat().st_size
expected_size = tile.get("size_bytes")
expected_checksum = str(tile.get("sha256") or "").strip().lower()
if expected_size != observed_size or len(expected_checksum) != 64:
raise cls._error(
error_prefix,
"TILE_INTEGRITY_MISMATCH",
"Tile size/checksum evidence is missing or no longer matches the staged file.",
details={
"tile_index": index,
"expected_size_bytes": expected_size,
"observed_size_bytes": observed_size,
},
)
observed_checksum = cls.file_sha256(candidate)
if observed_checksum != expected_checksum:
raise cls._error(
error_prefix,
"TILE_INTEGRITY_MISMATCH",
"Tile checksum no longer matches the immutable manifest evidence.",
details={
"tile_index": index,
"expected_sha256": expected_checksum,
"observed_sha256": observed_checksum,
},
)
resolved_paths.append(str(candidate))
declared_count = manifest.get("count")
if declared_count != len(resolved_paths):
raise cls._error(
error_prefix,
"TILE_INTEGRITY_MISMATCH",
"Tile manifest count does not match its tile records.",
details={"declared_count": declared_count, "tile_count": len(resolved_paths)},
)
return resolved_paths
@classmethod
def _validate_scope(
cls,
db,
dataset: Dataset,
manifest: dict[str, Any],
coverage,
*,
error_prefix: str,
) -> None:
manifest_bounds = cls._bounds_values(manifest.get("bounds"))
manifest_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs")
dataset_bounds = cls._bounds_values(dataset.bounds_json)
if dataset_bounds is None and isinstance(dataset.metadata_json, dict):
dataset_bounds = cls._bounds_values(
dataset.metadata_json.get("bounds_json") or dataset.metadata_json.get("bounds")
)
if manifest_bounds is None or not manifest_crs or dataset_bounds is None or not dataset.crs:
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Dataset and tile manifest require explicit CRS and finite bounds for inference.",
)
try:
manifest_extent = cls._to_epsg4326(manifest_bounds, manifest_crs)
dataset_extent = cls._to_epsg4326(dataset_bounds, dataset.crs)
except Exception as exc:
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Dataset or manifest bounds could not be normalized to EPSG:4326.",
details={"reason": str(exc)},
) from exc
tolerance = max(dataset_extent.bounds[2] - dataset_extent.bounds[0], dataset_extent.bounds[3] - dataset_extent.bounds[1]) * 1e-7 + 1e-10
if not manifest_extent.buffer(tolerance).covers(coverage):
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Tile union exceeds the extent declared by its manifest.",
details={"tile_union_bounds": list(coverage.bounds), "manifest_bounds": list(manifest_extent.bounds)},
)
if not dataset_extent.buffer(tolerance).covers(coverage):
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Tile union exceeds the persisted Dataset extent.",
details={"tile_union_bounds": list(coverage.bounds), "dataset_bounds": list(dataset_extent.bounds)},
)
if dataset.area_id is not None:
area = db.get(Area, dataset.area_id)
if area is None or area.geometry is None:
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Dataset references an Area that is unavailable for inference-scope validation.",
details={"area_id": str(dataset.area_id)},
)
area_geometry = to_shape(area.geometry)
if area_geometry.is_empty or not area_geometry.is_valid or not coverage.intersects(area_geometry):
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Tile union does not overlap the persisted Dataset Area.",
details={"area_id": str(dataset.area_id), "tile_union_bounds": list(coverage.bounds)},
)
@classmethod
def validate_for_inference(
cls,
db,
dataset: Dataset,
manifest: dict[str, Any],
*,
manifest_path: str | Path,
settings,
error_prefix: str,
) -> dict[str, Any]:
resolved_manifest_path = StorageService.assert_within_storage_root(
manifest_path,
label="tile manifest",
settings=settings,
)
expected = cls._validate_binding(db, dataset, manifest, error_prefix=error_prefix)
resolved_paths = cls._validate_tile_files(
manifest,
resolved_manifest_path,
settings=settings,
error_prefix=error_prefix,
)
coverage = cls._manifest_coverage(manifest, error_prefix=error_prefix)
cls._validate_scope(db, dataset, manifest, coverage, error_prefix=error_prefix)
return {
"manifest_contract_key": cls.CONTRACT_KEY,
"manifest_contract_version": cls.CONTRACT_VERSION,
"manifest_path": str(resolved_manifest_path),
"manifest_sha256": cls.file_sha256(resolved_manifest_path),
"source_dataset_id": expected["source_dataset_id"],
"source_dataset_checksum_sha256": expected.get("source_dataset_checksum_sha256"),
"source_snapshot_id": expected.get("source_snapshot_id"),
"dataset_version_id": expected.get("dataset_version_id"),
"source_area_id": expected.get("source_area_id"),
"tile_count": len(resolved_paths),
"tile_union_bounds_epsg4326": [float(value) for value in coverage.bounds],
}
def canonical_manifest_json(payload: dict[str, Any]) -> str:
"""Stable serializer shared by the writer and manifest-hash tests."""
return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
backend/app/utils/geometry.py
+82 -21
View File
@@ -1,43 +1,101 @@
from __future__ import annotations
from math import isfinite
from numbers import Real
from typing import Any
from pyproj import Transformer
from shapely import force_2d
from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, box, shape
from pyproj import CRS, Transformer
from shapely import force_2d, get_coordinates
from shapely.geometry import MultiPolygon, box, shape
from shapely.ops import transform
from shapely.validation import make_valid
# This is a deliberately broad guard envelope around Belgium and the Belgian
# North Sea. Exact legal/regional clipping remains the responsibility of the
# persisted coverage Areas; this boundary prevents an AOI with valid-looking
# but globally misplaced coordinates from entering the workbench.
BELGIUM_AND_NORTH_SEA_GUARD_BOUNDS = (1.5, 48.5, 7.5, 52.5)
def _raw_coordinates_are_finite(value: Any) -> bool:
if isinstance(value, (list, tuple)):
return bool(value) and all(_raw_coordinates_are_finite(item) for item in value)
return isinstance(value, Real) and not isinstance(value, bool) and isfinite(float(value))
def normalize_to_multipolygon(raw_geometry: dict[str, Any]) -> MultiPolygon:
geom = force_2d(shape(raw_geometry))
if isinstance(raw_geometry, dict) and "coordinates" in raw_geometry:
if not _raw_coordinates_are_finite(raw_geometry["coordinates"]):
raise ValueError("Geometry coordinates must be finite numbers")
try:
geom = force_2d(shape(raw_geometry))
except Exception as exc:
raise ValueError("Geometry is not valid GeoJSON") from exc
coordinates = get_coordinates(geom, include_z=False)
if coordinates.size == 0 or not all(isfinite(float(value)) for row in coordinates for value in row):
raise ValueError("Geometry coordinates must be finite numbers")
if geom.is_empty:
raise ValueError("Geometry is empty")
if not geom.is_valid:
geom = make_valid(geom)
if not geom.is_valid:
raise ValueError("Geometry is invalid and could not be repaired")
raise ValueError("Geometry is invalid")
if geom.geom_type == "Polygon":
return MultiPolygon([geom])
if geom.geom_type == "MultiPolygon":
return MultiPolygon(geom.geoms)
if isinstance(geom, GeometryCollection):
polygons = [g for g in geom.geoms if isinstance(g, Polygon)]
multipolygons = [g for g in geom.geoms if g.geom_type == "MultiPolygon"]
if not polygons and not multipolygons:
raise ValueError("Only polygon geometries are supported for AOI")
normalized = []
normalized.extend(polygons)
for mp in multipolygons:
normalized.extend(mp.geoms)
return MultiPolygon(normalized)
raise ValueError("Only Polygon or MultiPolygon geometries are accepted")
def normalize_area_to_epsg4326(
raw_geometry: dict[str, Any],
source_crs: str,
) -> tuple[MultiPolygon, str]:
"""Validate an AOI and normalize its declared CRS to canonical WGS84.
The returned CRS string preserves the caller's declaration for provenance;
the returned geometry is always finite, polygonal and stored as EPSG:4326.
"""
declared_crs = str(source_crs or "").strip()
if not declared_crs:
raise ValueError("Area CRS is required")
try:
parsed_crs = CRS.from_user_input(declared_crs)
except Exception as exc:
raise ValueError("Area CRS is unknown or invalid") from exc
if not (parsed_crs.is_geographic or parsed_crs.is_projected):
raise ValueError("Area CRS must be a geographic or projected two-dimensional CRS")
if len(parsed_crs.axis_info) != 2:
raise ValueError("Area CRS must have exactly two spatial axes")
geometry = normalize_to_multipolygon(raw_geometry)
target_crs = CRS.from_epsg(4326)
if not parsed_crs.equals(target_crs):
try:
transformer = Transformer.from_crs(parsed_crs, target_crs, always_xy=True)
geometry = normalize_to_multipolygon(
transform(transformer.transform, geometry).__geo_interface__
)
except ValueError:
raise
except Exception as exc:
raise ValueError("Area geometry could not be transformed to EPSG:4326") from exc
min_x, min_y, max_x, max_y = geometry.bounds
if not all(isfinite(value) for value in (min_x, min_y, max_x, max_y)):
raise ValueError("Transformed area geometry contains non-finite coordinates")
world_bounds = (-180.0, -90.0, 180.0, 90.0)
if min_x < world_bounds[0] or min_y < world_bounds[1] or max_x > world_bounds[2] or max_y > world_bounds[3]:
raise ValueError("Transformed area geometry falls outside the EPSG:4326 coordinate domain")
guard = box(*BELGIUM_AND_NORTH_SEA_GUARD_BOUNDS)
if not guard.intersects(geometry):
raise ValueError("Area geometry falls outside Belgium and the Belgian North Sea workbench domain")
if not guard.covers(geometry):
raise ValueError("Area geometry must remain within the Belgium and Belgian North Sea workbench domain")
return geometry, declared_crs
def area_bounds_multipolygon(geom: MultiPolygon):
return {
"min_x": float(geom.bounds[0]),
@@ -52,7 +110,10 @@ def area_m2(geom: MultiPolygon) -> float:
Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True).transform,
geom,
)
return float(projected.area)
result = float(projected.area)
if not isfinite(result) or result <= 0:
raise ValueError("Area geometry must have a finite positive surface")
return result
def geometry_bbox_polygon(geom: MultiPolygon):