fix(platform): govern geospatial analysis and raster handoffs
This commit is contained in:
@@ -1,8 +1,9 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends
|
from fastapi import APIRouter, Depends, Request
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.api.guest_scope import assert_guest_project_scope
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.db.session import get_db
|
from app.db.session import get_db
|
||||||
from app.models import Dataset
|
from app.models import Dataset
|
||||||
@@ -18,12 +19,15 @@ router = APIRouter(prefix="/analysis", tags=["analysis"])
|
|||||||
@router.post("/change-detection", response_model=Envelope[JobRead])
|
@router.post("/change-detection", response_model=Envelope[JobRead])
|
||||||
def run_change_detection(
|
def run_change_detection(
|
||||||
payload: ChangeDetectionRequest,
|
payload: ChangeDetectionRequest,
|
||||||
|
request: Request,
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
) -> dict:
|
) -> dict:
|
||||||
source_dataset = db.get(Dataset, payload.source_dataset_id)
|
source_dataset = db.get(Dataset, payload.source_dataset_id)
|
||||||
if not source_dataset:
|
if not source_dataset:
|
||||||
raise AppError(code="DATASET_NOT_FOUND", message="Source dataset not found", status_code=404)
|
raise AppError(code="DATASET_NOT_FOUND", message="Source dataset not found", status_code=404)
|
||||||
|
assert_guest_project_scope(request, source_dataset.project_id)
|
||||||
ChangeDetectionService._get_project_vector_dataset(db, payload.source_dataset_id, source_dataset.project_id, "Source")
|
ChangeDetectionService._get_project_vector_dataset(db, payload.source_dataset_id, source_dataset.project_id, "Source")
|
||||||
|
ChangeDetectionService._get_project_vector_dataset(db, payload.target_dataset_id, source_dataset.project_id, "Target")
|
||||||
job = JobService.run_sync_job(
|
job = JobService.run_sync_job(
|
||||||
db=db,
|
db=db,
|
||||||
project_id=source_dataset.project_id,
|
project_id=source_dataset.project_id,
|
||||||
|
|||||||
@@ -5,13 +5,13 @@ from datetime import datetime
|
|||||||
from typing import Any
|
from typing import Any
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Response
|
from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, Response
|
||||||
from fastapi import UploadFile
|
from fastapi import UploadFile
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
from app.models import Area, Project
|
from app.core.config import get_settings
|
||||||
|
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.db.session import get_db
|
from app.db.session import get_db
|
||||||
|
from app.models import Area, Project
|
||||||
from app.schemas import (
|
from app.schemas import (
|
||||||
BathymetryPartitionFinalizationResult,
|
BathymetryPartitionFinalizationResult,
|
||||||
BathymetrySourceProbeRead,
|
BathymetrySourceProbeRead,
|
||||||
@@ -1172,11 +1172,14 @@ def raster_tile_dataset(
|
|||||||
project_id: UUID,
|
project_id: UUID,
|
||||||
dataset_id: UUID,
|
dataset_id: UUID,
|
||||||
payload: RasterTileRequest,
|
payload: RasterTileRequest,
|
||||||
|
request: Request,
|
||||||
db: Session = Depends(get_db),
|
db: Session = Depends(get_db),
|
||||||
):
|
):
|
||||||
dataset = DatasetService.get_dataset(db, dataset_id)
|
dataset = DatasetService.get_dataset(db, dataset_id)
|
||||||
if dataset.project_id != project_id:
|
if dataset.project_id != project_id:
|
||||||
raise HTTPException(status_code=404, detail="Dataset not found")
|
raise HTTPException(status_code=404, detail="Dataset not found")
|
||||||
|
principal = getattr(request.state, "auth_principal", None)
|
||||||
|
guest_max_tiles = get_settings().yolo_max_tiles if getattr(principal, "role", None) == "guest" else None
|
||||||
job = _run_job_sync(
|
job = _run_job_sync(
|
||||||
db=db,
|
db=db,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
@@ -1189,6 +1192,7 @@ def raster_tile_dataset(
|
|||||||
tile_size=payload.tile_size,
|
tile_size=payload.tile_size,
|
||||||
overlap=payload.overlap,
|
overlap=payload.overlap,
|
||||||
output_name=payload.output_name,
|
output_name=payload.output_name,
|
||||||
|
max_tiles=guest_max_tiles,
|
||||||
),
|
),
|
||||||
)
|
)
|
||||||
return envelope(job)
|
return envelope(job)
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
from app.models import *
|
from app.models import * # noqa: F403 - legacy compatibility shim re-exports the package API
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ class AreaCreate(BaseModel):
|
|||||||
|
|
||||||
class AreaUpdate(BaseModel):
|
class AreaUpdate(BaseModel):
|
||||||
name: str | None = None
|
name: str | None = None
|
||||||
|
geometry: dict | None = None
|
||||||
crs: str | None = None
|
crs: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -42,6 +42,10 @@ class ModelAssetRead(BaseModel):
|
|||||||
size_bytes: int
|
size_bytes: int
|
||||||
sha256: str
|
sha256: str
|
||||||
active: bool
|
active: bool
|
||||||
|
runtime_available: bool
|
||||||
|
runtime_status: str
|
||||||
|
governed_validation_status: str
|
||||||
|
promotion_status: str
|
||||||
status: str
|
status: str
|
||||||
limitation_message: str
|
limitation_message: str
|
||||||
will_download_models: bool = False
|
will_download_models: bool = False
|
||||||
|
|||||||
@@ -12,7 +12,10 @@ from app.schemas.spw_terrain import SpwTerrainAcquireRequest
|
|||||||
from app.schemas.official_vector import OfficialVectorAcquireRequest
|
from app.schemas.official_vector import OfficialVectorAcquireRequest
|
||||||
from app.schemas.flood_hazard import FloodHazardAcquireRequest
|
from app.schemas.flood_hazard import FloodHazardAcquireRequest
|
||||||
from app.schemas.thematic_raster import ThematicRasterAcquireRequest
|
from app.schemas.thematic_raster import ThematicRasterAcquireRequest
|
||||||
from app.schemas.bathymetry import BathymetryProfileAcquireRequest, MdkBathymetryAcquireRequest
|
from app.schemas.bathymetry import (
|
||||||
|
BathymetryProfileAcquireRequest,
|
||||||
|
MdkBathymetryAcquireRequest,
|
||||||
|
)
|
||||||
from app.schemas.job import JobCreate
|
from app.schemas.job import JobCreate
|
||||||
from app.schemas.operations import VectorSelectionBBox
|
from app.schemas.operations import VectorSelectionBBox
|
||||||
from app.schemas.orthophoto import OrthophotoAcquireRequest
|
from app.schemas.orthophoto import OrthophotoAcquireRequest
|
||||||
@@ -20,12 +23,20 @@ from app.services.aoi_operation_service import AoiOperationService
|
|||||||
from app.services.grb_acquisition_service import GrbAcquisitionService
|
from app.services.grb_acquisition_service import GrbAcquisitionService
|
||||||
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
|
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
|
||||||
from app.services.spw_terrain_service import SpwTerrainService
|
from app.services.spw_terrain_service import SpwTerrainService
|
||||||
from app.services.official_vector_acquisition_service import OfficialVectorAcquisitionService
|
from app.services.official_vector_acquisition_service import (
|
||||||
|
OfficialVectorAcquisitionService,
|
||||||
|
)
|
||||||
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
|
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
|
||||||
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
|
from app.services.thematic_raster_acquisition_service import (
|
||||||
|
ThematicRasterAcquisitionService,
|
||||||
|
)
|
||||||
from app.services.walous_land_cover_service import WalousLandCoverService
|
from app.services.walous_land_cover_service import WalousLandCoverService
|
||||||
from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
|
from app.services.bathymetry_profile_acquisition_service import (
|
||||||
from app.services.mdk_bathymetry_acquisition_service import MdkBathymetryAcquisitionService
|
BathymetryProfileAcquisitionService,
|
||||||
|
)
|
||||||
|
from app.services.mdk_bathymetry_acquisition_service import (
|
||||||
|
MdkBathymetryAcquisitionService,
|
||||||
|
)
|
||||||
from app.services.job_service import JobService
|
from app.services.job_service import JobService
|
||||||
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
|
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
|
||||||
|
|
||||||
@@ -40,65 +51,202 @@ class AoiOperationExecutor:
|
|||||||
AoiOperationService._refresh_parent(db, operation_id)
|
AoiOperationService._refresh_parent(db, operation_id)
|
||||||
return AoiOperationService.read(db, project_id, operation_id)
|
return AoiOperationService.read(db, project_id, operation_id)
|
||||||
operation = db.get(AoiOperation, operation_id)
|
operation = db.get(AoiOperation, operation_id)
|
||||||
child = JobService.create_job(db, JobCreate(
|
child = JobService.create_job(
|
||||||
job_type=f"aoi.{operation.operation_type}.partition",
|
db,
|
||||||
project_id=project_id,
|
JobCreate(
|
||||||
parameters_json={
|
job_type=f"aoi.{operation.operation_type}.partition",
|
||||||
"aoi_operation_id": str(operation_id),
|
project_id=project_id,
|
||||||
"partition_id": str(partition.id),
|
parameters_json={
|
||||||
"partition_key": partition.partition_key,
|
"aoi_operation_id": str(operation_id),
|
||||||
"provider_key": partition.provider_key,
|
"partition_id": str(partition.id),
|
||||||
"product_key": partition.product_key,
|
"partition_key": partition.partition_key,
|
||||||
},
|
"provider_key": partition.provider_key,
|
||||||
))
|
"product_key": partition.product_key,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
)
|
||||||
partition = db.get(AoiOperationPartition, partition.id)
|
partition = db.get(AoiOperationPartition, partition.id)
|
||||||
partition.child_job_id = child.id
|
partition.child_job_id = child.id
|
||||||
db.add(partition); db.commit()
|
db.add(partition)
|
||||||
|
db.commit()
|
||||||
JobService.mark_running(db, child.id)
|
JobService.mark_running(db, child.id)
|
||||||
try:
|
try:
|
||||||
result = AoiOperationExecutor._dispatch(db, project_id, operation, partition)
|
result = AoiOperationExecutor._dispatch(
|
||||||
output_id = result.get("output_dataset_id") if isinstance(result, dict) else None
|
db, project_id, operation, partition
|
||||||
JobService.mark_success(db, child.id, result=result, output_dataset_id=UUID(str(output_id)) if output_id else None)
|
)
|
||||||
return AoiOperationService.complete(db, project_id, operation_id, partition.id, result)
|
output_id = (
|
||||||
|
result.get("output_dataset_id") if isinstance(result, dict) else None
|
||||||
|
)
|
||||||
|
JobService.mark_success(
|
||||||
|
db,
|
||||||
|
child.id,
|
||||||
|
result=result,
|
||||||
|
output_dataset_id=UUID(str(output_id)) if output_id else None,
|
||||||
|
)
|
||||||
|
return AoiOperationService.complete(
|
||||||
|
db, project_id, operation_id, partition.id, result
|
||||||
|
)
|
||||||
except AppError as exc:
|
except AppError as exc:
|
||||||
JobService.mark_failed(db, child.id, exc.message, {"code": exc.code, "details": exc.details})
|
JobService.mark_failed(
|
||||||
return AoiOperationService.fail(db, project_id, operation_id, partition.id, exc.message, AoiOperationExecutor._retryable(exc), {"code": exc.code, "details": exc.details})
|
db, child.id, exc.message, {"code": exc.code, "details": exc.details}
|
||||||
|
)
|
||||||
|
return AoiOperationService.fail(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
operation_id,
|
||||||
|
partition.id,
|
||||||
|
exc.message,
|
||||||
|
AoiOperationExecutor._retryable(exc),
|
||||||
|
{"code": exc.code, "details": exc.details},
|
||||||
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
try:
|
try:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
JobService.mark_failed(db, child.id, "Unexpected partition execution error", {"code": "AOI_PARTITION_INTERNAL_ERROR"})
|
JobService.mark_failed(
|
||||||
|
db,
|
||||||
|
child.id,
|
||||||
|
"Unexpected partition execution error",
|
||||||
|
{"code": "AOI_PARTITION_INTERNAL_ERROR"},
|
||||||
|
)
|
||||||
finally:
|
finally:
|
||||||
AoiOperationService.fail(db, project_id, operation_id, partition.id, "Unexpected partition execution error", True, {"code": "AOI_PARTITION_INTERNAL_ERROR"})
|
AoiOperationService.fail(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
operation_id,
|
||||||
|
partition.id,
|
||||||
|
"Unexpected partition execution error",
|
||||||
|
True,
|
||||||
|
{"code": "AOI_PARTITION_INTERNAL_ERROR"},
|
||||||
|
)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _dispatch(db, project_id: UUID, operation: AoiOperation, partition: AoiOperationPartition) -> dict:
|
def _dispatch(
|
||||||
|
db, project_id: UUID, operation: AoiOperation, partition: AoiOperationPartition
|
||||||
|
) -> dict:
|
||||||
geometry = to_shape(partition.geometry)
|
geometry = to_shape(partition.geometry)
|
||||||
min_x, min_y, max_x, max_y = geometry.bounds
|
min_x, min_y, max_x, max_y = geometry.bounds
|
||||||
bbox = VectorSelectionBBox(min_x=min_x, min_y=min_y, max_x=max_x, max_y=max_y, crs="EPSG:4326")
|
bbox = VectorSelectionBBox(
|
||||||
force_refresh = bool((operation.request_json or {}).get("parameters_json", {}).get("force_refresh", False))
|
min_x=min_x, min_y=min_y, max_x=max_x, max_y=max_y, crs="EPSG:4326"
|
||||||
|
)
|
||||||
|
force_refresh = bool(
|
||||||
|
(operation.request_json or {})
|
||||||
|
.get("parameters_json", {})
|
||||||
|
.get("force_refresh", False)
|
||||||
|
)
|
||||||
if partition.provider_key == "grb":
|
if partition.provider_key == "grb":
|
||||||
return GrbAcquisitionService.acquire(db, project_id, GrbAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
|
return GrbAcquisitionService.acquire(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
GrbAcquireRequest(
|
||||||
|
bbox=bbox,
|
||||||
|
area_id=operation.area_id,
|
||||||
|
product_key=partition.product_key,
|
||||||
|
force_refresh=force_refresh,
|
||||||
|
),
|
||||||
|
)
|
||||||
if partition.provider_key == "orthophoto":
|
if partition.provider_key == "orthophoto":
|
||||||
return OrthophotoAcquisitionService.acquire(db, project_id, OrthophotoAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
|
return OrthophotoAcquisitionService.acquire(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
OrthophotoAcquireRequest(
|
||||||
|
bbox=bbox,
|
||||||
|
area_id=operation.area_id,
|
||||||
|
product_key=partition.product_key,
|
||||||
|
force_refresh=force_refresh,
|
||||||
|
),
|
||||||
|
)
|
||||||
if partition.provider_key == "dhmv":
|
if partition.provider_key == "dhmv":
|
||||||
return DhmvAcquisitionService.acquire(db, project_id, DhmvAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
|
return DhmvAcquisitionService.acquire(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
DhmvAcquireRequest(
|
||||||
|
bbox=bbox,
|
||||||
|
area_id=operation.area_id,
|
||||||
|
product_key=partition.product_key,
|
||||||
|
force_refresh=force_refresh,
|
||||||
|
),
|
||||||
|
)
|
||||||
if partition.provider_key == "spw_terrain":
|
if partition.provider_key == "spw_terrain":
|
||||||
return SpwTerrainService.acquire(db, project_id, SpwTerrainAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
|
return SpwTerrainService.acquire(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
SpwTerrainAcquireRequest(
|
||||||
|
bbox=bbox,
|
||||||
|
area_id=operation.area_id,
|
||||||
|
product_key=partition.product_key,
|
||||||
|
force_refresh=force_refresh,
|
||||||
|
),
|
||||||
|
)
|
||||||
if partition.provider_key == "official_vector":
|
if partition.provider_key == "official_vector":
|
||||||
return OfficialVectorAcquisitionService.acquire(db, project_id, OfficialVectorAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
|
return OfficialVectorAcquisitionService.acquire(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
OfficialVectorAcquireRequest(
|
||||||
|
bbox=bbox,
|
||||||
|
area_id=operation.area_id,
|
||||||
|
product_key=partition.product_key,
|
||||||
|
force_refresh=force_refresh,
|
||||||
|
),
|
||||||
|
)
|
||||||
if partition.provider_key == "flood_hazard":
|
if partition.provider_key == "flood_hazard":
|
||||||
return FloodHazardAcquisitionService.acquire(db, project_id, FloodHazardAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
|
return FloodHazardAcquisitionService.acquire(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
FloodHazardAcquireRequest(
|
||||||
|
bbox=bbox,
|
||||||
|
area_id=operation.area_id,
|
||||||
|
product_key=partition.product_key,
|
||||||
|
force_refresh=force_refresh,
|
||||||
|
),
|
||||||
|
)
|
||||||
if partition.provider_key == "thematic_raster":
|
if partition.provider_key == "thematic_raster":
|
||||||
return ThematicRasterAcquisitionService.acquire(db, project_id, ThematicRasterAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
|
return ThematicRasterAcquisitionService.acquire(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
ThematicRasterAcquireRequest(
|
||||||
|
bbox=bbox,
|
||||||
|
area_id=operation.area_id,
|
||||||
|
product_key=partition.product_key,
|
||||||
|
force_refresh=force_refresh,
|
||||||
|
),
|
||||||
|
)
|
||||||
if partition.provider_key == "walous":
|
if partition.provider_key == "walous":
|
||||||
return WalousLandCoverService.acquire(db, project_id, ThematicRasterAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
|
return WalousLandCoverService.acquire(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
ThematicRasterAcquireRequest(
|
||||||
|
bbox=bbox,
|
||||||
|
area_id=operation.area_id,
|
||||||
|
product_key=partition.product_key,
|
||||||
|
force_refresh=force_refresh,
|
||||||
|
),
|
||||||
|
)
|
||||||
if partition.provider_key == "bathymetry_profiles":
|
if partition.provider_key == "bathymetry_profiles":
|
||||||
return BathymetryProfileAcquisitionService.acquire(db, project_id, BathymetryProfileAcquireRequest(bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh))
|
return BathymetryProfileAcquisitionService.acquire(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
BathymetryProfileAcquireRequest(
|
||||||
|
bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh
|
||||||
|
),
|
||||||
|
)
|
||||||
if partition.provider_key == "mdk_bathymetry":
|
if partition.provider_key == "mdk_bathymetry":
|
||||||
return MdkBathymetryAcquisitionService.acquire(db, project_id, MdkBathymetryAcquireRequest(bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh))
|
return MdkBathymetryAcquisitionService.acquire(
|
||||||
raise AppError(code="AOI_PROVIDER_UNSUPPORTED", message="No governed AOI executor is registered for this provider", details={"provider_key": partition.provider_key}, status_code=422)
|
db,
|
||||||
|
project_id,
|
||||||
|
MdkBathymetryAcquireRequest(
|
||||||
|
bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh
|
||||||
|
),
|
||||||
|
)
|
||||||
|
raise AppError(
|
||||||
|
code="AOI_PROVIDER_UNSUPPORTED",
|
||||||
|
message="No governed AOI executor is registered for this provider",
|
||||||
|
details={"provider_key": partition.provider_key},
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _retryable(error: AppError) -> bool:
|
def _retryable(error: AppError) -> bool:
|
||||||
return error.status_code >= 500 or error.code.endswith(("TIMEOUT", "UNAVAILABLE", "TLS_ERROR"))
|
return error.status_code >= 500 or error.code.endswith(
|
||||||
|
("TIMEOUT", "UNAVAILABLE", "TLS_ERROR")
|
||||||
|
)
|
||||||
|
|||||||
@@ -22,8 +22,11 @@ class AoiOperationService:
|
|||||||
_to_metric = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
_to_metric = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
||||||
_to_wgs84 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
|
_to_wgs84 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
|
||||||
SCOPE_AREA_NAMES = {
|
SCOPE_AREA_NAMES = {
|
||||||
"belgium": "Belgium land", "flanders": "Flanders", "wallonia": "Wallonia",
|
"belgium": "Belgium land",
|
||||||
"brussels": "Brussels-Capital Region", "belgian_north_sea": "Belgian part of the North Sea",
|
"flanders": "Flanders",
|
||||||
|
"wallonia": "Wallonia",
|
||||||
|
"brussels": "Brussels-Capital Region",
|
||||||
|
"belgian_north_sea": "Belgian part of the North Sea",
|
||||||
"territorial_sea": "Belgian territorial sea (0-12 nautical miles)",
|
"territorial_sea": "Belgian territorial sea (0-12 nautical miles)",
|
||||||
"exclusive_economic_zone": "Belgian exclusive economic zone beyond territorial sea",
|
"exclusive_economic_zone": "Belgian exclusive economic zone beyond territorial sea",
|
||||||
"continental_shelf": "Belgian continental shelf beyond territorial sea",
|
"continental_shelf": "Belgian continental shelf beyond territorial sea",
|
||||||
@@ -32,13 +35,19 @@ class AoiOperationService:
|
|||||||
@staticmethod
|
@staticmethod
|
||||||
def create(db, project_id: UUID, payload: AoiOperationCreate) -> dict:
|
def create(db, project_id: UUID, payload: AoiOperationCreate) -> dict:
|
||||||
if db.get(Project, project_id) is None:
|
if db.get(Project, project_id) is None:
|
||||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
raise AppError(
|
||||||
|
code="PROJECT_NOT_FOUND", message="Project not found", status_code=404
|
||||||
|
)
|
||||||
geometry = AoiOperationService._resolve_geometry(db, project_id, payload)
|
geometry = AoiOperationService._resolve_geometry(db, project_id, payload)
|
||||||
if payload.coverage_zone:
|
if payload.coverage_zone:
|
||||||
geometry = AoiOperationService._clip_to_zone(db, project_id, geometry, payload.coverage_zone)
|
geometry = AoiOperationService._clip_to_zone(
|
||||||
|
db, project_id, geometry, payload.coverage_zone
|
||||||
|
)
|
||||||
geometry = AoiOperationService._as_multipolygon(geometry)
|
geometry = AoiOperationService._as_multipolygon(geometry)
|
||||||
metric_geometry = transform(AoiOperationService._to_metric.transform, geometry)
|
metric_geometry = transform(AoiOperationService._to_metric.transform, geometry)
|
||||||
partition_side_m = AoiOperationService._partition_side(payload.provider_key, payload.max_partition_side_m)
|
partition_side_m = AoiOperationService._partition_side(
|
||||||
|
payload.provider_key, payload.max_partition_side_m
|
||||||
|
)
|
||||||
cells = AoiOperationService._partition(metric_geometry, partition_side_m)
|
cells = AoiOperationService._partition(metric_geometry, partition_side_m)
|
||||||
operation_id = uuid4()
|
operation_id = uuid4()
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
@@ -53,7 +62,9 @@ class AoiOperationService:
|
|||||||
plan_json={
|
plan_json={
|
||||||
"partition_strategy": "epsg31370_square_grid_intersection_v1",
|
"partition_strategy": "epsg31370_square_grid_intersection_v1",
|
||||||
"max_partition_side_m": partition_side_m,
|
"max_partition_side_m": partition_side_m,
|
||||||
"budget_source": "governed_provider_registry" if payload.max_partition_side_m is None else "stricter_operator_override",
|
"budget_source": "governed_provider_registry"
|
||||||
|
if payload.max_partition_side_m is None
|
||||||
|
else "stricter_operator_override",
|
||||||
"partition_count": len(cells),
|
"partition_count": len(cells),
|
||||||
"provider_key": payload.provider_key,
|
"provider_key": payload.provider_key,
|
||||||
"product_key": payload.product_key,
|
"product_key": payload.product_key,
|
||||||
@@ -65,13 +76,21 @@ class AoiOperationService:
|
|||||||
wgs84 = transform(AoiOperationService._to_wgs84.transform, cell)
|
wgs84 = transform(AoiOperationService._to_wgs84.transform, cell)
|
||||||
wgs84 = AoiOperationService._as_multipolygon(wgs84)
|
wgs84 = AoiOperationService._as_multipolygon(wgs84)
|
||||||
digest = sha256(wgs84.wkb).hexdigest()[:20]
|
digest = sha256(wgs84.wkb).hexdigest()[:20]
|
||||||
db.add(AoiOperationPartition(
|
db.add(
|
||||||
id=uuid4(), operation_id=operation_id,
|
AoiOperationPartition(
|
||||||
partition_key=f"{payload.provider_key}:{payload.product_key}:{ordinal:05d}:{digest}",
|
id=uuid4(),
|
||||||
provider_key=payload.provider_key, product_key=payload.product_key,
|
operation_id=operation_id,
|
||||||
ordinal=ordinal, status="queued", geometry=from_shape(wgs84, srid=4326),
|
partition_key=f"{payload.provider_key}:{payload.product_key}:{ordinal:05d}:{digest}",
|
||||||
attempt_count=0, max_attempts=payload.max_attempts, created_at=now,
|
provider_key=payload.provider_key,
|
||||||
))
|
product_key=payload.product_key,
|
||||||
|
ordinal=ordinal,
|
||||||
|
status="queued",
|
||||||
|
geometry=from_shape(wgs84, srid=4326),
|
||||||
|
attempt_count=0,
|
||||||
|
max_attempts=payload.max_attempts,
|
||||||
|
created_at=now,
|
||||||
|
)
|
||||||
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
return AoiOperationService.read(db, project_id, operation_id)
|
return AoiOperationService.read(db, project_id, operation_id)
|
||||||
|
|
||||||
@@ -79,13 +98,32 @@ class AoiOperationService:
|
|||||||
def _clip_to_zone(db, project_id: UUID, geometry, zone: str):
|
def _clip_to_zone(db, project_id: UUID, geometry, zone: str):
|
||||||
area_name = AoiOperationService.SCOPE_AREA_NAMES.get(zone)
|
area_name = AoiOperationService.SCOPE_AREA_NAMES.get(zone)
|
||||||
if area_name is None:
|
if area_name is None:
|
||||||
raise AppError(code="AOI_COVERAGE_ZONE_UNSUPPORTED", message="Unknown governed coverage zone", details={"coverage_zone": zone}, status_code=422)
|
raise AppError(
|
||||||
scope = db.query(Area).filter(Area.project_id == project_id, Area.name == area_name).first()
|
code="AOI_COVERAGE_ZONE_UNSUPPORTED",
|
||||||
|
message="Unknown governed coverage zone",
|
||||||
|
details={"coverage_zone": zone},
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
scope = (
|
||||||
|
db.query(Area)
|
||||||
|
.filter(Area.project_id == project_id, Area.name == area_name)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
if scope is None:
|
if scope is None:
|
||||||
raise AppError(code="AOI_COVERAGE_ZONE_NOT_MATERIALIZED", message="The governed coverage-zone geometry is not persisted in this project", details={"coverage_zone": zone}, status_code=409)
|
raise AppError(
|
||||||
|
code="AOI_COVERAGE_ZONE_NOT_MATERIALIZED",
|
||||||
|
message="The governed coverage-zone geometry is not persisted in this project",
|
||||||
|
details={"coverage_zone": zone},
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
clipped = geometry.intersection(to_shape(scope.geometry))
|
clipped = geometry.intersection(to_shape(scope.geometry))
|
||||||
if clipped.is_empty:
|
if clipped.is_empty:
|
||||||
raise AppError(code="AOI_OUTSIDE_PROVIDER_ZONE", message="The AOI does not intersect the provider coverage zone", details={"coverage_zone": zone}, status_code=422)
|
raise AppError(
|
||||||
|
code="AOI_OUTSIDE_PROVIDER_ZONE",
|
||||||
|
message="The AOI does not intersect the provider coverage zone",
|
||||||
|
details={"coverage_zone": zone},
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
return clipped
|
return clipped
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -94,19 +132,30 @@ class AoiOperationService:
|
|||||||
return MultiPolygon([geometry])
|
return MultiPolygon([geometry])
|
||||||
if isinstance(geometry, MultiPolygon):
|
if isinstance(geometry, MultiPolygon):
|
||||||
return geometry
|
return geometry
|
||||||
polygons = [part for part in getattr(geometry, "geoms", []) if isinstance(part, Polygon)]
|
polygons = [
|
||||||
|
part for part in getattr(geometry, "geoms", []) if isinstance(part, Polygon)
|
||||||
|
]
|
||||||
if not polygons:
|
if not polygons:
|
||||||
raise AppError(code="AOI_GEOMETRY_EMPTY", message="AOI contains no polygonal area after clipping", status_code=422)
|
raise AppError(
|
||||||
|
code="AOI_GEOMETRY_EMPTY",
|
||||||
|
message="AOI contains no polygonal area after clipping",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
return MultiPolygon(polygons)
|
return MultiPolygon(polygons)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _partition_side(provider_key: str, requested: float | None) -> float:
|
def _partition_side(provider_key: str, requested: float | None) -> float:
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
def raster_side(max_side_m: float, max_pixels: int, resolution_m: float) -> float:
|
|
||||||
|
def raster_side(
|
||||||
|
max_side_m: float, max_pixels: int, resolution_m: float
|
||||||
|
) -> float:
|
||||||
# Keep every square grid cell within both the provider's spatial
|
# Keep every square grid cell within both the provider's spatial
|
||||||
# extent limit and its decoded-pixel budget. The small safety
|
# extent limit and its decoded-pixel budget. The small safety
|
||||||
# margin absorbs ceil/edge rounding in the acquisition services.
|
# margin absorbs ceil/edge rounding in the acquisition services.
|
||||||
pixel_limited_side = math.sqrt(float(max_pixels)) * float(resolution_m) * 0.99
|
pixel_limited_side = (
|
||||||
|
math.sqrt(float(max_pixels)) * float(resolution_m) * 0.99
|
||||||
|
)
|
||||||
return min(float(max_side_m), pixel_limited_side)
|
return min(float(max_side_m), pixel_limited_side)
|
||||||
|
|
||||||
budgets = {
|
budgets = {
|
||||||
@@ -142,22 +191,37 @@ class AoiOperationService:
|
|||||||
"mdk_bathymetry": 20_000.0,
|
"mdk_bathymetry": 20_000.0,
|
||||||
}
|
}
|
||||||
if provider_key not in budgets:
|
if provider_key not in budgets:
|
||||||
raise AppError(code="AOI_PROVIDER_UNSUPPORTED", message="No governed partition budget is registered for this provider", details={"provider_key": provider_key}, status_code=422)
|
raise AppError(
|
||||||
|
code="AOI_PROVIDER_UNSUPPORTED",
|
||||||
|
message="No governed partition budget is registered for this provider",
|
||||||
|
details={"provider_key": provider_key},
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
governed = budgets[provider_key]
|
governed = budgets[provider_key]
|
||||||
return min(governed, float(requested)) if requested is not None else governed
|
return min(governed, float(requested)) if requested is not None else governed
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _resolve_geometry(db, project_id: UUID, payload: AoiOperationCreate):
|
def _resolve_geometry(db, project_id: UUID, payload: AoiOperationCreate):
|
||||||
if (payload.area_id is None) == (payload.bbox is None):
|
if (payload.area_id is None) == (payload.bbox is None):
|
||||||
raise AppError(code="AOI_SELECTION_REQUIRED", message="Provide exactly one area_id or bbox", status_code=422)
|
raise AppError(
|
||||||
|
code="AOI_SELECTION_REQUIRED",
|
||||||
|
message="Provide exactly one area_id or bbox",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
if payload.area_id is not None:
|
if payload.area_id is not None:
|
||||||
area = db.get(Area, payload.area_id)
|
area = db.get(Area, payload.area_id)
|
||||||
if area is None or area.project_id != project_id:
|
if area is None or area.project_id != project_id:
|
||||||
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
raise AppError(
|
||||||
|
code="AREA_NOT_FOUND", message="Area not found", status_code=404
|
||||||
|
)
|
||||||
return to_shape(area.geometry)
|
return to_shape(area.geometry)
|
||||||
bbox = payload.bbox
|
bbox = payload.bbox
|
||||||
if bbox is None or bbox.crs != "EPSG:4326":
|
if bbox is None or bbox.crs != "EPSG:4326":
|
||||||
raise AppError(code="INVALID_AOI_CRS", message="AOI bbox must use EPSG:4326", status_code=422)
|
raise AppError(
|
||||||
|
code="INVALID_AOI_CRS",
|
||||||
|
message="AOI bbox must use EPSG:4326",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
return box(bbox.min_x, bbox.min_y, bbox.max_x, bbox.max_y)
|
return box(bbox.min_x, bbox.min_y, bbox.max_x, bbox.max_y)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -166,11 +230,26 @@ class AoiOperationService:
|
|||||||
columns = max(1, math.ceil((max_x - min_x) / side_m))
|
columns = max(1, math.ceil((max_x - min_x) / side_m))
|
||||||
rows = max(1, math.ceil((max_y - min_y) / side_m))
|
rows = max(1, math.ceil((max_y - min_y) / side_m))
|
||||||
if columns * rows > AoiOperationService.MAX_PARTITIONS:
|
if columns * rows > AoiOperationService.MAX_PARTITIONS:
|
||||||
raise AppError(code="AOI_PARTITION_LIMIT_EXCEEDED", message="AOI requires too many bounded partitions", details={"candidate_count": columns * rows, "max_partitions": AoiOperationService.MAX_PARTITIONS}, status_code=422)
|
raise AppError(
|
||||||
|
code="AOI_PARTITION_LIMIT_EXCEEDED",
|
||||||
|
message="AOI requires too many bounded partitions",
|
||||||
|
details={
|
||||||
|
"candidate_count": columns * rows,
|
||||||
|
"max_partitions": AoiOperationService.MAX_PARTITIONS,
|
||||||
|
},
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
partitions = []
|
partitions = []
|
||||||
for row in range(rows):
|
for row in range(rows):
|
||||||
for column in range(columns):
|
for column in range(columns):
|
||||||
clipped = geometry.intersection(box(min_x + column * side_m, min_y + row * side_m, min(min_x + (column + 1) * side_m, max_x), min(min_y + (row + 1) * side_m, max_y)))
|
clipped = geometry.intersection(
|
||||||
|
box(
|
||||||
|
min_x + column * side_m,
|
||||||
|
min_y + row * side_m,
|
||||||
|
min(min_x + (column + 1) * side_m, max_x),
|
||||||
|
min(min_y + (row + 1) * side_m, max_y),
|
||||||
|
)
|
||||||
|
)
|
||||||
if not clipped.is_empty and clipped.area > 0:
|
if not clipped.is_empty and clipped.area > 0:
|
||||||
partitions.append(clipped)
|
partitions.append(clipped)
|
||||||
return partitions
|
return partitions
|
||||||
@@ -179,98 +258,224 @@ class AoiOperationService:
|
|||||||
def read(db, project_id: UUID, operation_id: UUID) -> dict:
|
def read(db, project_id: UUID, operation_id: UUID) -> dict:
|
||||||
operation = db.get(AoiOperation, operation_id)
|
operation = db.get(AoiOperation, operation_id)
|
||||||
if operation is None or operation.project_id != project_id:
|
if operation is None or operation.project_id != project_id:
|
||||||
raise AppError(code="AOI_OPERATION_NOT_FOUND", message="AOI operation not found", status_code=404)
|
raise AppError(
|
||||||
partitions = db.query(AoiOperationPartition).filter(AoiOperationPartition.operation_id == operation_id).order_by(AoiOperationPartition.ordinal).all()
|
code="AOI_OPERATION_NOT_FOUND",
|
||||||
|
message="AOI operation not found",
|
||||||
|
status_code=404,
|
||||||
|
)
|
||||||
|
partitions = (
|
||||||
|
db.query(AoiOperationPartition)
|
||||||
|
.filter(AoiOperationPartition.operation_id == operation_id)
|
||||||
|
.order_by(AoiOperationPartition.ordinal)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
counts = Counter(partition.status for partition in partitions)
|
counts = Counter(partition.status for partition in partitions)
|
||||||
complete = counts["success"] + counts["skipped"]
|
complete = counts["success"] + counts["skipped"]
|
||||||
return {
|
return {
|
||||||
"id": operation.id, "project_id": operation.project_id, "area_id": operation.area_id,
|
"id": operation.id,
|
||||||
"parent_job_id": operation.parent_job_id, "operation_type": operation.operation_type,
|
"project_id": operation.project_id,
|
||||||
"status": operation.status, "request_json": operation.request_json, "plan_json": operation.plan_json,
|
"area_id": operation.area_id,
|
||||||
"result_json": operation.result_json, "error_message": operation.error_message,
|
"parent_job_id": operation.parent_job_id,
|
||||||
|
"operation_type": operation.operation_type,
|
||||||
|
"status": operation.status,
|
||||||
|
"request_json": operation.request_json,
|
||||||
|
"plan_json": operation.plan_json,
|
||||||
|
"result_json": operation.result_json,
|
||||||
|
"error_message": operation.error_message,
|
||||||
"progress": round(complete / len(partitions), 6) if partitions else 0.0,
|
"progress": round(complete / len(partitions), 6) if partitions else 0.0,
|
||||||
"partition_counts": dict(counts), "partitions": partitions,
|
"partition_counts": dict(counts),
|
||||||
"created_at": operation.created_at, "started_at": operation.started_at, "finished_at": operation.finished_at,
|
"partitions": partitions,
|
||||||
|
"created_at": operation.created_at,
|
||||||
|
"started_at": operation.started_at,
|
||||||
|
"finished_at": operation.finished_at,
|
||||||
}
|
}
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def list(db, project_id: UUID, limit: int = 50) -> dict:
|
def list(db, project_id: UUID, limit: int = 50) -> dict:
|
||||||
rows = db.query(AoiOperation).filter(AoiOperation.project_id == project_id).order_by(AoiOperation.created_at.desc()).limit(limit).all()
|
rows = (
|
||||||
return {"items": [AoiOperationService.read(db, project_id, row.id) for row in rows], "total": len(rows)}
|
db.query(AoiOperation)
|
||||||
|
.filter(AoiOperation.project_id == project_id)
|
||||||
|
.order_by(AoiOperation.created_at.desc())
|
||||||
|
.limit(limit)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"items": [AoiOperationService.read(db, project_id, row.id) for row in rows],
|
||||||
|
"total": len(rows),
|
||||||
|
}
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def claim_next(db, project_id: UUID, operation_id: UUID):
|
def claim_next(db, project_id: UUID, operation_id: UUID):
|
||||||
operation = db.get(AoiOperation, operation_id)
|
operation = db.get(AoiOperation, operation_id)
|
||||||
if operation is None or operation.project_id != project_id:
|
if operation is None or operation.project_id != project_id:
|
||||||
raise AppError(code="AOI_OPERATION_NOT_FOUND", message="AOI operation not found", status_code=404)
|
raise AppError(
|
||||||
partition = db.query(AoiOperationPartition).filter(AoiOperationPartition.operation_id == operation_id, AoiOperationPartition.status == "queued").order_by(AoiOperationPartition.ordinal).with_for_update(skip_locked=True).first()
|
code="AOI_OPERATION_NOT_FOUND",
|
||||||
|
message="AOI operation not found",
|
||||||
|
status_code=404,
|
||||||
|
)
|
||||||
|
partition = (
|
||||||
|
db.query(AoiOperationPartition)
|
||||||
|
.filter(
|
||||||
|
AoiOperationPartition.operation_id == operation_id,
|
||||||
|
AoiOperationPartition.status == "queued",
|
||||||
|
)
|
||||||
|
.order_by(AoiOperationPartition.ordinal)
|
||||||
|
.with_for_update(skip_locked=True)
|
||||||
|
.first()
|
||||||
|
)
|
||||||
if partition is None:
|
if partition is None:
|
||||||
return None
|
return None
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
partition.status = "running"; partition.started_at = now; partition.attempt_count += 1; partition.error_message = None
|
partition.status = "running"
|
||||||
operation.status = "running"; operation.started_at = operation.started_at or now
|
partition.started_at = now
|
||||||
db.add(partition); db.add(operation); db.commit(); db.refresh(partition)
|
partition.attempt_count += 1
|
||||||
|
partition.error_message = None
|
||||||
|
operation.status = "running"
|
||||||
|
operation.started_at = operation.started_at or now
|
||||||
|
db.add(partition)
|
||||||
|
db.add(operation)
|
||||||
|
db.commit()
|
||||||
|
db.refresh(partition)
|
||||||
return partition
|
return partition
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def checkpoint(db, project_id: UUID, operation_id: UUID, partition_id: UUID, checkpoint: dict):
|
def checkpoint(
|
||||||
partition = AoiOperationService._partition_row(db, project_id, operation_id, partition_id)
|
db, project_id: UUID, operation_id: UUID, partition_id: UUID, checkpoint: dict
|
||||||
|
):
|
||||||
|
partition = AoiOperationService._partition_row(
|
||||||
|
db, project_id, operation_id, partition_id
|
||||||
|
)
|
||||||
if partition.status != "running":
|
if partition.status != "running":
|
||||||
raise AppError(code="AOI_PARTITION_NOT_RUNNING", message="Only a running partition can be checkpointed", status_code=409)
|
raise AppError(
|
||||||
partition.checkpoint_json = checkpoint; db.add(partition); db.commit(); db.refresh(partition)
|
code="AOI_PARTITION_NOT_RUNNING",
|
||||||
|
message="Only a running partition can be checkpointed",
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
|
partition.checkpoint_json = checkpoint
|
||||||
|
db.add(partition)
|
||||||
|
db.commit()
|
||||||
|
db.refresh(partition)
|
||||||
return partition
|
return partition
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def complete(db, project_id: UUID, operation_id: UUID, partition_id: UUID, result: dict, skipped: bool = False):
|
def complete(
|
||||||
partition = AoiOperationService._partition_row(db, project_id, operation_id, partition_id)
|
db,
|
||||||
|
project_id: UUID,
|
||||||
|
operation_id: UUID,
|
||||||
|
partition_id: UUID,
|
||||||
|
result: dict,
|
||||||
|
skipped: bool = False,
|
||||||
|
):
|
||||||
|
partition = AoiOperationService._partition_row(
|
||||||
|
db, project_id, operation_id, partition_id
|
||||||
|
)
|
||||||
if partition.status == "success" or partition.status == "skipped":
|
if partition.status == "success" or partition.status == "skipped":
|
||||||
return AoiOperationService.read(db, project_id, operation_id)
|
return AoiOperationService.read(db, project_id, operation_id)
|
||||||
if partition.status != "running":
|
if partition.status != "running":
|
||||||
raise AppError(code="AOI_PARTITION_NOT_RUNNING", message="Only a running partition can complete", status_code=409)
|
raise AppError(
|
||||||
partition.status = "skipped" if skipped else "success"; partition.result_json = result; partition.finished_at = datetime.now(timezone.utc)
|
code="AOI_PARTITION_NOT_RUNNING",
|
||||||
db.add(partition); db.commit(); AoiOperationService._refresh_parent(db, operation_id)
|
message="Only a running partition can complete",
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
|
partition.status = "skipped" if skipped else "success"
|
||||||
|
partition.result_json = result
|
||||||
|
partition.finished_at = datetime.now(timezone.utc)
|
||||||
|
db.add(partition)
|
||||||
|
db.commit()
|
||||||
|
AoiOperationService._refresh_parent(db, operation_id)
|
||||||
return AoiOperationService.read(db, project_id, operation_id)
|
return AoiOperationService.read(db, project_id, operation_id)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def fail(db, project_id: UUID, operation_id: UUID, partition_id: UUID, message: str, retryable: bool, details: dict):
|
def fail(
|
||||||
partition = AoiOperationService._partition_row(db, project_id, operation_id, partition_id)
|
db,
|
||||||
partition.error_message = message; partition.result_json = {"details": details}
|
project_id: UUID,
|
||||||
partition.status = "queued" if retryable and partition.attempt_count < partition.max_attempts else "failed"
|
operation_id: UUID,
|
||||||
partition.finished_at = None if partition.status == "queued" else datetime.now(timezone.utc)
|
partition_id: UUID,
|
||||||
db.add(partition); db.commit(); AoiOperationService._refresh_parent(db, operation_id)
|
message: str,
|
||||||
|
retryable: bool,
|
||||||
|
details: dict,
|
||||||
|
):
|
||||||
|
partition = AoiOperationService._partition_row(
|
||||||
|
db, project_id, operation_id, partition_id
|
||||||
|
)
|
||||||
|
partition.error_message = message
|
||||||
|
partition.result_json = {"details": details}
|
||||||
|
partition.status = (
|
||||||
|
"queued"
|
||||||
|
if retryable and partition.attempt_count < partition.max_attempts
|
||||||
|
else "failed"
|
||||||
|
)
|
||||||
|
partition.finished_at = (
|
||||||
|
None if partition.status == "queued" else datetime.now(timezone.utc)
|
||||||
|
)
|
||||||
|
db.add(partition)
|
||||||
|
db.commit()
|
||||||
|
AoiOperationService._refresh_parent(db, operation_id)
|
||||||
return AoiOperationService.read(db, project_id, operation_id)
|
return AoiOperationService.read(db, project_id, operation_id)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _partition_row(db, project_id, operation_id, partition_id):
|
def _partition_row(db, project_id, operation_id, partition_id):
|
||||||
operation = db.get(AoiOperation, operation_id); partition = db.get(AoiOperationPartition, partition_id)
|
operation = db.get(AoiOperation, operation_id)
|
||||||
if operation is None or operation.project_id != project_id or partition is None or partition.operation_id != operation_id:
|
partition = db.get(AoiOperationPartition, partition_id)
|
||||||
raise AppError(code="AOI_PARTITION_NOT_FOUND", message="AOI partition not found", status_code=404)
|
if (
|
||||||
|
operation is None
|
||||||
|
or operation.project_id != project_id
|
||||||
|
or partition is None
|
||||||
|
or partition.operation_id != operation_id
|
||||||
|
):
|
||||||
|
raise AppError(
|
||||||
|
code="AOI_PARTITION_NOT_FOUND",
|
||||||
|
message="AOI partition not found",
|
||||||
|
status_code=404,
|
||||||
|
)
|
||||||
return partition
|
return partition
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _refresh_parent(db, operation_id):
|
def _refresh_parent(db, operation_id):
|
||||||
operation = db.get(AoiOperation, operation_id)
|
operation = db.get(AoiOperation, operation_id)
|
||||||
partitions = db.query(AoiOperationPartition).filter(AoiOperationPartition.operation_id == operation_id).order_by(AoiOperationPartition.ordinal).all()
|
partitions = (
|
||||||
|
db.query(AoiOperationPartition)
|
||||||
|
.filter(AoiOperationPartition.operation_id == operation_id)
|
||||||
|
.order_by(AoiOperationPartition.ordinal)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
statuses = [partition.status for partition in partitions]
|
statuses = [partition.status for partition in partitions]
|
||||||
output_dataset_ids = []
|
output_dataset_ids = []
|
||||||
for partition in partitions:
|
for partition in partitions:
|
||||||
output_id = (partition.result_json or {}).get("output_dataset_id") if isinstance(partition.result_json, dict) else None
|
output_id = (
|
||||||
|
(partition.result_json or {}).get("output_dataset_id")
|
||||||
|
if isinstance(partition.result_json, dict)
|
||||||
|
else None
|
||||||
|
)
|
||||||
if output_id and str(output_id) not in output_dataset_ids:
|
if output_id and str(output_id) not in output_dataset_ids:
|
||||||
output_dataset_ids.append(str(output_id))
|
output_dataset_ids.append(str(output_id))
|
||||||
operation.result_json = {
|
operation.result_json = {
|
||||||
"partition_count": len(partitions),
|
"partition_count": len(partitions),
|
||||||
"completed_partition_count": sum(status in {"success", "skipped"} for status in statuses),
|
"completed_partition_count": sum(
|
||||||
|
status in {"success", "skipped"} for status in statuses
|
||||||
|
),
|
||||||
"failed_partition_count": statuses.count("failed"),
|
"failed_partition_count": statuses.count("failed"),
|
||||||
"output_dataset_ids": output_dataset_ids,
|
"output_dataset_ids": output_dataset_ids,
|
||||||
"merge_contract": "source_aware_spatial_union",
|
"merge_contract": "source_aware_spatial_union",
|
||||||
"vector_deduplication": "source_feature_id_then_geometry",
|
"vector_deduplication": "source_feature_id_then_geometry",
|
||||||
"raster_deduplication": "governed_mosaic_grid",
|
"raster_deduplication": "governed_mosaic_grid",
|
||||||
"complete_coverage": bool(statuses) and all(status in {"success", "skipped"} for status in statuses),
|
"complete_coverage": bool(statuses)
|
||||||
|
and all(status in {"success", "skipped"} for status in statuses),
|
||||||
}
|
}
|
||||||
now = datetime.now(timezone.utc)
|
now = datetime.now(timezone.utc)
|
||||||
if statuses and all(status in {"success", "skipped"} for status in statuses):
|
if statuses and all(status in {"success", "skipped"} for status in statuses):
|
||||||
operation.status = "success"; operation.finished_at = now; operation.error_message = None
|
operation.status = "success"
|
||||||
elif "failed" in statuses and not any(status in {"queued", "running"} for status in statuses):
|
operation.finished_at = now
|
||||||
operation.status = "partial" if any(status in {"success", "skipped"} for status in statuses) else "failed"; operation.finished_at = now
|
operation.error_message = None
|
||||||
|
elif "failed" in statuses and not any(
|
||||||
|
status in {"queued", "running"} for status in statuses
|
||||||
|
):
|
||||||
|
operation.status = (
|
||||||
|
"partial"
|
||||||
|
if any(status in {"success", "skipped"} for status in statuses)
|
||||||
|
else "failed"
|
||||||
|
)
|
||||||
|
operation.finished_at = now
|
||||||
operation.error_message = "One or more bounded source partitions failed; inspect partition evidence."
|
operation.error_message = "One or more bounded source partitions failed; inspect partition evidence."
|
||||||
db.add(operation); db.commit()
|
db.add(operation)
|
||||||
|
db.commit()
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from shapely.geometry import mapping
|
|||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.models import Area, Dataset, Project, VectorFeature
|
from app.models import Area, Dataset, Project, VectorFeature
|
||||||
from app.schemas.area import AreaCreate, AreaRead, AreaUpdate
|
from app.schemas.area import AreaCreate, AreaRead, AreaUpdate
|
||||||
from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_to_multipolygon
|
from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_area_to_epsg4326
|
||||||
|
|
||||||
|
|
||||||
class AreaService:
|
class AreaService:
|
||||||
@@ -126,7 +126,11 @@ class AreaService:
|
|||||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||||
|
|
||||||
try:
|
try:
|
||||||
multipolygon = normalize_to_multipolygon(payload.geometry)
|
multipolygon, original_crs = normalize_area_to_epsg4326(
|
||||||
|
payload.geometry,
|
||||||
|
payload.crs or "EPSG:4326",
|
||||||
|
)
|
||||||
|
metric_area = area_m2(multipolygon)
|
||||||
except ValueError as exc:
|
except ValueError as exc:
|
||||||
raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc
|
raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc
|
||||||
|
|
||||||
@@ -134,8 +138,8 @@ class AreaService:
|
|||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name=payload.name.strip() or "Unnamed area",
|
name=payload.name.strip() or "Unnamed area",
|
||||||
geometry=from_shape(multipolygon, srid=4326),
|
geometry=from_shape(multipolygon, srid=4326),
|
||||||
original_crs=payload.crs or "EPSG:4326",
|
original_crs=original_crs,
|
||||||
area_m2=area_m2(multipolygon),
|
area_m2=metric_area,
|
||||||
bbox=from_shape(geometry_bbox_polygon(multipolygon), srid=4326),
|
bbox=from_shape(geometry_bbox_polygon(multipolygon), srid=4326),
|
||||||
)
|
)
|
||||||
db.add(area)
|
db.add(area)
|
||||||
@@ -157,11 +161,28 @@ class AreaService:
|
|||||||
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
|
||||||
|
|
||||||
changed = False
|
changed = False
|
||||||
if payload.name:
|
if payload.name is not None and payload.name.strip():
|
||||||
area.name = payload.name.strip() or area.name
|
area.name = payload.name.strip() or area.name
|
||||||
changed = True
|
changed = True
|
||||||
if payload.crs:
|
if payload.crs is not None and payload.geometry is None:
|
||||||
area.original_crs = payload.crs
|
raise AppError(
|
||||||
|
code="INVALID_AREA_CRS_UPDATE",
|
||||||
|
message="crs can only be supplied together with replacement geometry",
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
if payload.geometry is not None:
|
||||||
|
try:
|
||||||
|
multipolygon, original_crs = normalize_area_to_epsg4326(
|
||||||
|
payload.geometry,
|
||||||
|
payload.crs or "EPSG:4326",
|
||||||
|
)
|
||||||
|
metric_area = area_m2(multipolygon)
|
||||||
|
except ValueError as exc:
|
||||||
|
raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc
|
||||||
|
area.geometry = from_shape(multipolygon, srid=4326)
|
||||||
|
area.original_crs = original_crs
|
||||||
|
area.area_m2 = metric_area
|
||||||
|
area.bbox = from_shape(geometry_bbox_polygon(multipolygon), srid=4326)
|
||||||
changed = True
|
changed = True
|
||||||
if not changed:
|
if not changed:
|
||||||
raise AppError(code="NO_CHANGES", message="No updatable fields provided", status_code=422)
|
raise AppError(code="NO_CHANGES", message="No updatable fields provided", status_code=422)
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import math
|
|||||||
from typing import Any, Callable
|
from typing import Any, Callable
|
||||||
from urllib.error import HTTPError, URLError
|
from urllib.error import HTTPError, URLError
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode
|
||||||
from urllib.request import Request, urlopen
|
from urllib.request import Request
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from geoalchemy2.shape import to_shape
|
from geoalchemy2.shape import to_shape
|
||||||
|
|||||||
@@ -112,7 +112,6 @@ class BathymetryRasterAnalysisService:
|
|||||||
try:
|
try:
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import rasterio
|
import rasterio
|
||||||
from rasterio.features import geometry_mask
|
|
||||||
from rasterio.mask import mask
|
from rasterio.mask import mask
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
raise AppError(
|
raise AppError(
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ from shapely.geometry import MultiPoint, shape
|
|||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
|
from app.core.config import get_settings
|
||||||
from app.models import Area, Dataset, DatasetVersion, Project
|
from app.models import Area, Dataset, DatasetVersion, Project
|
||||||
from app.services.data_contract_validation import (
|
from app.services.data_contract_validation import (
|
||||||
ContractKind,
|
ContractKind,
|
||||||
@@ -476,6 +477,29 @@ class DatasetService:
|
|||||||
content_type=content_type,
|
content_type=content_type,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _persist_vector_source_evidence_from_path(
|
||||||
|
cls,
|
||||||
|
*,
|
||||||
|
project_id: UUID,
|
||||||
|
dataset_id: UUID,
|
||||||
|
original_filename: str,
|
||||||
|
source_path: str | Path,
|
||||||
|
content_type: str | None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
safe_filename = StorageService._safe_filename(original_filename)
|
||||||
|
evidence_path = (
|
||||||
|
StorageService.dataset_root(str(project_id), str(dataset_id), "vector")
|
||||||
|
/ "provenance"
|
||||||
|
/ f"{dataset_id}_source_{safe_filename}"
|
||||||
|
)
|
||||||
|
return StorageService.persist_file_from_path(
|
||||||
|
str(evidence_path),
|
||||||
|
source_path,
|
||||||
|
original_filename=safe_filename,
|
||||||
|
content_type=content_type,
|
||||||
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _record_vector_source_evidence(
|
def _record_vector_source_evidence(
|
||||||
cls,
|
cls,
|
||||||
@@ -853,6 +877,52 @@ class DatasetService:
|
|||||||
raise AppError(code="INVALID_UPLOAD", message="Missing file name", status_code=400)
|
raise AppError(code="INVALID_UPLOAD", message="Missing file name", status_code=400)
|
||||||
return filename
|
return filename
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def _stage_upload(
|
||||||
|
*,
|
||||||
|
project_id: UUID,
|
||||||
|
dataset_id: uuid.UUID,
|
||||||
|
dataset_type: str,
|
||||||
|
filename: str,
|
||||||
|
file: UploadFile,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
settings = get_settings()
|
||||||
|
max_upload_mb = int(settings.max_upload_mb)
|
||||||
|
if DatasetService._canonical_dataset_type(dataset_type) == "vector":
|
||||||
|
max_upload_mb = min(max_upload_mb, int(settings.max_in_memory_vector_mb))
|
||||||
|
return await StorageService.persist_upload_file(
|
||||||
|
project_id=str(project_id),
|
||||||
|
dataset_id=str(dataset_id),
|
||||||
|
dataset_type=dataset_type,
|
||||||
|
original_filename=filename,
|
||||||
|
upload=file,
|
||||||
|
content_type=file.content_type,
|
||||||
|
max_bytes=max_upload_mb * 1024 * 1024,
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _read_staged_vector_bytes(storage_info: dict[str, Any]) -> bytes:
|
||||||
|
settings = get_settings()
|
||||||
|
max_bytes = min(
|
||||||
|
int(settings.max_upload_mb),
|
||||||
|
int(settings.max_in_memory_vector_mb),
|
||||||
|
) * 1024 * 1024
|
||||||
|
path = Path(str(storage_info["storage_path"]))
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
content = stream.read(max_bytes + 1)
|
||||||
|
if len(content) > max_bytes:
|
||||||
|
StorageService.remove_dataset_file(str(path))
|
||||||
|
raise AppError(
|
||||||
|
code="UPLOAD_TOO_LARGE",
|
||||||
|
message="Vector upload exceeds the bounded in-memory parsing limit.",
|
||||||
|
details={
|
||||||
|
"max_bytes": max_bytes,
|
||||||
|
"max_in_memory_vector_mb": max_bytes // (1024 * 1024),
|
||||||
|
},
|
||||||
|
status_code=413,
|
||||||
|
)
|
||||||
|
return content
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def list_datasets(db: Session, project_id: UUID, limit: int = 50, offset: int = 0) -> tuple[list[DatasetCreateResponse], int]:
|
def list_datasets(db: Session, project_id: UUID, limit: int = 50, offset: int = 0) -> tuple[list[DatasetCreateResponse], int]:
|
||||||
total = db.query(Dataset).filter(Dataset.project_id == project_id).count()
|
total = db.query(Dataset).filter(Dataset.project_id == project_id).count()
|
||||||
@@ -1135,15 +1205,15 @@ class DatasetService:
|
|||||||
status_code=415,
|
status_code=415,
|
||||||
)
|
)
|
||||||
|
|
||||||
raw = await file.read()
|
dataset_id = uuid.uuid4()
|
||||||
storage_info = StorageService.persist_dataset_file(
|
storage_info = await DatasetService._stage_upload(
|
||||||
project_id=str(project_id),
|
project_id=project_id,
|
||||||
dataset_id=str(dataset_id := uuid.uuid4()),
|
dataset_id=dataset_id,
|
||||||
dataset_type=canonical_type,
|
dataset_type=canonical_type,
|
||||||
original_filename=filename,
|
filename=filename,
|
||||||
content=raw,
|
file=file,
|
||||||
content_type=file.content_type,
|
|
||||||
)
|
)
|
||||||
|
raw = DatasetService._read_staged_vector_bytes(storage_info) if canonical_type == "vector" else None
|
||||||
|
|
||||||
metadata: dict[str, Any] = {}
|
metadata: dict[str, Any] = {}
|
||||||
vector_payload: dict[str, Any] | None = None
|
vector_payload: dict[str, Any] | None = None
|
||||||
@@ -1151,6 +1221,7 @@ class DatasetService:
|
|||||||
try:
|
try:
|
||||||
status = "validating"
|
status = "validating"
|
||||||
if canonical_type == "vector":
|
if canonical_type == "vector":
|
||||||
|
assert raw is not None
|
||||||
try:
|
try:
|
||||||
text = raw.decode("utf-8")
|
text = raw.decode("utf-8")
|
||||||
except UnicodeDecodeError as exc:
|
except UnicodeDecodeError as exc:
|
||||||
@@ -1319,8 +1390,15 @@ class DatasetService:
|
|||||||
temporal_granularity=temporal_granularity,
|
temporal_granularity=temporal_granularity,
|
||||||
source_version=source_version,
|
source_version=source_version,
|
||||||
)
|
)
|
||||||
raw = await file.read()
|
dataset_id = uuid.uuid4()
|
||||||
checksum_sha256 = StorageService.calculate_checksum_sha256(raw)
|
storage_info = await DatasetService._stage_upload(
|
||||||
|
project_id=project_id,
|
||||||
|
dataset_id=dataset_id,
|
||||||
|
dataset_type=canonical_type,
|
||||||
|
filename=filename,
|
||||||
|
file=file,
|
||||||
|
)
|
||||||
|
checksum_sha256 = storage_info["checksum_sha256"]
|
||||||
ingest_key = DatasetService._ingest_key(
|
ingest_key = DatasetService._ingest_key(
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
source_key="manual",
|
source_key="manual",
|
||||||
@@ -1333,6 +1411,7 @@ class DatasetService:
|
|||||||
)
|
)
|
||||||
existing = DatasetService._find_existing_ingest(db, project_id, ingest_key)
|
existing = DatasetService._find_existing_ingest(db, project_id, ingest_key)
|
||||||
if existing is not None:
|
if existing is not None:
|
||||||
|
StorageService.remove_dataset_file(storage_info["storage_path"])
|
||||||
return DatasetService._to_response(existing)
|
return DatasetService._to_response(existing)
|
||||||
|
|
||||||
raw_source_metadata = dict(source_metadata or {})
|
raw_source_metadata = dict(source_metadata or {})
|
||||||
@@ -1361,8 +1440,7 @@ class DatasetService:
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
dataset_id = uuid.uuid4()
|
raw = DatasetService._read_staged_vector_bytes(storage_info) if canonical_type == "vector" else None
|
||||||
storage_info: dict[str, Any] | None = None
|
|
||||||
storage_content = raw
|
storage_content = raw
|
||||||
source_evidence: dict[str, Any] | None = None
|
source_evidence: dict[str, Any] | None = None
|
||||||
imported_at = datetime.now(timezone.utc)
|
imported_at = datetime.now(timezone.utc)
|
||||||
@@ -1372,6 +1450,7 @@ class DatasetService:
|
|||||||
parser_error: tuple[str, str] | None = None
|
parser_error: tuple[str, str] | None = None
|
||||||
try:
|
try:
|
||||||
if canonical_type == "vector":
|
if canonical_type == "vector":
|
||||||
|
assert raw is not None
|
||||||
try:
|
try:
|
||||||
payload = json.loads(raw.decode("utf-8"))
|
payload = json.loads(raw.decode("utf-8"))
|
||||||
except UnicodeDecodeError as exc:
|
except UnicodeDecodeError as exc:
|
||||||
@@ -1394,22 +1473,22 @@ class DatasetService:
|
|||||||
)
|
)
|
||||||
if DatasetService._vector_storage_requires_canonicalization(source_crs):
|
if DatasetService._vector_storage_requires_canonicalization(source_crs):
|
||||||
storage_content = DatasetService._canonical_vector_storage_bytes(canonical_vector_payload)
|
storage_content = DatasetService._canonical_vector_storage_bytes(canonical_vector_payload)
|
||||||
source_evidence = DatasetService._persist_vector_source_evidence(
|
source_evidence = DatasetService._persist_vector_source_evidence_from_path(
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
dataset_id=dataset_id,
|
dataset_id=dataset_id,
|
||||||
original_filename=filename,
|
original_filename=filename,
|
||||||
content=raw,
|
source_path=storage_info["storage_path"],
|
||||||
|
content_type=file.content_type,
|
||||||
|
)
|
||||||
|
storage_info = StorageService.persist_dataset_file(
|
||||||
|
project_id=str(project_id),
|
||||||
|
dataset_id=str(dataset_id),
|
||||||
|
dataset_type=canonical_type,
|
||||||
|
original_filename=filename,
|
||||||
|
content=storage_content,
|
||||||
content_type=file.content_type,
|
content_type=file.content_type,
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
storage_info = StorageService.persist_dataset_file(
|
|
||||||
project_id=str(project_id),
|
|
||||||
dataset_id=str(dataset_id),
|
|
||||||
dataset_type=canonical_type,
|
|
||||||
original_filename=filename,
|
|
||||||
content=raw,
|
|
||||||
content_type=file.content_type,
|
|
||||||
)
|
|
||||||
metadata = extract_raster_metadata(storage_info["storage_path"])
|
metadata = extract_raster_metadata(storage_info["storage_path"])
|
||||||
metadata["dataset_type"] = "raster"
|
metadata["dataset_type"] = "raster"
|
||||||
source_crs = metadata.get("crs")
|
source_crs = metadata.get("crs")
|
||||||
@@ -1422,16 +1501,7 @@ class DatasetService:
|
|||||||
"processing_code": code,
|
"processing_code": code,
|
||||||
}
|
}
|
||||||
|
|
||||||
if storage_info is None:
|
computed_storage_checksum_sha256 = storage_info["checksum_sha256"]
|
||||||
storage_info = StorageService.persist_dataset_file(
|
|
||||||
project_id=str(project_id),
|
|
||||||
dataset_id=str(dataset_id),
|
|
||||||
dataset_type=canonical_type,
|
|
||||||
original_filename=filename,
|
|
||||||
content=storage_content,
|
|
||||||
content_type=file.content_type,
|
|
||||||
)
|
|
||||||
computed_storage_checksum_sha256 = StorageService.calculate_checksum_sha256(storage_content)
|
|
||||||
if source_evidence is not None:
|
if source_evidence is not None:
|
||||||
resolved_source_crs = source_crs or DatasetService.CANONICAL_VECTOR_CRS
|
resolved_source_crs = source_crs or DatasetService.CANONICAL_VECTOR_CRS
|
||||||
DatasetService._record_vector_source_evidence(
|
DatasetService._record_vector_source_evidence(
|
||||||
@@ -1502,7 +1572,7 @@ class DatasetService:
|
|||||||
feature_collection=canonical_vector_payload or {"type": "FeatureCollection", "features": []},
|
feature_collection=canonical_vector_payload or {"type": "FeatureCollection", "features": []},
|
||||||
checksum_sha256=storage_info["checksum_sha256"],
|
checksum_sha256=storage_info["checksum_sha256"],
|
||||||
computed_checksum_sha256=computed_storage_checksum_sha256,
|
computed_checksum_sha256=computed_storage_checksum_sha256,
|
||||||
content=storage_content,
|
content=None,
|
||||||
source_registry_id=str(source_registry.id),
|
source_registry_id=str(source_registry.id),
|
||||||
source_snapshot_id=str(source_snapshot.id),
|
source_snapshot_id=str(source_snapshot.id),
|
||||||
imported_at=imported_at,
|
imported_at=imported_at,
|
||||||
@@ -1541,8 +1611,8 @@ class DatasetService:
|
|||||||
bounds=DatasetService._extract_raster_bounds_json(metadata),
|
bounds=DatasetService._extract_raster_bounds_json(metadata),
|
||||||
resolution=resolution,
|
resolution=resolution,
|
||||||
checksum_sha256=storage_info["checksum_sha256"],
|
checksum_sha256=storage_info["checksum_sha256"],
|
||||||
computed_checksum_sha256=checksum_sha256,
|
computed_checksum_sha256=storage_info["checksum_sha256"],
|
||||||
content=raw,
|
content=None,
|
||||||
source_registry_id=str(source_registry.id),
|
source_registry_id=str(source_registry.id),
|
||||||
source_snapshot_id=str(source_snapshot.id),
|
source_snapshot_id=str(source_snapshot.id),
|
||||||
imported_at=imported_at,
|
imported_at=imported_at,
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from collections import Counter
|
from collections import Counter
|
||||||
from typing import Any
|
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ from app.services.storage_service import StorageService
|
|||||||
from app.services.quality_service import QualityService
|
from app.services.quality_service import QualityService
|
||||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenance, RuntimeModelProvenanceService
|
from app.services.runtime_model_provenance_service import RuntimeModelProvenance, RuntimeModelProvenanceService
|
||||||
from app.services.temporal_compatibility_service import TemporalCompatibilityService
|
from app.services.temporal_compatibility_service import TemporalCompatibilityService
|
||||||
|
from app.services.tile_manifest_service import TileManifestService
|
||||||
from app.services.yolo_adapter import YoloDetectionAdapter
|
from app.services.yolo_adapter import YoloDetectionAdapter
|
||||||
|
|
||||||
|
|
||||||
@@ -968,6 +969,18 @@ class DetectionService:
|
|||||||
yolo_adapter_class: Type[YoloDetectionAdapter],
|
yolo_adapter_class: Type[YoloDetectionAdapter],
|
||||||
) -> tuple[list[Detection], dict[str, Any]]:
|
) -> tuple[list[Detection], dict[str, Any]]:
|
||||||
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles, settings)
|
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles, settings)
|
||||||
|
dataset = db.get(Dataset, dataset_id)
|
||||||
|
if dataset is None:
|
||||||
|
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
||||||
|
manifest_binding = TileManifestService.validate_for_inference(
|
||||||
|
db,
|
||||||
|
dataset,
|
||||||
|
manifest,
|
||||||
|
manifest_path=tile_manifest_path or "",
|
||||||
|
settings=settings,
|
||||||
|
error_prefix="DETECTION",
|
||||||
|
)
|
||||||
|
DetectionService._attach_tile_manifest_binding(analysis_run, job, manifest_binding)
|
||||||
model_path = Path(settings.yolo_model_path or "").expanduser()
|
model_path = Path(settings.yolo_model_path or "").expanduser()
|
||||||
runtime_model_provenance = RuntimeModelProvenanceService.validate_for_production_runtime(
|
runtime_model_provenance = RuntimeModelProvenanceService.validate_for_production_runtime(
|
||||||
db=db,
|
db=db,
|
||||||
@@ -1075,9 +1088,23 @@ class DetectionService:
|
|||||||
"tile_edge_truncated_count": len(candidates) - len(edge_filtered_candidates),
|
"tile_edge_truncated_count": len(candidates) - len(edge_filtered_candidates),
|
||||||
"duplicate_iou_threshold": float(settings.yolo_duplicate_iou_threshold),
|
"duplicate_iou_threshold": float(settings.yolo_duplicate_iou_threshold),
|
||||||
"containment_suppression_threshold": float(settings.yolo_containment_nms_threshold),
|
"containment_suppression_threshold": float(settings.yolo_containment_nms_threshold),
|
||||||
|
"tile_manifest_binding": manifest_binding,
|
||||||
"runtime_model_provenance": runtime_model_provenance.as_dict(),
|
"runtime_model_provenance": runtime_model_provenance.as_dict(),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _attach_tile_manifest_binding(
|
||||||
|
analysis_run: AnalysisRun,
|
||||||
|
job: Job,
|
||||||
|
binding: dict[str, Any],
|
||||||
|
) -> None:
|
||||||
|
analysis_parameters = dict(analysis_run.parameters_json or {})
|
||||||
|
analysis_parameters["tile_manifest_binding"] = dict(binding)
|
||||||
|
analysis_run.parameters_json = analysis_parameters
|
||||||
|
job_parameters = dict(job.parameters_json or {})
|
||||||
|
job_parameters["tile_manifest_binding"] = dict(binding)
|
||||||
|
job.parameters_json = job_parameters
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _attach_runtime_model_provenance(
|
def _attach_runtime_model_provenance(
|
||||||
analysis_run: AnalysisRun,
|
analysis_run: AnalysisRun,
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from pathlib import Path
|
|||||||
from typing import Any, Callable
|
from typing import Any, Callable
|
||||||
from urllib.error import HTTPError, URLError
|
from urllib.error import HTTPError, URLError
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode
|
||||||
from urllib.request import Request, urlopen
|
from urllib.request import Request
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from geoalchemy2.shape import to_shape
|
from geoalchemy2.shape import to_shape
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from pathlib import Path
|
|||||||
from typing import Any, Callable
|
from typing import Any, Callable
|
||||||
from urllib.error import HTTPError, URLError
|
from urllib.error import HTTPError, URLError
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode
|
||||||
from urllib.request import Request, urlopen
|
from urllib.request import Request
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
from xml.etree import ElementTree
|
from xml.etree import ElementTree
|
||||||
|
|
||||||
|
|||||||
@@ -232,7 +232,6 @@ class FloodHazardAnalysisService:
|
|||||||
try:
|
try:
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import rasterio
|
import rasterio
|
||||||
from rasterio.features import geometry_mask
|
|
||||||
from rasterio.mask import mask
|
from rasterio.mask import mask
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for flood-hazard analysis", status_code=503) from exc
|
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for flood-hazard analysis", status_code=503) from exc
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ from datetime import UTC, datetime
|
|||||||
from typing import Any, Callable
|
from typing import Any, Callable
|
||||||
from urllib.error import HTTPError, URLError
|
from urllib.error import HTTPError, URLError
|
||||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||||
from urllib.request import Request, urlopen
|
from urllib.request import Request
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from app.core.config import Settings, get_settings
|
from app.core.config import Settings, get_settings
|
||||||
|
|||||||
@@ -6,11 +6,12 @@ import ssl
|
|||||||
from typing import Any, Callable
|
from typing import Any, Callable
|
||||||
from urllib.error import HTTPError, URLError
|
from urllib.error import HTTPError, URLError
|
||||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||||
from urllib.request import Request, urlopen
|
from urllib.request import Request
|
||||||
from xml.etree import ElementTree
|
from xml.etree import ElementTree
|
||||||
|
|
||||||
from app.core.config import Settings, get_settings
|
from app.core.config import Settings, get_settings
|
||||||
from app.schemas.bathymetry import BathymetrySourceProbeRead
|
from app.schemas.bathymetry import BathymetrySourceProbeRead
|
||||||
|
from app.services.outbound_request_guard import guarded_opener
|
||||||
|
|
||||||
|
|
||||||
class MdkBathymetryProbeService:
|
class MdkBathymetryProbeService:
|
||||||
|
|||||||
@@ -79,11 +79,15 @@ class ModelAssetCatalogService:
|
|||||||
size_bytes=path.stat().st_size,
|
size_bytes=path.stat().st_size,
|
||||||
sha256=ModelAssetCatalogService._sha256(path),
|
sha256=ModelAssetCatalogService._sha256(path),
|
||||||
active=active_model_path == resolved_path,
|
active=active_model_path == resolved_path,
|
||||||
status="approved" if active_model_path == resolved_path else "available",
|
runtime_available=True,
|
||||||
|
runtime_status="active" if active_model_path == resolved_path else "available",
|
||||||
|
governed_validation_status="not_verified_by_catalog",
|
||||||
|
promotion_status="not_verified_by_catalog",
|
||||||
|
status="runtime_active" if active_model_path == resolved_path else "runtime_available",
|
||||||
limitation_message=(
|
limitation_message=(
|
||||||
"Approved local runtime model asset. GeoIntel will not download or mutate model weights."
|
"Active local runtime model asset. Runtime selection is not evidence of governed validation or promotion."
|
||||||
if active_model_path == resolved_path
|
if active_model_path == resolved_path
|
||||||
else "Local development model asset. Configure it explicitly before production use."
|
else "Local runtime model asset. Governed validation and promotion are not established by this catalog."
|
||||||
),
|
),
|
||||||
will_download_models=False,
|
will_download_models=False,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from pathlib import Path
|
|||||||
from typing import Any, Callable
|
from typing import Any, Callable
|
||||||
from urllib.error import HTTPError, URLError
|
from urllib.error import HTTPError, URLError
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode
|
||||||
from urllib.request import Request, urlopen
|
from urllib.request import Request
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from geoalchemy2.shape import to_shape
|
from geoalchemy2.shape import to_shape
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
|
||||||
import uuid
|
import uuid
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
from hashlib import sha256
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@@ -16,6 +16,7 @@ from app.models import Area, Dataset, DatasetVersion
|
|||||||
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
|
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
|
||||||
from app.services.raster_service import extract_raster_metadata
|
from app.services.raster_service import extract_raster_metadata
|
||||||
from app.services.storage_service import StorageService
|
from app.services.storage_service import StorageService
|
||||||
|
from app.services.tile_manifest_service import TileManifestService, canonical_manifest_json
|
||||||
|
|
||||||
|
|
||||||
def _import_rasterio():
|
def _import_rasterio():
|
||||||
@@ -70,6 +71,39 @@ class RasterOperationsService:
|
|||||||
raise AppError(code="DATASET_FILE_MISSING", message="Stored raster file missing", status_code=404)
|
raise AppError(code="DATASET_FILE_MISSING", message="Stored raster file missing", status_code=404)
|
||||||
return dataset
|
return dataset
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _validate_storage_checksum(dataset: Dataset) -> None:
|
||||||
|
"""Refuse tiling pixels that no longer match the governed Dataset row."""
|
||||||
|
|
||||||
|
expected_checksum = str(dataset.checksum_sha256 or "").strip().lower()
|
||||||
|
governed_artifact = bool(getattr(dataset, "data_contract_key", None))
|
||||||
|
valid_checksum = len(expected_checksum) == 64 and all(character in "0123456789abcdef" for character in expected_checksum)
|
||||||
|
if not valid_checksum:
|
||||||
|
if expected_checksum or governed_artifact:
|
||||||
|
raise AppError(
|
||||||
|
code="DATASET_STORAGE_CHECKSUM_UNVERIFIABLE",
|
||||||
|
message="Governed raster storage requires a valid SHA-256 checksum before tiling.",
|
||||||
|
details={"checksum_sha256": dataset.checksum_sha256},
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
|
return
|
||||||
|
|
||||||
|
digest = sha256()
|
||||||
|
with Path(str(dataset.storage_path)).open("rb") as stream:
|
||||||
|
for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
actual_checksum = digest.hexdigest()
|
||||||
|
if actual_checksum != expected_checksum:
|
||||||
|
raise AppError(
|
||||||
|
code="DATASET_STORAGE_CHECKSUM_MISMATCH",
|
||||||
|
message="Raster dataset storage no longer matches its validated checksum.",
|
||||||
|
details={
|
||||||
|
"expected_checksum_sha256": expected_checksum,
|
||||||
|
"actual_checksum_sha256": actual_checksum,
|
||||||
|
},
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _raster_dependencies() -> tuple[Any, Any]:
|
def _raster_dependencies() -> tuple[Any, Any]:
|
||||||
try:
|
try:
|
||||||
@@ -970,14 +1004,17 @@ class RasterOperationsService:
|
|||||||
tile_size: int = 512,
|
tile_size: int = 512,
|
||||||
overlap: int = 64,
|
overlap: int = 64,
|
||||||
output_name: str | None = None,
|
output_name: str | None = None,
|
||||||
|
max_tiles: int | None = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
RasterOperationsService._validate_tile_request(tile_size=tile_size, overlap=overlap)
|
RasterOperationsService._validate_tile_request(tile_size=tile_size, overlap=overlap)
|
||||||
|
if max_tiles is not None and max_tiles <= 0:
|
||||||
|
raise AppError(code="INVALID_PARAMETERS", message="max_tiles must be positive", status_code=400)
|
||||||
dataset = RasterOperationsService._load_dataset(db, dataset_id)
|
dataset = RasterOperationsService._load_dataset(db, dataset_id)
|
||||||
|
RasterOperationsService._validate_storage_checksum(dataset)
|
||||||
rasterio, _ = RasterOperationsService._raster_dependencies()
|
rasterio, _ = RasterOperationsService._raster_dependencies()
|
||||||
|
|
||||||
tile_set_id = str(uuid.uuid4())
|
tile_set_id = str(uuid.uuid4())
|
||||||
tile_root = StorageService.raster_tiles_root(str(dataset.project_id), str(dataset.id), tile_set_id)
|
tile_root = StorageService.raster_tiles_root(str(dataset.project_id), str(dataset.id), tile_set_id)
|
||||||
tile_root.mkdir(parents=True, exist_ok=True)
|
|
||||||
|
|
||||||
manifest_tiles: list[dict[str, Any]] = []
|
manifest_tiles: list[dict[str, Any]] = []
|
||||||
tile_paths: list[str] = []
|
tile_paths: list[str] = []
|
||||||
@@ -997,9 +1034,23 @@ class RasterOperationsService:
|
|||||||
source_width = int(source.width)
|
source_width = int(source.width)
|
||||||
source_height = int(source.height)
|
source_height = int(source.height)
|
||||||
step = max(1, tile_size - overlap)
|
step = max(1, tile_size - overlap)
|
||||||
|
x_offsets = RasterOperationsService._tile_offsets(source_width, tile_size, step)
|
||||||
|
y_offsets = RasterOperationsService._tile_offsets(source_height, tile_size, step)
|
||||||
|
expected_tile_count = len(x_offsets) * len(y_offsets)
|
||||||
|
if max_tiles is not None and expected_tile_count > max_tiles:
|
||||||
|
raise AppError(
|
||||||
|
code="RASTER_TILE_LIMIT_EXCEEDED",
|
||||||
|
message="Raster tile generation exceeds the guest analysis limit.",
|
||||||
|
details={
|
||||||
|
"expected_tile_count": expected_tile_count,
|
||||||
|
"max_tiles": max_tiles,
|
||||||
|
},
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
tile_root.mkdir(parents=True, exist_ok=True)
|
||||||
tile_index = 0
|
tile_index = 0
|
||||||
for yoff in range(0, source_height, step):
|
for yoff in y_offsets:
|
||||||
for xoff in range(0, source_width, step):
|
for xoff in x_offsets:
|
||||||
tile_width = min(tile_size, source_width - xoff)
|
tile_width = min(tile_size, source_width - xoff)
|
||||||
tile_height = min(tile_size, source_height - yoff)
|
tile_height = min(tile_size, source_height - yoff)
|
||||||
if tile_width <= 0 or tile_height <= 0:
|
if tile_width <= 0 or tile_height <= 0:
|
||||||
@@ -1022,6 +1073,7 @@ class RasterOperationsService:
|
|||||||
tile_dest.write(tile_data)
|
tile_dest.write(tile_data)
|
||||||
|
|
||||||
tile_paths.append(str(tile_path))
|
tile_paths.append(str(tile_path))
|
||||||
|
tile_integrity = TileManifestService.tile_integrity(tile_path)
|
||||||
manifest_tiles.append(
|
manifest_tiles.append(
|
||||||
{
|
{
|
||||||
"path": str(tile_path),
|
"path": str(tile_path),
|
||||||
@@ -1030,6 +1082,7 @@ class RasterOperationsService:
|
|||||||
"transform": [float(item) for item in transform.to_gdal()],
|
"transform": [float(item) for item in transform.to_gdal()],
|
||||||
"crs": source_crs,
|
"crs": source_crs,
|
||||||
"index": tile_index,
|
"index": tile_index,
|
||||||
|
**tile_integrity,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
tile_index += 1
|
tile_index += 1
|
||||||
@@ -1044,9 +1097,8 @@ class RasterOperationsService:
|
|||||||
bounds = source_metadata.get("bounds", [0.0, 0.0, 0.0, 0.0])
|
bounds = source_metadata.get("bounds", [0.0, 0.0, 0.0, 0.0])
|
||||||
manifest_crs = source_crs or source_metadata.get("crs") or dataset.crs
|
manifest_crs = source_crs or source_metadata.get("crs") or dataset.crs
|
||||||
manifest_payload = {
|
manifest_payload = {
|
||||||
|
**TileManifestService.dataset_binding(db, dataset),
|
||||||
"tile_set_id": tile_set_id,
|
"tile_set_id": tile_set_id,
|
||||||
"source_dataset_id": str(dataset.id),
|
|
||||||
"source_raster_id": str(dataset.id),
|
|
||||||
"crs": manifest_crs,
|
"crs": manifest_crs,
|
||||||
"source_crs": manifest_crs,
|
"source_crs": manifest_crs,
|
||||||
"dataset_crs": dataset.crs,
|
"dataset_crs": dataset.crs,
|
||||||
@@ -1066,7 +1118,7 @@ class RasterOperationsService:
|
|||||||
"tile_server": None,
|
"tile_server": None,
|
||||||
}
|
}
|
||||||
manifest_path = tile_root / "manifest.json"
|
manifest_path = tile_root / "manifest.json"
|
||||||
manifest_path.write_text(json.dumps(manifest_payload), encoding="utf-8")
|
manifest_path.write_text(canonical_manifest_json(manifest_payload), encoding="utf-8")
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"dataset_id": str(dataset.id),
|
"dataset_id": str(dataset.id),
|
||||||
@@ -1079,3 +1131,17 @@ class RasterOperationsService:
|
|||||||
"count": len(manifest_tiles),
|
"count": len(manifest_tiles),
|
||||||
"manifest": manifest_payload,
|
"manifest": manifest_payload,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _tile_offsets(dimension: int, tile_size: int, step: int) -> list[int]:
|
||||||
|
"""Return full-tile starts plus one unique edge-aligned final start."""
|
||||||
|
|
||||||
|
if dimension <= 0 or tile_size <= 0 or step <= 0:
|
||||||
|
raise AppError(code="INVALID_PARAMETERS", message="Raster tile dimensions must be positive", status_code=400)
|
||||||
|
if dimension <= tile_size:
|
||||||
|
return [0]
|
||||||
|
final_start = dimension - tile_size
|
||||||
|
offsets = list(range(0, final_start + 1, step))
|
||||||
|
if offsets[-1] != final_start:
|
||||||
|
offsets.append(final_start)
|
||||||
|
return offsets
|
||||||
|
|||||||
@@ -8,7 +8,6 @@ from typing import Any
|
|||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from pyproj import Transformer
|
from pyproj import Transformer
|
||||||
from shapely.geometry import mapping
|
|
||||||
from shapely.ops import transform as shapely_transform
|
from shapely.ops import transform as shapely_transform
|
||||||
|
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
@@ -116,7 +115,6 @@ class RasterPartitionAnalysisService:
|
|||||||
try:
|
try:
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import rasterio
|
import rasterio
|
||||||
from rasterio.features import geometry_mask
|
|
||||||
from rasterio.merge import merge
|
from rasterio.merge import merge
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
raise AppError(
|
raise AppError(
|
||||||
|
|||||||
@@ -34,6 +34,7 @@ from app.services.segmentation_adapter import (
|
|||||||
SamSegmentationAdapter,
|
SamSegmentationAdapter,
|
||||||
YoloSegmentationAdapter,
|
YoloSegmentationAdapter,
|
||||||
)
|
)
|
||||||
|
from app.services.tile_manifest_service import TileManifestService
|
||||||
|
|
||||||
|
|
||||||
class SegmentationService:
|
class SegmentationService:
|
||||||
@@ -749,6 +750,18 @@ class SegmentationService:
|
|||||||
sam_adapter_class: type[SamSegmentationAdapter],
|
sam_adapter_class: type[SamSegmentationAdapter],
|
||||||
) -> tuple[list[Segmentation], dict[str, Any]]:
|
) -> tuple[list[Segmentation], dict[str, Any]]:
|
||||||
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles, settings)
|
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles, settings)
|
||||||
|
dataset = db.get(Dataset, dataset_id)
|
||||||
|
if dataset is None:
|
||||||
|
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
||||||
|
manifest_binding = TileManifestService.validate_for_inference(
|
||||||
|
db,
|
||||||
|
dataset,
|
||||||
|
manifest,
|
||||||
|
manifest_path=tile_manifest_path or "",
|
||||||
|
settings=settings,
|
||||||
|
error_prefix="SEGMENTATION",
|
||||||
|
)
|
||||||
|
DetectionService._attach_tile_manifest_binding(analysis_run, job, manifest_binding)
|
||||||
if model_name == settings.sam_model_id:
|
if model_name == settings.sam_model_id:
|
||||||
model_path = Path(settings.sam_model_path or "").expanduser()
|
model_path = Path(settings.sam_model_path or "").expanduser()
|
||||||
allowed_frameworks = ("ultralytics/sam", "sam", "ultralytics", "pytorch")
|
allowed_frameworks = ("ultralytics/sam", "sam", "ultralytics", "pytorch")
|
||||||
@@ -861,6 +874,7 @@ class SegmentationService:
|
|||||||
"duplicate_iou_threshold": float(settings.segmentation_duplicate_iou_threshold),
|
"duplicate_iou_threshold": float(settings.segmentation_duplicate_iou_threshold),
|
||||||
"containment_suppression_threshold": float(settings.segmentation_containment_nms_threshold),
|
"containment_suppression_threshold": float(settings.segmentation_containment_nms_threshold),
|
||||||
"tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()),
|
"tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()),
|
||||||
|
"tile_manifest_binding": manifest_binding,
|
||||||
"runtime_model_provenance": runtime_model_provenance.as_dict(),
|
"runtime_model_provenance": runtime_model_provenance.as_dict(),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from threading import Lock
|
|||||||
from typing import Any, Callable
|
from typing import Any, Callable
|
||||||
from urllib.error import HTTPError, URLError
|
from urllib.error import HTTPError, URLError
|
||||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||||
from urllib.request import Request, urlopen
|
from urllib.request import Request
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
from xml.etree import ElementTree
|
from xml.etree import ElementTree
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,8 @@ from app.core.errors import AppError
|
|||||||
|
|
||||||
|
|
||||||
class StorageService:
|
class StorageService:
|
||||||
|
UPLOAD_CHUNK_SIZE = 8 * 1024 * 1024
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _base_dir() -> Path:
|
def _base_dir() -> Path:
|
||||||
return Path(get_settings().storage_root).resolve()
|
return Path(get_settings().storage_root).resolve()
|
||||||
@@ -143,6 +145,75 @@ class StorageService:
|
|||||||
}
|
}
|
||||||
return metadata
|
return metadata
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
async def persist_upload_file(
|
||||||
|
*,
|
||||||
|
project_id: str,
|
||||||
|
dataset_id: str,
|
||||||
|
dataset_type: str,
|
||||||
|
original_filename: str,
|
||||||
|
upload: Any,
|
||||||
|
content_type: str | None,
|
||||||
|
max_bytes: int,
|
||||||
|
chunk_size: int | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Stream an UploadFile to governed storage with a hard byte limit.
|
||||||
|
|
||||||
|
The reverse proxy limit is defense in depth. This backend boundary is
|
||||||
|
authoritative as direct/loopback requests can bypass that proxy.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if max_bytes <= 0:
|
||||||
|
raise ValueError("max_bytes must be positive")
|
||||||
|
resolved_chunk_size = chunk_size or StorageService.UPLOAD_CHUNK_SIZE
|
||||||
|
if resolved_chunk_size <= 0:
|
||||||
|
raise ValueError("chunk_size must be positive")
|
||||||
|
normalized_type = StorageService.normalize_dataset_type(dataset_type)
|
||||||
|
file_path = Path(
|
||||||
|
StorageService.dataset_file_path(
|
||||||
|
project_id,
|
||||||
|
dataset_id,
|
||||||
|
normalized_type,
|
||||||
|
original_filename,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
file_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
size_bytes = 0
|
||||||
|
try:
|
||||||
|
with file_path.open("wb") as stream:
|
||||||
|
while True:
|
||||||
|
chunk = await upload.read(resolved_chunk_size)
|
||||||
|
if not chunk:
|
||||||
|
break
|
||||||
|
size_bytes += len(chunk)
|
||||||
|
if size_bytes > max_bytes:
|
||||||
|
raise AppError(
|
||||||
|
code="UPLOAD_TOO_LARGE",
|
||||||
|
message="Upload exceeds the configured backend size limit.",
|
||||||
|
details={
|
||||||
|
"max_bytes": max_bytes,
|
||||||
|
"max_upload_mb": max_bytes // (1024 * 1024),
|
||||||
|
},
|
||||||
|
status_code=413,
|
||||||
|
)
|
||||||
|
stream.write(chunk)
|
||||||
|
digest.update(chunk)
|
||||||
|
except Exception:
|
||||||
|
file_path.unlink(missing_ok=True)
|
||||||
|
parent = file_path.parent
|
||||||
|
if parent.exists() and parent.is_dir() and not any(parent.iterdir()):
|
||||||
|
parent.rmdir()
|
||||||
|
raise
|
||||||
|
return {
|
||||||
|
"original_filename": StorageService._safe_filename(original_filename),
|
||||||
|
"stored_filename": file_path.name,
|
||||||
|
"content_type": content_type or "application/octet-stream",
|
||||||
|
"size_bytes": size_bytes,
|
||||||
|
"checksum_sha256": digest.hexdigest(),
|
||||||
|
"storage_path": str(file_path),
|
||||||
|
}
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def persist_dataset_file_from_path(
|
def persist_dataset_file_from_path(
|
||||||
project_id: str,
|
project_id: str,
|
||||||
@@ -198,6 +269,34 @@ class StorageService:
|
|||||||
}
|
}
|
||||||
return metadata
|
return metadata
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def persist_file_from_path(
|
||||||
|
storage_path: str,
|
||||||
|
source_path: str | Path,
|
||||||
|
original_filename: str,
|
||||||
|
content_type: str | None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
source = Path(source_path).resolve()
|
||||||
|
if not source.is_file():
|
||||||
|
raise FileNotFoundError(f"Source artifact does not exist: {source}")
|
||||||
|
target = Path(storage_path)
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
size_bytes = 0
|
||||||
|
with source.open("rb") as input_stream, target.open("wb") as output_stream:
|
||||||
|
for chunk in iter(lambda: input_stream.read(StorageService.UPLOAD_CHUNK_SIZE), b""):
|
||||||
|
output_stream.write(chunk)
|
||||||
|
digest.update(chunk)
|
||||||
|
size_bytes += len(chunk)
|
||||||
|
return {
|
||||||
|
"original_filename": StorageService._safe_filename(original_filename),
|
||||||
|
"stored_filename": target.name,
|
||||||
|
"content_type": content_type or "application/octet-stream",
|
||||||
|
"size_bytes": size_bytes,
|
||||||
|
"checksum_sha256": digest.hexdigest(),
|
||||||
|
"storage_path": str(target),
|
||||||
|
}
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def remove_dataset_file(path: str) -> None:
|
def remove_dataset_file(path: str) -> None:
|
||||||
target = Path(path)
|
target = Path(path)
|
||||||
|
|||||||
@@ -111,7 +111,6 @@ class TerrainAnalysisService:
|
|||||||
try:
|
try:
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import rasterio
|
import rasterio
|
||||||
from rasterio.features import geometry_mask
|
|
||||||
from rasterio.mask import mask
|
from rasterio.mask import mask
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
raise AppError(
|
raise AppError(
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from pathlib import Path
|
|||||||
from typing import Any, Callable
|
from typing import Any, Callable
|
||||||
from urllib.error import HTTPError, URLError
|
from urllib.error import HTTPError, URLError
|
||||||
from urllib.parse import urlencode
|
from urllib.parse import urlencode
|
||||||
from urllib.request import Request, urlopen
|
from urllib.request import Request
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from geoalchemy2.shape import to_shape
|
from geoalchemy2.shape import to_shape
|
||||||
@@ -407,7 +407,6 @@ class ThematicRasterAcquisitionService:
|
|||||||
if len(coverages) == 1:
|
if len(coverages) == 1:
|
||||||
return coverages[0]
|
return coverages[0]
|
||||||
try:
|
try:
|
||||||
import rasterio
|
|
||||||
from rasterio.io import MemoryFile
|
from rasterio.io import MemoryFile
|
||||||
from rasterio.merge import merge
|
from rasterio.merge import merge
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
|
|||||||
@@ -96,7 +96,6 @@ class ThematicRasterAnalysisService:
|
|||||||
try:
|
try:
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import rasterio
|
import rasterio
|
||||||
from rasterio.features import geometry_mask
|
|
||||||
from rasterio.mask import mask
|
from rasterio.mask import mask
|
||||||
except ImportError as exc:
|
except ImportError as exc:
|
||||||
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for thematic raster analysis", status_code=503) from exc
|
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for thematic raster analysis", status_code=503) from exc
|
||||||
|
|||||||
@@ -0,0 +1,485 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from hashlib import sha256
|
||||||
|
import json
|
||||||
|
from math import isfinite
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
from geoalchemy2.shape import to_shape
|
||||||
|
from pyproj import CRS, Transformer
|
||||||
|
from shapely.geometry import box
|
||||||
|
from shapely.ops import transform as shapely_transform
|
||||||
|
from shapely.ops import unary_union
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.models import Area, Dataset, DatasetVersion
|
||||||
|
from app.services.storage_service import StorageService
|
||||||
|
|
||||||
|
|
||||||
|
class TileManifestService:
|
||||||
|
"""Versioned provenance and integrity contract for inference tile sets."""
|
||||||
|
|
||||||
|
CONTRACT_KEY = "geointel.raster.tile-manifest"
|
||||||
|
CONTRACT_VERSION = "2.0.0"
|
||||||
|
_BINDING_FIELDS = (
|
||||||
|
"source_dataset_id",
|
||||||
|
"source_dataset_checksum_sha256",
|
||||||
|
"source_dataset_size_bytes",
|
||||||
|
"source_registry_id",
|
||||||
|
"source_snapshot_id",
|
||||||
|
"source_snapshot_checksum_sha256",
|
||||||
|
"data_contract_key",
|
||||||
|
"data_contract_version",
|
||||||
|
"source_version",
|
||||||
|
"dataset_version_id",
|
||||||
|
"dataset_version",
|
||||||
|
"dataset_version_checksum_sha256",
|
||||||
|
"source_area_id",
|
||||||
|
"source_area_geometry_sha256",
|
||||||
|
)
|
||||||
|
_REQUIRED_INFERENCE_BINDING_FIELDS = (
|
||||||
|
"source_dataset_checksum_sha256",
|
||||||
|
"source_registry_id",
|
||||||
|
"source_snapshot_id",
|
||||||
|
"source_snapshot_checksum_sha256",
|
||||||
|
"data_contract_key",
|
||||||
|
"data_contract_version",
|
||||||
|
"dataset_version_id",
|
||||||
|
"dataset_version",
|
||||||
|
"dataset_version_checksum_sha256",
|
||||||
|
)
|
||||||
|
_CHECKSUM_FIELDS = (
|
||||||
|
"source_dataset_checksum_sha256",
|
||||||
|
"source_snapshot_checksum_sha256",
|
||||||
|
"dataset_version_checksum_sha256",
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def file_sha256(path: str | Path) -> str:
|
||||||
|
digest = sha256()
|
||||||
|
with Path(path).open("rb") as stream:
|
||||||
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _latest_dataset_version(dataset: Dataset) -> DatasetVersion | None:
|
||||||
|
versions = list(dataset.versions or [])
|
||||||
|
if not versions:
|
||||||
|
return None
|
||||||
|
return max(versions, key=lambda item: (int(item.version or 0), str(item.id or "")))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _source_snapshot_checksum(dataset: Dataset) -> str | None:
|
||||||
|
snapshot = dataset.source_snapshot
|
||||||
|
checksum = getattr(snapshot, "checksum_sha256", None) if snapshot is not None else None
|
||||||
|
return str(checksum).lower() if checksum else None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _area_geometry_binding(db, dataset: Dataset) -> tuple[str | None, str | None]:
|
||||||
|
if dataset.area_id is None:
|
||||||
|
return None, None
|
||||||
|
area = db.get(Area, dataset.area_id)
|
||||||
|
if area is None or area.geometry is None:
|
||||||
|
return str(dataset.area_id), None
|
||||||
|
geometry = to_shape(area.geometry)
|
||||||
|
return str(dataset.area_id), sha256(geometry.wkb).hexdigest()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def dataset_binding(cls, db, dataset: Dataset) -> dict[str, Any]:
|
||||||
|
version = cls._latest_dataset_version(dataset)
|
||||||
|
area_id, area_geometry_sha256 = cls._area_geometry_binding(db, dataset)
|
||||||
|
return {
|
||||||
|
"manifest_contract_key": cls.CONTRACT_KEY,
|
||||||
|
"manifest_contract_version": cls.CONTRACT_VERSION,
|
||||||
|
"source_dataset_id": str(dataset.id),
|
||||||
|
"source_raster_id": str(dataset.id),
|
||||||
|
"source_dataset_checksum_sha256": (
|
||||||
|
str(dataset.checksum_sha256).lower() if dataset.checksum_sha256 else None
|
||||||
|
),
|
||||||
|
"source_dataset_size_bytes": dataset.size_bytes,
|
||||||
|
"source_registry_id": str(dataset.source_registry_id) if dataset.source_registry_id else None,
|
||||||
|
"source_snapshot_id": str(dataset.source_snapshot_id) if dataset.source_snapshot_id else None,
|
||||||
|
"source_snapshot_checksum_sha256": cls._source_snapshot_checksum(dataset),
|
||||||
|
"data_contract_key": dataset.data_contract_key,
|
||||||
|
"data_contract_version": dataset.data_contract_version,
|
||||||
|
"source_version": dataset.source_version,
|
||||||
|
"dataset_version_id": str(version.id) if version is not None and version.id else None,
|
||||||
|
"dataset_version": int(version.version) if version is not None and version.version is not None else None,
|
||||||
|
"dataset_version_checksum_sha256": (
|
||||||
|
str(version.checksum_sha256).lower()
|
||||||
|
if version is not None and version.checksum_sha256
|
||||||
|
else None
|
||||||
|
),
|
||||||
|
"source_area_id": area_id,
|
||||||
|
"source_area_geometry_sha256": area_geometry_sha256,
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def tile_integrity(path: str | Path) -> dict[str, Any]:
|
||||||
|
resolved = Path(path)
|
||||||
|
return {
|
||||||
|
"size_bytes": resolved.stat().st_size,
|
||||||
|
"sha256": TileManifestService.file_sha256(resolved),
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _error(
|
||||||
|
error_prefix: str,
|
||||||
|
suffix: str,
|
||||||
|
message: str,
|
||||||
|
*,
|
||||||
|
details: dict[str, Any] | None = None,
|
||||||
|
) -> AppError:
|
||||||
|
return AppError(
|
||||||
|
code=f"{error_prefix}_TILE_MANIFEST_{suffix}",
|
||||||
|
message=message,
|
||||||
|
details=details,
|
||||||
|
status_code=422,
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _bounds_values(value: Any) -> tuple[float, float, float, float] | None:
|
||||||
|
if isinstance(value, dict):
|
||||||
|
aliases = (
|
||||||
|
("min_x", "min_y", "max_x", "max_y"),
|
||||||
|
("minx", "miny", "maxx", "maxy"),
|
||||||
|
("left", "bottom", "right", "top"),
|
||||||
|
)
|
||||||
|
selected = next(
|
||||||
|
([value.get(key) for key in keys] for keys in aliases if all(key in value for key in keys)),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
elif isinstance(value, (list, tuple)) and len(value) == 4:
|
||||||
|
selected = list(value)
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
bounds = tuple(float(item) for item in selected) if selected is not None else None
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
if bounds is None or not all(isfinite(item) for item in bounds):
|
||||||
|
return None
|
||||||
|
if bounds[0] >= bounds[2] or bounds[1] >= bounds[3]:
|
||||||
|
return None
|
||||||
|
return bounds
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _to_epsg4326(bounds: tuple[float, float, float, float], raw_crs: Any):
|
||||||
|
source_crs = CRS.from_user_input(raw_crs)
|
||||||
|
geometry = box(*bounds)
|
||||||
|
if not source_crs.equals(CRS.from_epsg(4326)):
|
||||||
|
transformer = Transformer.from_crs(source_crs, "EPSG:4326", always_xy=True)
|
||||||
|
geometry = shapely_transform(transformer.transform, geometry)
|
||||||
|
if geometry.is_empty or not geometry.is_valid:
|
||||||
|
raise ValueError("Bounds do not form a valid transformed geometry")
|
||||||
|
if not all(isfinite(float(value)) for value in geometry.bounds):
|
||||||
|
raise ValueError("Bounds transform to non-finite coordinates")
|
||||||
|
return geometry
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _manifest_coverage(cls, manifest: dict[str, Any], *, error_prefix: str):
|
||||||
|
tiles = manifest.get("tiles")
|
||||||
|
default_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs")
|
||||||
|
parts = []
|
||||||
|
for index, tile in enumerate(tiles if isinstance(tiles, list) else []):
|
||||||
|
if not isinstance(tile, dict):
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"SCOPE_MISMATCH",
|
||||||
|
"Tile manifest entries must be objects with explicit spatial metadata.",
|
||||||
|
details={"tile_index": index},
|
||||||
|
)
|
||||||
|
bounds = cls._bounds_values(tile.get("bounds"))
|
||||||
|
raw_crs = tile.get("crs") or default_crs
|
||||||
|
if bounds is None or not raw_crs:
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"SCOPE_MISMATCH",
|
||||||
|
"Every inference tile requires finite bounds and an explicit CRS.",
|
||||||
|
details={"tile_index": index},
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
parts.append(cls._to_epsg4326(bounds, raw_crs))
|
||||||
|
except Exception as exc:
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"SCOPE_MISMATCH",
|
||||||
|
"Inference tile bounds or CRS could not be normalized to EPSG:4326.",
|
||||||
|
details={"tile_index": index, "reason": str(exc)},
|
||||||
|
) from exc
|
||||||
|
coverage = unary_union(parts)
|
||||||
|
if coverage.is_empty or not coverage.is_valid:
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"SCOPE_MISMATCH",
|
||||||
|
"Inference tile union is empty or invalid.",
|
||||||
|
)
|
||||||
|
min_x, min_y, max_x, max_y = coverage.bounds
|
||||||
|
if min_x < -180 or min_y < -90 or max_x > 180 or max_y > 90:
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"SCOPE_MISMATCH",
|
||||||
|
"Inference tile union falls outside EPSG:4326 bounds.",
|
||||||
|
details={"bounds": list(coverage.bounds)},
|
||||||
|
)
|
||||||
|
return coverage
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _validate_binding(cls, db, dataset: Dataset, manifest: dict[str, Any], *, error_prefix: str) -> dict[str, Any]:
|
||||||
|
if (
|
||||||
|
manifest.get("manifest_contract_key") != cls.CONTRACT_KEY
|
||||||
|
or manifest.get("manifest_contract_version") != cls.CONTRACT_VERSION
|
||||||
|
):
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"PROVENANCE_MISMATCH",
|
||||||
|
"Inference requires a versioned GeoIntel tile-manifest contract.",
|
||||||
|
details={
|
||||||
|
"required_contract": f"{cls.CONTRACT_KEY}@{cls.CONTRACT_VERSION}",
|
||||||
|
"manifest_contract": (
|
||||||
|
f"{manifest.get('manifest_contract_key')}@{manifest.get('manifest_contract_version')}"
|
||||||
|
),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
expected = cls.dataset_binding(db, dataset)
|
||||||
|
missing = [
|
||||||
|
field
|
||||||
|
for field in cls._REQUIRED_INFERENCE_BINDING_FIELDS
|
||||||
|
if expected.get(field) in {None, ""}
|
||||||
|
]
|
||||||
|
invalid_checksums = [
|
||||||
|
field
|
||||||
|
for field in cls._CHECKSUM_FIELDS
|
||||||
|
if len(str(expected.get(field) or "")) != 64
|
||||||
|
or any(character not in "0123456789abcdef" for character in str(expected.get(field) or "").lower())
|
||||||
|
]
|
||||||
|
if missing or invalid_checksums:
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"PROVENANCE_MISMATCH",
|
||||||
|
"The requested Dataset lacks complete immutable provenance for inference tiling.",
|
||||||
|
details={
|
||||||
|
"missing_fields": missing,
|
||||||
|
"invalid_checksum_fields": invalid_checksums,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
manifest_dataset_id = manifest.get("source_dataset_id") or manifest.get("source_raster_id")
|
||||||
|
if str(manifest_dataset_id or "") != expected["source_dataset_id"]:
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"DATASET_MISMATCH",
|
||||||
|
"Tile manifest belongs to a different raster Dataset.",
|
||||||
|
details={
|
||||||
|
"requested_dataset_id": expected["source_dataset_id"],
|
||||||
|
"manifest_dataset_id": manifest_dataset_id,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
mismatches = {}
|
||||||
|
for field in cls._BINDING_FIELDS:
|
||||||
|
expected_value = expected.get(field)
|
||||||
|
if expected_value is None or field == "source_dataset_id":
|
||||||
|
continue
|
||||||
|
observed_value = manifest.get(field)
|
||||||
|
if str(observed_value) != str(expected_value):
|
||||||
|
mismatches[field] = {"expected": expected_value, "observed": observed_value}
|
||||||
|
if mismatches:
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"PROVENANCE_MISMATCH",
|
||||||
|
"Tile manifest provenance no longer matches the requested Dataset snapshot.",
|
||||||
|
details={"mismatches": mismatches},
|
||||||
|
)
|
||||||
|
return expected
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _validate_tile_files(
|
||||||
|
cls,
|
||||||
|
manifest: dict[str, Any],
|
||||||
|
manifest_path: Path,
|
||||||
|
*,
|
||||||
|
settings,
|
||||||
|
error_prefix: str,
|
||||||
|
) -> list[str]:
|
||||||
|
resolved_paths: list[str] = []
|
||||||
|
seen_paths: set[Path] = set()
|
||||||
|
for index, tile in enumerate(manifest["tiles"]):
|
||||||
|
raw_path = tile.get("path") if isinstance(tile, dict) else None
|
||||||
|
if not isinstance(raw_path, str) or not raw_path.strip():
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"TILE_INTEGRITY_MISMATCH",
|
||||||
|
"Every inference tile requires a path and immutable integrity evidence.",
|
||||||
|
details={"tile_index": index},
|
||||||
|
)
|
||||||
|
candidate = Path(raw_path).expanduser()
|
||||||
|
if not candidate.is_absolute():
|
||||||
|
candidate = manifest_path.parent / candidate
|
||||||
|
candidate = StorageService.assert_within_storage_root(
|
||||||
|
candidate,
|
||||||
|
label="raster tile",
|
||||||
|
settings=settings,
|
||||||
|
)
|
||||||
|
if not candidate.is_file():
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"TILE_INTEGRITY_MISMATCH",
|
||||||
|
"An inference tile referenced by the manifest does not exist.",
|
||||||
|
details={"tile_index": index, "tile_path": str(candidate)},
|
||||||
|
)
|
||||||
|
if candidate in seen_paths:
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"TILE_INTEGRITY_MISMATCH",
|
||||||
|
"A tile path occurs more than once in the inference manifest.",
|
||||||
|
details={"tile_index": index, "tile_path": str(candidate)},
|
||||||
|
)
|
||||||
|
seen_paths.add(candidate)
|
||||||
|
observed_size = candidate.stat().st_size
|
||||||
|
expected_size = tile.get("size_bytes")
|
||||||
|
expected_checksum = str(tile.get("sha256") or "").strip().lower()
|
||||||
|
if expected_size != observed_size or len(expected_checksum) != 64:
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"TILE_INTEGRITY_MISMATCH",
|
||||||
|
"Tile size/checksum evidence is missing or no longer matches the staged file.",
|
||||||
|
details={
|
||||||
|
"tile_index": index,
|
||||||
|
"expected_size_bytes": expected_size,
|
||||||
|
"observed_size_bytes": observed_size,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
observed_checksum = cls.file_sha256(candidate)
|
||||||
|
if observed_checksum != expected_checksum:
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"TILE_INTEGRITY_MISMATCH",
|
||||||
|
"Tile checksum no longer matches the immutable manifest evidence.",
|
||||||
|
details={
|
||||||
|
"tile_index": index,
|
||||||
|
"expected_sha256": expected_checksum,
|
||||||
|
"observed_sha256": observed_checksum,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
resolved_paths.append(str(candidate))
|
||||||
|
declared_count = manifest.get("count")
|
||||||
|
if declared_count != len(resolved_paths):
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"TILE_INTEGRITY_MISMATCH",
|
||||||
|
"Tile manifest count does not match its tile records.",
|
||||||
|
details={"declared_count": declared_count, "tile_count": len(resolved_paths)},
|
||||||
|
)
|
||||||
|
return resolved_paths
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _validate_scope(
|
||||||
|
cls,
|
||||||
|
db,
|
||||||
|
dataset: Dataset,
|
||||||
|
manifest: dict[str, Any],
|
||||||
|
coverage,
|
||||||
|
*,
|
||||||
|
error_prefix: str,
|
||||||
|
) -> None:
|
||||||
|
manifest_bounds = cls._bounds_values(manifest.get("bounds"))
|
||||||
|
manifest_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs")
|
||||||
|
dataset_bounds = cls._bounds_values(dataset.bounds_json)
|
||||||
|
if dataset_bounds is None and isinstance(dataset.metadata_json, dict):
|
||||||
|
dataset_bounds = cls._bounds_values(
|
||||||
|
dataset.metadata_json.get("bounds_json") or dataset.metadata_json.get("bounds")
|
||||||
|
)
|
||||||
|
if manifest_bounds is None or not manifest_crs or dataset_bounds is None or not dataset.crs:
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"SCOPE_MISMATCH",
|
||||||
|
"Dataset and tile manifest require explicit CRS and finite bounds for inference.",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
manifest_extent = cls._to_epsg4326(manifest_bounds, manifest_crs)
|
||||||
|
dataset_extent = cls._to_epsg4326(dataset_bounds, dataset.crs)
|
||||||
|
except Exception as exc:
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"SCOPE_MISMATCH",
|
||||||
|
"Dataset or manifest bounds could not be normalized to EPSG:4326.",
|
||||||
|
details={"reason": str(exc)},
|
||||||
|
) from exc
|
||||||
|
tolerance = max(dataset_extent.bounds[2] - dataset_extent.bounds[0], dataset_extent.bounds[3] - dataset_extent.bounds[1]) * 1e-7 + 1e-10
|
||||||
|
if not manifest_extent.buffer(tolerance).covers(coverage):
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"SCOPE_MISMATCH",
|
||||||
|
"Tile union exceeds the extent declared by its manifest.",
|
||||||
|
details={"tile_union_bounds": list(coverage.bounds), "manifest_bounds": list(manifest_extent.bounds)},
|
||||||
|
)
|
||||||
|
if not dataset_extent.buffer(tolerance).covers(coverage):
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"SCOPE_MISMATCH",
|
||||||
|
"Tile union exceeds the persisted Dataset extent.",
|
||||||
|
details={"tile_union_bounds": list(coverage.bounds), "dataset_bounds": list(dataset_extent.bounds)},
|
||||||
|
)
|
||||||
|
if dataset.area_id is not None:
|
||||||
|
area = db.get(Area, dataset.area_id)
|
||||||
|
if area is None or area.geometry is None:
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"SCOPE_MISMATCH",
|
||||||
|
"Dataset references an Area that is unavailable for inference-scope validation.",
|
||||||
|
details={"area_id": str(dataset.area_id)},
|
||||||
|
)
|
||||||
|
area_geometry = to_shape(area.geometry)
|
||||||
|
if area_geometry.is_empty or not area_geometry.is_valid or not coverage.intersects(area_geometry):
|
||||||
|
raise cls._error(
|
||||||
|
error_prefix,
|
||||||
|
"SCOPE_MISMATCH",
|
||||||
|
"Tile union does not overlap the persisted Dataset Area.",
|
||||||
|
details={"area_id": str(dataset.area_id), "tile_union_bounds": list(coverage.bounds)},
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def validate_for_inference(
|
||||||
|
cls,
|
||||||
|
db,
|
||||||
|
dataset: Dataset,
|
||||||
|
manifest: dict[str, Any],
|
||||||
|
*,
|
||||||
|
manifest_path: str | Path,
|
||||||
|
settings,
|
||||||
|
error_prefix: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
resolved_manifest_path = StorageService.assert_within_storage_root(
|
||||||
|
manifest_path,
|
||||||
|
label="tile manifest",
|
||||||
|
settings=settings,
|
||||||
|
)
|
||||||
|
expected = cls._validate_binding(db, dataset, manifest, error_prefix=error_prefix)
|
||||||
|
resolved_paths = cls._validate_tile_files(
|
||||||
|
manifest,
|
||||||
|
resolved_manifest_path,
|
||||||
|
settings=settings,
|
||||||
|
error_prefix=error_prefix,
|
||||||
|
)
|
||||||
|
coverage = cls._manifest_coverage(manifest, error_prefix=error_prefix)
|
||||||
|
cls._validate_scope(db, dataset, manifest, coverage, error_prefix=error_prefix)
|
||||||
|
return {
|
||||||
|
"manifest_contract_key": cls.CONTRACT_KEY,
|
||||||
|
"manifest_contract_version": cls.CONTRACT_VERSION,
|
||||||
|
"manifest_path": str(resolved_manifest_path),
|
||||||
|
"manifest_sha256": cls.file_sha256(resolved_manifest_path),
|
||||||
|
"source_dataset_id": expected["source_dataset_id"],
|
||||||
|
"source_dataset_checksum_sha256": expected.get("source_dataset_checksum_sha256"),
|
||||||
|
"source_snapshot_id": expected.get("source_snapshot_id"),
|
||||||
|
"dataset_version_id": expected.get("dataset_version_id"),
|
||||||
|
"source_area_id": expected.get("source_area_id"),
|
||||||
|
"tile_count": len(resolved_paths),
|
||||||
|
"tile_union_bounds_epsg4326": [float(value) for value in coverage.bounds],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def canonical_manifest_json(payload: dict[str, Any]) -> str:
|
||||||
|
"""Stable serializer shared by the writer and manifest-hash tests."""
|
||||||
|
|
||||||
|
return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
|
||||||
@@ -1,43 +1,101 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from math import isfinite
|
||||||
|
from numbers import Real
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from pyproj import Transformer
|
from pyproj import CRS, Transformer
|
||||||
from shapely import force_2d
|
from shapely import force_2d, get_coordinates
|
||||||
from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, box, shape
|
from shapely.geometry import MultiPolygon, box, shape
|
||||||
from shapely.ops import transform
|
from shapely.ops import transform
|
||||||
from shapely.validation import make_valid
|
|
||||||
|
|
||||||
|
# This is a deliberately broad guard envelope around Belgium and the Belgian
|
||||||
|
# North Sea. Exact legal/regional clipping remains the responsibility of the
|
||||||
|
# persisted coverage Areas; this boundary prevents an AOI with valid-looking
|
||||||
|
# but globally misplaced coordinates from entering the workbench.
|
||||||
|
BELGIUM_AND_NORTH_SEA_GUARD_BOUNDS = (1.5, 48.5, 7.5, 52.5)
|
||||||
|
|
||||||
|
|
||||||
|
def _raw_coordinates_are_finite(value: Any) -> bool:
|
||||||
|
if isinstance(value, (list, tuple)):
|
||||||
|
return bool(value) and all(_raw_coordinates_are_finite(item) for item in value)
|
||||||
|
return isinstance(value, Real) and not isinstance(value, bool) and isfinite(float(value))
|
||||||
|
|
||||||
|
|
||||||
def normalize_to_multipolygon(raw_geometry: dict[str, Any]) -> MultiPolygon:
|
def normalize_to_multipolygon(raw_geometry: dict[str, Any]) -> MultiPolygon:
|
||||||
geom = force_2d(shape(raw_geometry))
|
if isinstance(raw_geometry, dict) and "coordinates" in raw_geometry:
|
||||||
|
if not _raw_coordinates_are_finite(raw_geometry["coordinates"]):
|
||||||
|
raise ValueError("Geometry coordinates must be finite numbers")
|
||||||
|
try:
|
||||||
|
geom = force_2d(shape(raw_geometry))
|
||||||
|
except Exception as exc:
|
||||||
|
raise ValueError("Geometry is not valid GeoJSON") from exc
|
||||||
|
coordinates = get_coordinates(geom, include_z=False)
|
||||||
|
if coordinates.size == 0 or not all(isfinite(float(value)) for row in coordinates for value in row):
|
||||||
|
raise ValueError("Geometry coordinates must be finite numbers")
|
||||||
if geom.is_empty:
|
if geom.is_empty:
|
||||||
raise ValueError("Geometry is empty")
|
raise ValueError("Geometry is empty")
|
||||||
|
|
||||||
if not geom.is_valid:
|
if not geom.is_valid:
|
||||||
geom = make_valid(geom)
|
raise ValueError("Geometry is invalid")
|
||||||
|
|
||||||
if not geom.is_valid:
|
|
||||||
raise ValueError("Geometry is invalid and could not be repaired")
|
|
||||||
|
|
||||||
if geom.geom_type == "Polygon":
|
if geom.geom_type == "Polygon":
|
||||||
return MultiPolygon([geom])
|
return MultiPolygon([geom])
|
||||||
if geom.geom_type == "MultiPolygon":
|
if geom.geom_type == "MultiPolygon":
|
||||||
return MultiPolygon(geom.geoms)
|
return MultiPolygon(geom.geoms)
|
||||||
if isinstance(geom, GeometryCollection):
|
|
||||||
polygons = [g for g in geom.geoms if isinstance(g, Polygon)]
|
|
||||||
multipolygons = [g for g in geom.geoms if g.geom_type == "MultiPolygon"]
|
|
||||||
if not polygons and not multipolygons:
|
|
||||||
raise ValueError("Only polygon geometries are supported for AOI")
|
|
||||||
normalized = []
|
|
||||||
normalized.extend(polygons)
|
|
||||||
for mp in multipolygons:
|
|
||||||
normalized.extend(mp.geoms)
|
|
||||||
return MultiPolygon(normalized)
|
|
||||||
|
|
||||||
raise ValueError("Only Polygon or MultiPolygon geometries are accepted")
|
raise ValueError("Only Polygon or MultiPolygon geometries are accepted")
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_area_to_epsg4326(
|
||||||
|
raw_geometry: dict[str, Any],
|
||||||
|
source_crs: str,
|
||||||
|
) -> tuple[MultiPolygon, str]:
|
||||||
|
"""Validate an AOI and normalize its declared CRS to canonical WGS84.
|
||||||
|
|
||||||
|
The returned CRS string preserves the caller's declaration for provenance;
|
||||||
|
the returned geometry is always finite, polygonal and stored as EPSG:4326.
|
||||||
|
"""
|
||||||
|
|
||||||
|
declared_crs = str(source_crs or "").strip()
|
||||||
|
if not declared_crs:
|
||||||
|
raise ValueError("Area CRS is required")
|
||||||
|
try:
|
||||||
|
parsed_crs = CRS.from_user_input(declared_crs)
|
||||||
|
except Exception as exc:
|
||||||
|
raise ValueError("Area CRS is unknown or invalid") from exc
|
||||||
|
if not (parsed_crs.is_geographic or parsed_crs.is_projected):
|
||||||
|
raise ValueError("Area CRS must be a geographic or projected two-dimensional CRS")
|
||||||
|
if len(parsed_crs.axis_info) != 2:
|
||||||
|
raise ValueError("Area CRS must have exactly two spatial axes")
|
||||||
|
|
||||||
|
geometry = normalize_to_multipolygon(raw_geometry)
|
||||||
|
target_crs = CRS.from_epsg(4326)
|
||||||
|
if not parsed_crs.equals(target_crs):
|
||||||
|
try:
|
||||||
|
transformer = Transformer.from_crs(parsed_crs, target_crs, always_xy=True)
|
||||||
|
geometry = normalize_to_multipolygon(
|
||||||
|
transform(transformer.transform, geometry).__geo_interface__
|
||||||
|
)
|
||||||
|
except ValueError:
|
||||||
|
raise
|
||||||
|
except Exception as exc:
|
||||||
|
raise ValueError("Area geometry could not be transformed to EPSG:4326") from exc
|
||||||
|
|
||||||
|
min_x, min_y, max_x, max_y = geometry.bounds
|
||||||
|
if not all(isfinite(value) for value in (min_x, min_y, max_x, max_y)):
|
||||||
|
raise ValueError("Transformed area geometry contains non-finite coordinates")
|
||||||
|
world_bounds = (-180.0, -90.0, 180.0, 90.0)
|
||||||
|
if min_x < world_bounds[0] or min_y < world_bounds[1] or max_x > world_bounds[2] or max_y > world_bounds[3]:
|
||||||
|
raise ValueError("Transformed area geometry falls outside the EPSG:4326 coordinate domain")
|
||||||
|
guard = box(*BELGIUM_AND_NORTH_SEA_GUARD_BOUNDS)
|
||||||
|
if not guard.intersects(geometry):
|
||||||
|
raise ValueError("Area geometry falls outside Belgium and the Belgian North Sea workbench domain")
|
||||||
|
if not guard.covers(geometry):
|
||||||
|
raise ValueError("Area geometry must remain within the Belgium and Belgian North Sea workbench domain")
|
||||||
|
return geometry, declared_crs
|
||||||
|
|
||||||
|
|
||||||
def area_bounds_multipolygon(geom: MultiPolygon):
|
def area_bounds_multipolygon(geom: MultiPolygon):
|
||||||
return {
|
return {
|
||||||
"min_x": float(geom.bounds[0]),
|
"min_x": float(geom.bounds[0]),
|
||||||
@@ -52,7 +110,10 @@ def area_m2(geom: MultiPolygon) -> float:
|
|||||||
Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True).transform,
|
Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True).transform,
|
||||||
geom,
|
geom,
|
||||||
)
|
)
|
||||||
return float(projected.area)
|
result = float(projected.area)
|
||||||
|
if not isfinite(result) or result <= 0:
|
||||||
|
raise ValueError("Area geometry must have a finite positive surface")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
def geometry_bbox_polygon(geom: MultiPolygon):
|
def geometry_bbox_polygon(geom: MultiPolygon):
|
||||||
|
|||||||
@@ -14,10 +14,13 @@ SCRIPTS_ROOT = REPOSITORY_ROOT / "scripts"
|
|||||||
if SCRIPTS_ROOT.is_dir():
|
if SCRIPTS_ROOT.is_dir():
|
||||||
sys.path.insert(0, str(SCRIPTS_ROOT))
|
sys.path.insert(0, str(SCRIPTS_ROOT))
|
||||||
|
|
||||||
from app.core.config import get_settings
|
from app.core.config import get_settings # noqa: E402 - imported after backend path bootstrap
|
||||||
from app.db.session import SessionLocal
|
from app.db.session import SessionLocal # noqa: E402 - imported after backend path bootstrap
|
||||||
from app.models import Export, Project
|
from app.models import Export, Project # noqa: E402 - imported after backend path bootstrap
|
||||||
from release_backup_guard import require_confirmation, verify_current_backup
|
from release_backup_guard import ( # noqa: E402 - imported after scripts path bootstrap
|
||||||
|
require_confirmation,
|
||||||
|
verify_current_backup,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
DEMO_PROJECT_NAME = "GeoIntel Demo - Building QA"
|
DEMO_PROJECT_NAME = "GeoIntel Demo - Building QA"
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from uuid import uuid4
|
|||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.models import AnalysisRun, Detection, Job
|
from app.models import Job
|
||||||
from app.services.analysis_job_worker import AnalysisJobWorker
|
from app.services.analysis_job_worker import AnalysisJobWorker
|
||||||
from app.services.detection_service import DetectionService
|
from app.services.detection_service import DetectionService
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,159 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from geoalchemy2.shape import from_shape, to_shape
|
||||||
|
from pyproj import Transformer
|
||||||
|
import pytest
|
||||||
|
from shapely.geometry import Polygon, mapping
|
||||||
|
from shapely.ops import transform
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.models import Area, Project
|
||||||
|
from app.schemas.area import AreaCreate, AreaUpdate
|
||||||
|
from app.services.area_service import AreaService
|
||||||
|
from app.utils.geometry import area_m2, normalize_area_to_epsg4326
|
||||||
|
|
||||||
|
|
||||||
|
class FakeSession:
|
||||||
|
def __init__(self, objects=None) -> None:
|
||||||
|
self.objects = objects or {}
|
||||||
|
self.added = []
|
||||||
|
self.commits = 0
|
||||||
|
self.refreshes = []
|
||||||
|
|
||||||
|
def get(self, model, item_id):
|
||||||
|
return self.objects.get((model, item_id))
|
||||||
|
|
||||||
|
def add(self, item) -> None:
|
||||||
|
self.added.append(item)
|
||||||
|
|
||||||
|
def commit(self) -> None:
|
||||||
|
self.commits += 1
|
||||||
|
|
||||||
|
def refresh(self, item) -> None:
|
||||||
|
self.refreshes.append(item)
|
||||||
|
|
||||||
|
|
||||||
|
def _wgs84_polygon(offset: float = 0.0) -> Polygon:
|
||||||
|
return Polygon(
|
||||||
|
[
|
||||||
|
(5.00 + offset, 51.00),
|
||||||
|
(5.01 + offset, 51.00),
|
||||||
|
(5.01 + offset, 51.01),
|
||||||
|
(5.00 + offset, 51.01),
|
||||||
|
(5.00 + offset, 51.00),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _to_lambert(geometry: Polygon) -> Polygon:
|
||||||
|
transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
||||||
|
return transform(transformer.transform, geometry)
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_area_transforms_declared_lambert_geometry_before_storage() -> None:
|
||||||
|
project_id = uuid4()
|
||||||
|
db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")})
|
||||||
|
source = _wgs84_polygon()
|
||||||
|
|
||||||
|
area = AreaService.create_area(
|
||||||
|
db,
|
||||||
|
project_id,
|
||||||
|
AreaCreate(name="Lambert AOI", geometry=mapping(_to_lambert(source)), crs="EPSG:31370"),
|
||||||
|
)
|
||||||
|
|
||||||
|
stored = to_shape(area.geometry)
|
||||||
|
assert stored.bounds == pytest.approx(source.bounds, abs=1e-7)
|
||||||
|
assert area.original_crs == "EPSG:31370"
|
||||||
|
assert area.area_m2 == pytest.approx(area_m2(normalize_area_to_epsg4326(mapping(source), "EPSG:4326")[0]))
|
||||||
|
assert area.area_m2 and area.area_m2 > 0
|
||||||
|
assert to_shape(area.bbox).bounds == pytest.approx(source.bounds, abs=1e-7)
|
||||||
|
|
||||||
|
|
||||||
|
def test_patch_area_replaces_geometry_and_recomputes_all_spatial_fields() -> None:
|
||||||
|
area_id = uuid4()
|
||||||
|
project_id = uuid4()
|
||||||
|
original = _wgs84_polygon()
|
||||||
|
normalized, _ = normalize_area_to_epsg4326(mapping(original), "EPSG:4326")
|
||||||
|
area = Area(
|
||||||
|
id=area_id,
|
||||||
|
project_id=project_id,
|
||||||
|
name="Original",
|
||||||
|
geometry=from_shape(normalized, srid=4326),
|
||||||
|
bbox=from_shape(normalized.envelope, srid=4326),
|
||||||
|
original_crs="EPSG:4326",
|
||||||
|
area_m2=area_m2(normalized),
|
||||||
|
)
|
||||||
|
db = FakeSession({(Area, area_id): area})
|
||||||
|
replacement = _wgs84_polygon(offset=0.05)
|
||||||
|
|
||||||
|
updated = AreaService.update_area(
|
||||||
|
db,
|
||||||
|
area_id,
|
||||||
|
AreaUpdate(
|
||||||
|
name="Replacement",
|
||||||
|
geometry=mapping(_to_lambert(replacement)),
|
||||||
|
crs="EPSG:31370",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert updated.name == "Replacement"
|
||||||
|
assert updated.original_crs == "EPSG:31370"
|
||||||
|
assert to_shape(updated.geometry).bounds == pytest.approx(replacement.bounds, abs=1e-7)
|
||||||
|
assert to_shape(updated.bbox).bounds == pytest.approx(replacement.bounds, abs=1e-7)
|
||||||
|
assert updated.area_m2 and updated.area_m2 > 0
|
||||||
|
assert db.commits == 1
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("geometry", "crs", "message_fragment"),
|
||||||
|
[
|
||||||
|
(mapping(_wgs84_polygon()), "EPSG:not-real", "unknown or invalid"),
|
||||||
|
(mapping(_wgs84_polygon()), "EPSG:4979", "exactly two spatial axes"),
|
||||||
|
(
|
||||||
|
{
|
||||||
|
"type": "Polygon",
|
||||||
|
"coordinates": [[[5.0, 51.0], [float("nan"), 51.0], [5.1, 51.1], [5.0, 51.0]]],
|
||||||
|
},
|
||||||
|
"EPSG:4326",
|
||||||
|
"finite",
|
||||||
|
),
|
||||||
|
(mapping(Polygon([(10.0, 51.0), (10.1, 51.0), (10.1, 51.1), (10.0, 51.0)])), "EPSG:4326", "workbench domain"),
|
||||||
|
(
|
||||||
|
{
|
||||||
|
"type": "Polygon",
|
||||||
|
"coordinates": [[[5.0, 51.0], [5.1, 51.1], [5.1, 51.0], [5.0, 51.1], [5.0, 51.0]]],
|
||||||
|
},
|
||||||
|
"EPSG:4326",
|
||||||
|
"invalid",
|
||||||
|
),
|
||||||
|
({"type": "Point", "coordinates": [5.0, 51.0]}, "EPSG:4326", "Polygon or MultiPolygon"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_create_area_rejects_invalid_crs_nonfinite_and_out_of_domain_geometry(
|
||||||
|
geometry: dict,
|
||||||
|
crs: str,
|
||||||
|
message_fragment: str,
|
||||||
|
) -> None:
|
||||||
|
project_id = uuid4()
|
||||||
|
db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")})
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
AreaService.create_area(db, project_id, AreaCreate(name="Invalid", geometry=geometry, crs=crs))
|
||||||
|
|
||||||
|
assert exc_info.value.code == "INVALID_GEOMETRY"
|
||||||
|
assert message_fragment in exc_info.value.message
|
||||||
|
assert db.commits == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_patch_area_rejects_crs_without_replacement_geometry() -> None:
|
||||||
|
area_id = uuid4()
|
||||||
|
area = Area(id=area_id, project_id=uuid4(), name="AOI", original_crs="EPSG:4326")
|
||||||
|
db = FakeSession({(Area, area_id): area})
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
AreaService.update_area(db, area_id, AreaUpdate(crs="EPSG:31370"))
|
||||||
|
|
||||||
|
assert exc_info.value.code == "INVALID_AREA_CRS_UPDATE"
|
||||||
|
assert db.commits == 0
|
||||||
@@ -190,6 +190,17 @@ def test_passed_manual_or_experimental_dataset_cannot_cross_production_boundary(
|
|||||||
assert "experimental_source_not_allowed_for_purpose" in exc_info.value.details["reasons"]
|
assert "experimental_source_not_allowed_for_purpose" in exc_info.value.details["reasons"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_fully_governed_demo_fixture_still_cannot_enter_production_inference() -> None:
|
||||||
|
fixture = _governed_dataset(source_key="fixture", classification="experimental")
|
||||||
|
fixture.source_metadata = {"fixture": True, "usage": "offline demo raster workflow only"}
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
DatasetConsumptionGate.assert_eligible(fixture, purpose="production_inference")
|
||||||
|
|
||||||
|
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||||
|
assert "experimental_source_not_allowed_for_purpose" in exc_info.value.details["reasons"]
|
||||||
|
|
||||||
|
|
||||||
def test_reference_validation_requires_authoritative_ground_truth_reference() -> None:
|
def test_reference_validation_requires_authoritative_ground_truth_reference() -> None:
|
||||||
reference = _governed_dataset()
|
reference = _governed_dataset()
|
||||||
reference.dataset_type = "vector"
|
reference.dataset_type = "vector"
|
||||||
|
|||||||
@@ -17,7 +17,9 @@ import pytest
|
|||||||
|
|
||||||
np = pytest.importorskip("numpy")
|
np = pytest.importorskip("numpy")
|
||||||
|
|
||||||
from app.services.flood_hazard_analysis_service import FloodHazardCellStatistics
|
from app.services.flood_hazard_analysis_service import ( # noqa: E402 - optional NumPy gate precedes service import
|
||||||
|
FloodHazardCellStatistics,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
NODATA = -9999.0
|
NODATA = -9999.0
|
||||||
|
|||||||
@@ -11,10 +11,20 @@ from fastapi.testclient import TestClient
|
|||||||
from app.core.config import Settings
|
from app.core.config import Settings
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.main import app
|
from app.main import app
|
||||||
from app.models import AnalysisRun, Dataset, Detection, Job, Project, SourceRegistry, SourceSnapshot
|
from app.models import (
|
||||||
|
AnalysisRun,
|
||||||
|
Dataset,
|
||||||
|
DatasetVersion,
|
||||||
|
Detection,
|
||||||
|
Job,
|
||||||
|
Project,
|
||||||
|
SourceRegistry,
|
||||||
|
SourceSnapshot,
|
||||||
|
)
|
||||||
from app.services.detection_service import DetectionService
|
from app.services.detection_service import DetectionService
|
||||||
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
||||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||||
|
from app.services.tile_manifest_service import TileManifestService
|
||||||
|
|
||||||
|
|
||||||
class FakeSession:
|
class FakeSession:
|
||||||
@@ -98,6 +108,8 @@ def _project_and_raster_dataset():
|
|||||||
source_name="test-derived-raster",
|
source_name="test-derived-raster",
|
||||||
storage_path="storage/uploads/source.tif",
|
storage_path="storage/uploads/source.tif",
|
||||||
checksum_sha256=checksum,
|
checksum_sha256=checksum,
|
||||||
|
crs="EPSG:4326",
|
||||||
|
bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0},
|
||||||
source_registry_id=source_registry_id,
|
source_registry_id=source_registry_id,
|
||||||
source_snapshot_id=source_snapshot_id,
|
source_snapshot_id=source_snapshot_id,
|
||||||
data_contract_key="geointel.raster.geotiff",
|
data_contract_key="geointel.raster.geotiff",
|
||||||
@@ -110,17 +122,22 @@ def _project_and_raster_dataset():
|
|||||||
)
|
)
|
||||||
dataset.source_registry = source_registry
|
dataset.source_registry = source_registry
|
||||||
dataset.source_snapshot = source_snapshot
|
dataset.source_snapshot = source_snapshot
|
||||||
|
dataset.versions.append(
|
||||||
|
DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1, checksum_sha256=checksum)
|
||||||
|
)
|
||||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||||
return db, project_id, dataset_id
|
return db, project_id, dataset_id
|
||||||
|
|
||||||
|
|
||||||
def _manifest(tmp_path: Path) -> Path:
|
def _manifest(tmp_path: Path, db: FakeSession, dataset: Dataset) -> Path:
|
||||||
tile_path = tmp_path / "tile_0000.tif"
|
tile_path = tmp_path / "tile_0000.tif"
|
||||||
tile_path.write_bytes(b"tile")
|
tile_path.write_bytes(b"tile")
|
||||||
|
binding = TileManifestService.dataset_binding(db, dataset)
|
||||||
manifest_path = tmp_path / "manifest.json"
|
manifest_path = tmp_path / "manifest.json"
|
||||||
manifest_path.write_text(
|
manifest_path.write_text(
|
||||||
json.dumps(
|
json.dumps(
|
||||||
{
|
{
|
||||||
|
**binding,
|
||||||
"tile_set_id": "tiles-fixture",
|
"tile_set_id": "tiles-fixture",
|
||||||
"count": 1,
|
"count": 1,
|
||||||
"crs": "EPSG:4326",
|
"crs": "EPSG:4326",
|
||||||
@@ -133,6 +150,7 @@ def _manifest(tmp_path: Path) -> Path:
|
|||||||
"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01],
|
"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01],
|
||||||
"crs": "EPSG:4326",
|
"crs": "EPSG:4326",
|
||||||
"index": 0,
|
"index": 0,
|
||||||
|
**TileManifestService.tile_integrity(tile_path),
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
}
|
}
|
||||||
@@ -226,7 +244,11 @@ def test_model_asset_catalog_lists_supported_local_model_files(tmp_path: Path) -
|
|||||||
assert asset.size_bytes == len(b"local model")
|
assert asset.size_bytes == len(b"local model")
|
||||||
assert len(asset.sha256) == 64
|
assert len(asset.sha256) == 64
|
||||||
assert asset.active is True
|
assert asset.active is True
|
||||||
assert asset.status == "approved"
|
assert asset.runtime_available is True
|
||||||
|
assert asset.runtime_status == "active"
|
||||||
|
assert asset.governed_validation_status == "not_verified_by_catalog"
|
||||||
|
assert asset.promotion_status == "not_verified_by_catalog"
|
||||||
|
assert asset.status == "runtime_active"
|
||||||
assert asset.will_download_models is False
|
assert asset.will_download_models is False
|
||||||
|
|
||||||
|
|
||||||
@@ -257,7 +279,10 @@ def test_model_asset_catalog_only_exposes_explicit_active_asset_in_runtime(tmp_p
|
|||||||
assert response.total == 1
|
assert response.total == 1
|
||||||
assert response.items[0].filename == active_file.name
|
assert response.items[0].filename == active_file.name
|
||||||
assert response.items[0].active is True
|
assert response.items[0].active is True
|
||||||
assert response.items[0].status == "approved"
|
assert response.items[0].runtime_status == "active"
|
||||||
|
assert response.items[0].governed_validation_status == "not_verified_by_catalog"
|
||||||
|
assert response.items[0].promotion_status == "not_verified_by_catalog"
|
||||||
|
assert response.items[0].status == "runtime_active"
|
||||||
|
|
||||||
|
|
||||||
def test_model_asset_catalog_rejects_unknown_asset(tmp_path: Path) -> None:
|
def test_model_asset_catalog_rejects_unknown_asset(tmp_path: Path) -> None:
|
||||||
@@ -284,6 +309,9 @@ def test_model_assets_api_returns_canonical_envelope(monkeypatch, tmp_path: Path
|
|||||||
assert payload["data"]["total"] == 1
|
assert payload["data"]["total"] == 1
|
||||||
assert payload["data"]["items"][0]["model_asset_id"] == "building-detector-pt"
|
assert payload["data"]["items"][0]["model_asset_id"] == "building-detector-pt"
|
||||||
assert payload["data"]["items"][0]["active"] is True
|
assert payload["data"]["items"][0]["active"] is True
|
||||||
|
assert payload["data"]["items"][0]["runtime_status"] == "active"
|
||||||
|
assert payload["data"]["items"][0]["governed_validation_status"] == "not_verified_by_catalog"
|
||||||
|
assert payload["data"]["items"][0]["promotion_status"] == "not_verified_by_catalog"
|
||||||
assert payload["data"]["items"][0]["will_download_models"] is False
|
assert payload["data"]["items"][0]["will_download_models"] is False
|
||||||
|
|
||||||
|
|
||||||
@@ -309,7 +337,7 @@ def test_detection_run_persists_selected_model_asset_parameters(tmp_path, monkey
|
|||||||
model_id="yolo-configured",
|
model_id="yolo-configured",
|
||||||
model_asset_id="building-detector-pt",
|
model_asset_id="building-detector-pt",
|
||||||
confidence_threshold=0.5,
|
confidence_threshold=0.5,
|
||||||
tile_manifest_path=str(_manifest(tmp_path)),
|
tile_manifest_path=str(_manifest(tmp_path, db, db.get(Dataset, dataset_id))),
|
||||||
settings=settings,
|
settings=settings,
|
||||||
yolo_adapter_class=MockYoloAdapter,
|
yolo_adapter_class=MockYoloAdapter,
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -18,10 +18,10 @@ import pytest
|
|||||||
np = pytest.importorskip("numpy")
|
np = pytest.importorskip("numpy")
|
||||||
rasterio = pytest.importorskip("rasterio")
|
rasterio = pytest.importorskip("rasterio")
|
||||||
|
|
||||||
from rasterio.transform import from_origin
|
from rasterio.transform import from_origin # noqa: E402 - optional rasterio gate precedes imports
|
||||||
from shapely.geometry import box
|
from shapely.geometry import box # noqa: E402 - optional rasterio gate precedes imports
|
||||||
|
|
||||||
from app.services.raster_cell_selection import select_cells
|
from app.services.raster_cell_selection import select_cells # noqa: E402 - optional rasterio gate precedes service import
|
||||||
|
|
||||||
|
|
||||||
# 100 m cells, origin at the top-left corner of a 3x3 grid.
|
# 100 m cells, origin at the top-left corner of a 3x3 grid.
|
||||||
|
|||||||
@@ -1,14 +1,16 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from types import ModuleType, SimpleNamespace
|
from types import SimpleNamespace
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import importlib
|
import importlib
|
||||||
|
from hashlib import sha256
|
||||||
|
|
||||||
from geoalchemy2.shape import from_shape
|
from geoalchemy2.shape import from_shape
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.models import Area, Dataset, DatasetVersion
|
from app.models import Area, Dataset, DatasetVersion
|
||||||
from app.services.raster_operations_service import RasterOperationsService
|
from app.services.raster_operations_service import RasterOperationsService
|
||||||
|
from app.services.storage_service import StorageService
|
||||||
from app.api.routes.datasets import _run_job_sync
|
from app.api.routes.datasets import _run_job_sync
|
||||||
from shapely.geometry import box
|
from shapely.geometry import box
|
||||||
import pytest
|
import pytest
|
||||||
@@ -688,6 +690,104 @@ def test_raster_tile_returns_manifest_payload(monkeypatch, tmp_path) -> None:
|
|||||||
assert payload["manifest"]["tile_server"] is None
|
assert payload["manifest"]["tile_server"] is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("dimension", "expected"),
|
||||||
|
[
|
||||||
|
(512, [0]),
|
||||||
|
(513, [0, 1]),
|
||||||
|
(960, [0, 448]),
|
||||||
|
(961, [0, 448, 449]),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_raster_tile_offsets_use_full_tiles_and_one_unique_edge_start(dimension, expected) -> None:
|
||||||
|
assert RasterOperationsService._tile_offsets(dimension, tile_size=512, step=448) == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_raster_tile_rejects_limit_before_creating_output(monkeypatch, tmp_path) -> None:
|
||||||
|
project_id = uuid4()
|
||||||
|
dataset_id = uuid4()
|
||||||
|
source = tmp_path / "large-raster.tif"
|
||||||
|
source.write_bytes(b"source")
|
||||||
|
dataset = Dataset(
|
||||||
|
id=dataset_id,
|
||||||
|
project_id=project_id,
|
||||||
|
name="large-raster.tif",
|
||||||
|
dataset_type="raster",
|
||||||
|
source="user_upload",
|
||||||
|
storage_path=str(source),
|
||||||
|
original_filename="large-raster.tif",
|
||||||
|
stored_filename="large-raster.tif",
|
||||||
|
content_type="image/tiff",
|
||||||
|
size_bytes=6,
|
||||||
|
)
|
||||||
|
db = FakeSession([dataset])
|
||||||
|
|
||||||
|
class FakeSource:
|
||||||
|
width = 2048
|
||||||
|
height = 2048
|
||||||
|
count = 1
|
||||||
|
crs = None
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, exc_type, exc, tb):
|
||||||
|
return None
|
||||||
|
|
||||||
|
fake_rasterio = SimpleNamespace(open=lambda _path: FakeSource())
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.raster_operations_service._import_rasterio",
|
||||||
|
lambda: (fake_rasterio, SimpleNamespace()),
|
||||||
|
)
|
||||||
|
tile_root = tmp_path / "tiles-that-must-not-exist"
|
||||||
|
monkeypatch.setattr(
|
||||||
|
StorageService,
|
||||||
|
"raster_tiles_root",
|
||||||
|
staticmethod(lambda *_args: tile_root),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as error:
|
||||||
|
RasterOperationsService.tile(db, dataset_id, tile_size=512, overlap=64, max_tiles=1)
|
||||||
|
|
||||||
|
assert error.value.code == "RASTER_TILE_LIMIT_EXCEEDED"
|
||||||
|
assert error.value.details == {"expected_tile_count": 25, "max_tiles": 1}
|
||||||
|
assert not tile_root.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_raster_tile_rejects_changed_source_bytes_before_creating_output(monkeypatch, tmp_path) -> None:
|
||||||
|
project_id = uuid4()
|
||||||
|
dataset_id = uuid4()
|
||||||
|
source = tmp_path / "changed-raster.tif"
|
||||||
|
source.write_bytes(b"changed")
|
||||||
|
dataset = Dataset(
|
||||||
|
id=dataset_id,
|
||||||
|
project_id=project_id,
|
||||||
|
name="changed-raster.tif",
|
||||||
|
dataset_type="raster",
|
||||||
|
source="user_upload",
|
||||||
|
storage_path=str(source),
|
||||||
|
original_filename="changed-raster.tif",
|
||||||
|
stored_filename="changed-raster.tif",
|
||||||
|
content_type="image/tiff",
|
||||||
|
size_bytes=7,
|
||||||
|
checksum_sha256=sha256(b"original").hexdigest(),
|
||||||
|
data_contract_key="raster.generic",
|
||||||
|
)
|
||||||
|
db = FakeSession([dataset])
|
||||||
|
tile_root = tmp_path / "tiles-that-must-not-exist"
|
||||||
|
monkeypatch.setattr(
|
||||||
|
StorageService,
|
||||||
|
"raster_tiles_root",
|
||||||
|
staticmethod(lambda *_args: tile_root),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as error:
|
||||||
|
RasterOperationsService.tile(db, dataset_id)
|
||||||
|
|
||||||
|
assert error.value.code == "DATASET_STORAGE_CHECKSUM_MISMATCH"
|
||||||
|
assert not tile_root.exists()
|
||||||
|
|
||||||
|
|
||||||
def test_raster_clip_persists_derived_dataset(monkeypatch, tmp_path) -> None:
|
def test_raster_clip_persists_derived_dataset(monkeypatch, tmp_path) -> None:
|
||||||
project_id = uuid4()
|
project_id = uuid4()
|
||||||
dataset_id = uuid4()
|
dataset_id = uuid4()
|
||||||
@@ -1393,4 +1493,3 @@ def test_run_job_sync_serializes_index_job_output_dataset_id(monkeypatch) -> Non
|
|||||||
assert result["job_type"] == "raster.ndvi"
|
assert result["job_type"] == "raster.ndvi"
|
||||||
assert result["output_dataset_id"] == str(output_dataset_id)
|
assert result["output_dataset_id"] == str(output_dataset_id)
|
||||||
assert result["result_json"]["output_dataset_id"] == str(output_dataset_id)
|
assert result["result_json"]["output_dataset_id"] == str(output_dataset_id)
|
||||||
|
|
||||||
|
|||||||
@@ -9,11 +9,12 @@ import pytest
|
|||||||
from geoalchemy2.shape import to_shape
|
from geoalchemy2.shape import to_shape
|
||||||
|
|
||||||
from app.core.config import Settings
|
from app.core.config import Settings
|
||||||
from app.models import Dataset, Project, Segmentation, SourceRegistry, SourceSnapshot
|
from app.models import Dataset, DatasetVersion, Project, Segmentation, SourceRegistry, SourceSnapshot
|
||||||
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
|
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
|
||||||
from app.services.model_registry_service import ModelRegistryService
|
from app.services.model_registry_service import ModelRegistryService
|
||||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||||
from app.services.segmentation_service import SegmentationService
|
from app.services.segmentation_service import SegmentationService
|
||||||
|
from app.services.tile_manifest_service import TileManifestService
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|
||||||
@@ -122,6 +123,8 @@ def _project_and_dataset(dataset_type: str = "raster"):
|
|||||||
source_name="digitaal_vlaanderen_orthophoto",
|
source_name="digitaal_vlaanderen_orthophoto",
|
||||||
storage_path="storage/uploads/ortho.tif",
|
storage_path="storage/uploads/ortho.tif",
|
||||||
checksum_sha256=checksum,
|
checksum_sha256=checksum,
|
||||||
|
crs="EPSG:4326",
|
||||||
|
bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0},
|
||||||
source_registry_id=source_id,
|
source_registry_id=source_id,
|
||||||
source_snapshot_id=snapshot_id,
|
source_snapshot_id=snapshot_id,
|
||||||
data_contract_key="geointel.raster.geotiff",
|
data_contract_key="geointel.raster.geotiff",
|
||||||
@@ -134,6 +137,9 @@ def _project_and_dataset(dataset_type: str = "raster"):
|
|||||||
)
|
)
|
||||||
dataset.source_registry = source
|
dataset.source_registry = source
|
||||||
dataset.source_snapshot = snapshot
|
dataset.source_snapshot = snapshot
|
||||||
|
dataset.versions.append(
|
||||||
|
DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1, checksum_sha256=checksum)
|
||||||
|
)
|
||||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||||
return db, project_id, dataset_id
|
return db, project_id, dataset_id
|
||||||
|
|
||||||
@@ -249,28 +255,36 @@ def _write_configured_model_sidecars(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _manifest(tmp_path: Path, tile_count: int = 1) -> Path:
|
def _manifest(
|
||||||
|
tmp_path: Path,
|
||||||
|
tile_count: int = 1,
|
||||||
|
*,
|
||||||
|
db: FakeSession | None = None,
|
||||||
|
dataset: Dataset | None = None,
|
||||||
|
) -> Path:
|
||||||
tiles = []
|
tiles = []
|
||||||
for index in range(tile_count):
|
for index in range(tile_count):
|
||||||
tile_path = tmp_path / f"tile_{index:04d}.tif"
|
tile_path = tmp_path / f"tile_{index:04d}.tif"
|
||||||
tile_path.write_bytes(b"fixture")
|
tile_path.write_bytes(b"fixture")
|
||||||
tiles.append(
|
tile = {
|
||||||
{
|
|
||||||
"path": str(tile_path),
|
"path": str(tile_path),
|
||||||
"pixel_window": [0, 0, 100, 100],
|
"pixel_window": [0, 0, 100, 100],
|
||||||
"bounds": [4.0, 51.0, 5.0, 52.0],
|
"bounds": [4.0, 51.0, 5.0, 52.0],
|
||||||
"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01],
|
"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01],
|
||||||
"crs": "EPSG:4326",
|
"crs": "EPSG:4326",
|
||||||
"index": index,
|
"index": index,
|
||||||
|
**TileManifestService.tile_integrity(tile_path),
|
||||||
}
|
}
|
||||||
)
|
tiles.append(tile)
|
||||||
|
binding = TileManifestService.dataset_binding(db or FakeSession(), dataset) if dataset is not None else {}
|
||||||
manifest_path = tmp_path / "manifest.json"
|
manifest_path = tmp_path / "manifest.json"
|
||||||
manifest_path.write_text(
|
manifest_path.write_text(
|
||||||
json.dumps(
|
json.dumps(
|
||||||
{
|
{
|
||||||
|
**binding,
|
||||||
"tile_set_id": "tiles-fixture",
|
"tile_set_id": "tiles-fixture",
|
||||||
"source_dataset_id": str(uuid4()),
|
"source_dataset_id": binding.get("source_dataset_id", str(uuid4())),
|
||||||
"source_raster_id": str(uuid4()),
|
"source_raster_id": binding.get("source_raster_id", str(uuid4())),
|
||||||
"crs": "EPSG:4326",
|
"crs": "EPSG:4326",
|
||||||
"bounds": [4.0, 51.0, 5.0, 52.0],
|
"bounds": [4.0, 51.0, 5.0, 52.0],
|
||||||
"tile_size": 100,
|
"tile_size": 100,
|
||||||
@@ -424,7 +438,9 @@ def test_configured_segmentation_rejects_unbound_model_snapshot_before_adapter_l
|
|||||||
dataset_id=dataset_id,
|
dataset_id=dataset_id,
|
||||||
model_id="yolo-seg-configured",
|
model_id="yolo-seg-configured",
|
||||||
confidence_threshold=0.5,
|
confidence_threshold=0.5,
|
||||||
tile_manifest_path=str(_manifest(tmp_path)),
|
tile_manifest_path=str(
|
||||||
|
_manifest(tmp_path, db=db, dataset=db.get(Dataset, dataset_id))
|
||||||
|
),
|
||||||
settings=settings,
|
settings=settings,
|
||||||
yolo_seg_adapter_class=NeverLoadSegAdapter,
|
yolo_seg_adapter_class=NeverLoadSegAdapter,
|
||||||
sam_adapter_class=ClassAgnosticSamAdapter,
|
sam_adapter_class=ClassAgnosticSamAdapter,
|
||||||
@@ -440,7 +456,7 @@ def test_configured_yolo_seg_run_persists_georeferenced_masks(tmp_path: Path) ->
|
|||||||
db, project_id, dataset_id = _project_and_dataset()
|
db, project_id, dataset_id = _project_and_dataset()
|
||||||
settings = _settings(tmp_path)
|
settings = _settings(tmp_path)
|
||||||
_write_configured_model_sidecars(tmp_path, settings, include_sam=False, db=db)
|
_write_configured_model_sidecars(tmp_path, settings, include_sam=False, db=db)
|
||||||
manifest_path = _manifest(tmp_path)
|
manifest_path = _manifest(tmp_path, db=db, dataset=db.get(Dataset, dataset_id))
|
||||||
|
|
||||||
response = SegmentationService.run_segmentation(
|
response = SegmentationService.run_segmentation(
|
||||||
db=db,
|
db=db,
|
||||||
@@ -479,7 +495,7 @@ def test_configured_sam_run_is_class_agnostic(tmp_path: Path) -> None:
|
|||||||
db, project_id, dataset_id = _project_and_dataset()
|
db, project_id, dataset_id = _project_and_dataset()
|
||||||
settings = _settings(tmp_path)
|
settings = _settings(tmp_path)
|
||||||
_write_configured_model_sidecars(tmp_path, settings, include_yolo=False, db=db)
|
_write_configured_model_sidecars(tmp_path, settings, include_yolo=False, db=db)
|
||||||
manifest_path = _manifest(tmp_path)
|
manifest_path = _manifest(tmp_path, db=db, dataset=db.get(Dataset, dataset_id))
|
||||||
|
|
||||||
response = SegmentationService.run_segmentation(
|
response = SegmentationService.run_segmentation(
|
||||||
db=db,
|
db=db,
|
||||||
|
|||||||
@@ -15,14 +15,18 @@ import pytest
|
|||||||
np = pytest.importorskip("numpy")
|
np = pytest.importorskip("numpy")
|
||||||
rasterio = pytest.importorskip("rasterio")
|
rasterio = pytest.importorskip("rasterio")
|
||||||
|
|
||||||
from pyproj import Transformer
|
from pyproj import Transformer # noqa: E402 - optional rasterio gate precedes geospatial imports
|
||||||
from rasterio.transform import from_origin
|
from rasterio.transform import from_origin # noqa: E402 - optional rasterio gate precedes geospatial imports
|
||||||
|
|
||||||
from app.core.config import Settings
|
from app.core.config import Settings # noqa: E402 - optional rasterio gate precedes app imports
|
||||||
from app.models import Dataset
|
from app.models import Dataset # noqa: E402 - optional rasterio gate precedes app imports
|
||||||
from app.schemas.flood_hazard import FloodHazardSelectionRequest
|
from app.schemas.flood_hazard import FloodHazardSelectionRequest # noqa: E402 - optional rasterio gate precedes app imports
|
||||||
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
|
from app.services.flood_hazard_acquisition_service import ( # noqa: E402 - optional rasterio gate precedes app imports
|
||||||
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
|
FloodHazardAcquisitionService,
|
||||||
|
)
|
||||||
|
from app.services.flood_hazard_analysis_service import ( # noqa: E402 - optional rasterio gate precedes app imports
|
||||||
|
FloodHazardAnalysisService,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
TO_4326 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
|
TO_4326 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
|
||||||
|
|||||||
@@ -14,7 +14,10 @@ def test_raster_tile_manifest_can_handoff_to_segmentation_lab() -> None:
|
|||||||
|
|
||||||
assert "segmentationTileManifestPath" in hook
|
assert "segmentationTileManifestPath" in hook
|
||||||
assert "setSegmentationTileManifestPath" in hook
|
assert "setSegmentationTileManifestPath" in hook
|
||||||
assert "tile_manifest_path: segmentationTileManifestPath.trim() || null" in hook
|
assert "let manifestPath = segmentationTileManifestPath.trim()" in hook
|
||||||
|
assert "datasetsApi.rasterInspect(projectId, datasetId)" in hook
|
||||||
|
assert "datasetsApi.rasterTile(projectId, datasetId" in hook
|
||||||
|
assert "tile_manifest_path: manifestPath" in hook
|
||||||
assert "segmentationTileManifestPath={segmentationTileManifestPath}" in app
|
assert "segmentationTileManifestPath={segmentationTileManifestPath}" in app
|
||||||
assert "onSetTileManifestPath={setSegmentationTileManifestPath}" in app
|
assert "onSetTileManifestPath={setSegmentationTileManifestPath}" in app
|
||||||
assert "onUseTileManifestForSegmentation: useRasterTileManifestForSegmentation" in app
|
assert "onUseTileManifestForSegmentation: useRasterTileManifestForSegmentation" in app
|
||||||
@@ -24,4 +27,3 @@ def test_raster_tile_manifest_can_handoff_to_segmentation_lab() -> None:
|
|||||||
assert "Gebruik voor segmentatie" in raster_controls
|
assert "Gebruik voor segmentatie" in raster_controls
|
||||||
assert "disabled={!latestRasterTileManifestPath}" in raster_controls
|
assert "disabled={!latestRasterTileManifestPath}" in raster_controls
|
||||||
assert "Beeldtegelmanifest" in segmentation_lab
|
assert "Beeldtegelmanifest" in segmentation_lab
|
||||||
assert "Beeldtegelmanifest" in segmentation_lab
|
|
||||||
|
|||||||
@@ -1,6 +1,5 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import re
|
|
||||||
import uuid
|
import uuid
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
@@ -11,7 +10,7 @@ from shapely.geometry import Polygon, box
|
|||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.models import Dataset, VectorFeature
|
from app.models import Dataset, VectorFeature
|
||||||
from app.services.vector_feature_service import VectorFeatureService
|
from app.services.vector_feature_service import VectorFeatureService
|
||||||
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace, read_feature
|
from tests.frontend_contract import assert_calls, assert_wired, read_map_workspace, read_feature
|
||||||
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|||||||
@@ -12,8 +12,12 @@ class FakeUploadFile:
|
|||||||
filename = "real-orthophoto.tif"
|
filename = "real-orthophoto.tif"
|
||||||
content_type = "image/tiff"
|
content_type = "image/tiff"
|
||||||
|
|
||||||
async def read(self) -> bytes:
|
def __init__(self) -> None:
|
||||||
return b"fake-raster"
|
self._content = b"fake-raster"
|
||||||
|
|
||||||
|
async def read(self, size: int) -> bytes:
|
||||||
|
chunk, self._content = self._content[:size], self._content[size:]
|
||||||
|
return chunk
|
||||||
|
|
||||||
|
|
||||||
class FakeSession:
|
class FakeSession:
|
||||||
@@ -40,16 +44,19 @@ def test_raster_upload_maps_metadata_bounds_resolution_and_bands(monkeypatch) ->
|
|||||||
project_id = uuid4()
|
project_id = uuid4()
|
||||||
db = FakeSession(project_id)
|
db = FakeSession(project_id)
|
||||||
|
|
||||||
monkeypatch.setattr(
|
async def persist_upload_file(**_kwargs):
|
||||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
return {
|
||||||
lambda **_: {
|
|
||||||
"storage_path": "/tmp/real-orthophoto.tif",
|
"storage_path": "/tmp/real-orthophoto.tif",
|
||||||
"original_filename": "real-orthophoto.tif",
|
"original_filename": "real-orthophoto.tif",
|
||||||
"stored_filename": "real-orthophoto.tif",
|
"stored_filename": "real-orthophoto.tif",
|
||||||
"content_type": "image/tiff",
|
"content_type": "image/tiff",
|
||||||
"size_bytes": 11,
|
"size_bytes": 11,
|
||||||
"checksum_sha256": "checksum",
|
"checksum_sha256": "checksum",
|
||||||
},
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.dataset_service.StorageService.persist_upload_file",
|
||||||
|
persist_upload_file,
|
||||||
)
|
)
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.services.dataset_service.extract_raster_metadata",
|
"app.services.dataset_service.extract_raster_metadata",
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import re
|
|
||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace, read_feature
|
from tests.frontend_contract import assert_calls, assert_wired, read_map_workspace, read_feature
|
||||||
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace, read_feature
|
from tests.frontend_contract import assert_calls, read_map_workspace, read_feature
|
||||||
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|||||||
@@ -251,7 +251,8 @@ def test_end_user_dataset_sources_are_human_readable() -> None:
|
|||||||
assert "statbel: 'Statbel'" in display
|
assert "statbel: 'Statbel'" in display
|
||||||
assert "getDatasetSourceDisplayName(activeThemeDataset)" in workspace
|
assert "getDatasetSourceDisplayName(activeThemeDataset)" in workspace
|
||||||
assert "getDatasetSourceDisplayName(resultDataset)" in workspace
|
assert "getDatasetSourceDisplayName(resultDataset)" in workspace
|
||||||
assert "Zoek optioneel een gemeente" in workspace
|
assert 'aria-label="Optioneel een gemeente zoeken"' in workspace
|
||||||
|
assert 'placeholder="Gemeentenaam of NIS-code"' in workspace
|
||||||
assert "latestDatasetBySeries" in catalog
|
assert "latestDatasetBySeries" in catalog
|
||||||
assert "Historische meetmomenten" in catalog
|
assert "Historische meetmomenten" in catalog
|
||||||
assert "getDatasetSourceDisplayName(dataset)" in catalog
|
assert "getDatasetSourceDisplayName(dataset)" in catalog
|
||||||
|
|||||||
@@ -20,6 +20,7 @@ from app.db.session import get_db
|
|||||||
from app.main import app
|
from app.main import app
|
||||||
from app.models import Area, Dataset, DatasetVersion, Job, Project, SourceRegistry, SourceSnapshot
|
from app.models import Area, Dataset, DatasetVersion, Job, Project, SourceRegistry, SourceSnapshot
|
||||||
from app.schemas.orthophoto import OrthophotoAcquireRequest
|
from app.schemas.orthophoto import OrthophotoAcquireRequest
|
||||||
|
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||||
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
|
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
|
||||||
from tests.frontend_contract import read_feature
|
from tests.frontend_contract import read_feature
|
||||||
|
|
||||||
@@ -262,6 +263,9 @@ def test_regional_orthophoto_products_bind_provider_and_governed_scope(
|
|||||||
assert snapshot.checksum_sha256 == dataset.checksum_sha256
|
assert snapshot.checksum_sha256 == dataset.checksum_sha256
|
||||||
assert snapshot.ingest_status == "ingested"
|
assert snapshot.ingest_status == "ingested"
|
||||||
assert snapshot.freshness_status == "current"
|
assert snapshot.freshness_status == "current"
|
||||||
|
dataset.source_registry = source
|
||||||
|
dataset.source_snapshot = snapshot
|
||||||
|
assert DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference").eligible is True
|
||||||
|
|
||||||
prepared = OrthophotoAcquisitionService._prepared_request(_selection_payload(product_key="1971"), settings)
|
prepared = OrthophotoAcquisitionService._prepared_request(_selection_payload(product_key="1971"), settings)
|
||||||
assert prepared["params"]["LAYERS"] == "OKZPAN71VL"
|
assert prepared["params"]["LAYERS"] == "OKZPAN71VL"
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace
|
from tests.frontend_contract import assert_wired, read_map_workspace
|
||||||
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import importlib.util
|
import importlib.util
|
||||||
import json
|
|
||||||
import sys
|
import sys
|
||||||
import zipfile
|
import zipfile
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|||||||
@@ -7,7 +7,6 @@ from uuid import uuid4
|
|||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
import pytest
|
import pytest
|
||||||
import rasterio
|
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from pyproj import Transformer
|
from pyproj import Transformer
|
||||||
from rasterio.io import MemoryFile
|
from rasterio.io import MemoryFile
|
||||||
|
|||||||
@@ -252,7 +252,6 @@ def test_refresh_api_and_frontend_remain_explicit_only() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_map_theme_ranking_prefers_newer_observation_over_feature_count() -> None:
|
def test_map_theme_ranking_prefers_newer_observation_over_feature_count() -> None:
|
||||||
root = Path(__file__).resolve().parents[2]
|
|
||||||
workspace = read_map_workspace()
|
workspace = read_map_workspace()
|
||||||
observed_sort = workspace.index("const observedAtDifference")
|
observed_sort = workspace.index("const observedAtDifference")
|
||||||
feature_tiebreaker = workspace.index("right.feature_count", observed_sort)
|
feature_tiebreaker = workspace.index("right.feature_count", observed_sort)
|
||||||
@@ -262,7 +261,6 @@ def test_map_theme_ranking_prefers_newer_observation_over_feature_count() -> Non
|
|||||||
|
|
||||||
|
|
||||||
def test_map_workspace_restores_theme_from_selected_dataset() -> None:
|
def test_map_workspace_restores_theme_from_selected_dataset() -> None:
|
||||||
root = Path(__file__).resolve().parents[2]
|
|
||||||
workspace = read_map_workspace()
|
workspace = read_map_workspace()
|
||||||
assert "function themeIdForDataset(" in workspace
|
assert "function themeIdForDataset(" in workspace
|
||||||
assert "useState<DataThemeId>(() =>" in workspace
|
assert "useState<DataThemeId>(() =>" in workspace
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
import json
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import ssl
|
import ssl
|
||||||
import sys
|
import sys
|
||||||
@@ -31,7 +30,7 @@ if str(SCRIPTS) not in sys.path:
|
|||||||
sys.path.insert(0, str(SCRIPTS))
|
sys.path.insert(0, str(SCRIPTS))
|
||||||
|
|
||||||
import provision_flanders_geographic_scope as flanders_scope # noqa: E402
|
import provision_flanders_geographic_scope as flanders_scope # noqa: E402
|
||||||
from tests.frontend_contract import read_map_workspace, read_feature
|
from tests.frontend_contract import read_map_workspace, read_feature # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
class BinaryResponse:
|
class BinaryResponse:
|
||||||
@@ -136,6 +135,32 @@ def test_mdk_probe_parses_capabilities_without_enabling_acquisition() -> None:
|
|||||||
assert seen["timeout"] == 20
|
assert seen["timeout"] == 20
|
||||||
|
|
||||||
|
|
||||||
|
def test_mdk_probe_default_opener_uses_guarded_strict_tls_path(monkeypatch) -> None:
|
||||||
|
seen = {}
|
||||||
|
|
||||||
|
def guarded_factory(expected_url):
|
||||||
|
seen["expected_url"] = expected_url
|
||||||
|
|
||||||
|
def open_request(request, timeout):
|
||||||
|
seen["request_url"] = request.full_url
|
||||||
|
seen["timeout"] = timeout
|
||||||
|
return BinaryResponse(capabilities_xml())
|
||||||
|
|
||||||
|
return open_request
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.mdk_bathymetry_probe_service.guarded_opener",
|
||||||
|
guarded_factory,
|
||||||
|
)
|
||||||
|
|
||||||
|
result = MdkBathymetryProbeService.probe(settings=Settings(_env_file=None))
|
||||||
|
|
||||||
|
assert result["status"] == "reachable"
|
||||||
|
assert seen["expected_url"] == seen["request_url"]
|
||||||
|
assert seen["expected_url"].startswith("https://")
|
||||||
|
assert seen["timeout"] == 20
|
||||||
|
|
||||||
|
|
||||||
def test_mdk_probe_reports_tls_failure_and_never_uses_insecure_fallback() -> None:
|
def test_mdk_probe_reports_tls_failure_and_never_uses_insecure_fallback() -> None:
|
||||||
calls = 0
|
calls = 0
|
||||||
|
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace, read_feature
|
from tests.frontend_contract import assert_wired, read_map_workspace, read_feature
|
||||||
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|||||||
@@ -19,7 +19,7 @@ from app.models import Area, Dataset, Job, Project
|
|||||||
from app.schemas.grb import GrbAcquireRequest
|
from app.schemas.grb import GrbAcquireRequest
|
||||||
from app.services.dataset_service import DatasetService
|
from app.services.dataset_service import DatasetService
|
||||||
from app.services.grb_acquisition_service import GrbAcquisitionService
|
from app.services.grb_acquisition_service import GrbAcquisitionService
|
||||||
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace, read_feature
|
from tests.frontend_contract import assert_wired, read_map_workspace, read_feature
|
||||||
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import importlib.util
|
import importlib.util
|
||||||
import io
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import sys
|
import sys
|
||||||
import zipfile
|
import zipfile
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
@@ -187,23 +188,31 @@ def test_dataset_upload_persists_vector_features(monkeypatch, tmp_path) -> None:
|
|||||||
filename = "reference.geojson"
|
filename = "reference.geojson"
|
||||||
content_type = "application/geo+json"
|
content_type = "application/geo+json"
|
||||||
|
|
||||||
async def read(self) -> bytes:
|
def __init__(self) -> None:
|
||||||
import json
|
import json
|
||||||
|
|
||||||
return json.dumps(payload).encode("utf-8")
|
self._content = json.dumps(payload).encode("utf-8")
|
||||||
|
|
||||||
|
async def read(self, size: int) -> bytes:
|
||||||
|
chunk, self._content = self._content[:size], self._content[size:]
|
||||||
|
return chunk
|
||||||
|
|
||||||
storage_path = tmp_path / "reference.geojson"
|
storage_path = tmp_path / "reference.geojson"
|
||||||
storage_path.write_text("{}", encoding="utf-8")
|
storage_path.write_text(json.dumps(payload), encoding="utf-8")
|
||||||
monkeypatch.setattr(
|
|
||||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
async def persist_upload_file(**_kwargs):
|
||||||
lambda **_kwargs: {
|
return {
|
||||||
"storage_path": str(storage_path),
|
"storage_path": str(storage_path),
|
||||||
"original_filename": "reference.geojson",
|
"original_filename": "reference.geojson",
|
||||||
"stored_filename": "reference.geojson",
|
"stored_filename": "reference.geojson",
|
||||||
"content_type": "application/geo+json",
|
"content_type": "application/geo+json",
|
||||||
"size_bytes": 2,
|
"size_bytes": storage_path.stat().st_size,
|
||||||
"checksum_sha256": "0" * 64,
|
"checksum_sha256": "0" * 64,
|
||||||
},
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.dataset_service.StorageService.persist_upload_file",
|
||||||
|
persist_upload_file,
|
||||||
)
|
)
|
||||||
|
|
||||||
result = asyncio.run(
|
result = asyncio.run(
|
||||||
@@ -236,19 +245,28 @@ def test_dataset_upload_rolls_back_dataset_and_file_when_vector_indexing_fails(m
|
|||||||
filename = "invalid.geojson"
|
filename = "invalid.geojson"
|
||||||
content_type = "application/geo+json"
|
content_type = "application/geo+json"
|
||||||
|
|
||||||
async def read(self) -> bytes:
|
def __init__(self) -> None:
|
||||||
return b'{"type":"FeatureCollection","features":[]}'
|
self._content = b'{"type":"FeatureCollection","features":[]}'
|
||||||
|
|
||||||
monkeypatch.setattr(
|
async def read(self, size: int) -> bytes:
|
||||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
chunk, self._content = self._content[:size], self._content[size:]
|
||||||
lambda **_kwargs: {
|
return chunk
|
||||||
|
|
||||||
|
storage_path.write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8")
|
||||||
|
|
||||||
|
async def persist_upload_file(**_kwargs):
|
||||||
|
return {
|
||||||
"storage_path": str(storage_path),
|
"storage_path": str(storage_path),
|
||||||
"original_filename": "invalid.geojson",
|
"original_filename": "invalid.geojson",
|
||||||
"stored_filename": "invalid.geojson",
|
"stored_filename": "invalid.geojson",
|
||||||
"content_type": "application/geo+json",
|
"content_type": "application/geo+json",
|
||||||
"size_bytes": 2,
|
"size_bytes": storage_path.stat().st_size,
|
||||||
"checksum_sha256": "0" * 64,
|
"checksum_sha256": "0" * 64,
|
||||||
},
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.dataset_service.StorageService.persist_upload_file",
|
||||||
|
persist_upload_file,
|
||||||
)
|
)
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
VectorFeatureService,
|
VectorFeatureService,
|
||||||
|
|||||||
@@ -13,12 +13,23 @@ from shapely.geometry import box, mapping
|
|||||||
|
|
||||||
from app.core.config import Settings
|
from app.core.config import Settings
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project, SourceRegistry, SourceSnapshot
|
from app.models import (
|
||||||
|
AnalysisRun,
|
||||||
|
Area,
|
||||||
|
Dataset,
|
||||||
|
DatasetVersion,
|
||||||
|
Detection,
|
||||||
|
Job,
|
||||||
|
Project,
|
||||||
|
SourceRegistry,
|
||||||
|
SourceSnapshot,
|
||||||
|
)
|
||||||
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
|
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
|
||||||
from app.services.detection_service import DetectionService
|
from app.services.detection_service import DetectionService
|
||||||
from app.services.model_registry_service import ModelRegistryService
|
from app.services.model_registry_service import ModelRegistryService
|
||||||
from app.services.model_validation_scope_service import ModelValidationScopeService
|
from app.services.model_validation_scope_service import ModelValidationScopeService
|
||||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||||
|
from app.services.tile_manifest_service import TileManifestService
|
||||||
from app.services.yolo_adapter import YoloDetectionAdapter
|
from app.services.yolo_adapter import YoloDetectionAdapter
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
@@ -189,6 +200,8 @@ def _project_and_dataset(dataset_type: str = "raster"):
|
|||||||
source_name="test-derived-raster",
|
source_name="test-derived-raster",
|
||||||
storage_path="storage/uploads/source.tif",
|
storage_path="storage/uploads/source.tif",
|
||||||
checksum_sha256=checksum,
|
checksum_sha256=checksum,
|
||||||
|
crs="EPSG:4326",
|
||||||
|
bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0},
|
||||||
source_registry_id=source_registry_id,
|
source_registry_id=source_registry_id,
|
||||||
source_snapshot_id=source_snapshot_id,
|
source_snapshot_id=source_snapshot_id,
|
||||||
data_contract_key="geointel.raster.geotiff",
|
data_contract_key="geointel.raster.geotiff",
|
||||||
@@ -201,6 +214,9 @@ def _project_and_dataset(dataset_type: str = "raster"):
|
|||||||
)
|
)
|
||||||
dataset.source_registry = source_registry
|
dataset.source_registry = source_registry
|
||||||
dataset.source_snapshot = source_snapshot
|
dataset.source_snapshot = source_snapshot
|
||||||
|
dataset.versions.append(
|
||||||
|
DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1, checksum_sha256=checksum)
|
||||||
|
)
|
||||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||||
return db, project_id, dataset_id
|
return db, project_id, dataset_id
|
||||||
|
|
||||||
@@ -309,28 +325,34 @@ def _write_model_sidecar(
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def _manifest(tmp_path: Path, tile_count: int = 1) -> Path:
|
def _manifest(tmp_path: Path, tile_count: int = 1, dataset: Dataset | None = None) -> Path:
|
||||||
tiles = []
|
tiles = []
|
||||||
for index in range(tile_count):
|
for index in range(tile_count):
|
||||||
tile_path = tmp_path / f"tile_{index:04d}.tif"
|
tile_path = tmp_path / f"tile_{index:04d}.tif"
|
||||||
tile_path.write_bytes(b"fixture")
|
tile_path.write_bytes(b"fixture")
|
||||||
tiles.append(
|
tile = {
|
||||||
{
|
|
||||||
"path": str(tile_path),
|
"path": str(tile_path),
|
||||||
"pixel_window": [0, 0, 100, 100],
|
"pixel_window": [0, 0, 100, 100],
|
||||||
"bounds": [4.0, 51.0, 5.0, 52.0],
|
"bounds": [4.0, 51.0, 5.0, 52.0],
|
||||||
"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01],
|
"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01],
|
||||||
"crs": "EPSG:4326",
|
"crs": "EPSG:4326",
|
||||||
"index": index,
|
"index": index,
|
||||||
|
**TileManifestService.tile_integrity(tile_path),
|
||||||
}
|
}
|
||||||
)
|
tiles.append(tile)
|
||||||
|
binding = (
|
||||||
|
TileManifestService.dataset_binding(SimpleNamespace(get=lambda *_args: None), dataset)
|
||||||
|
if dataset is not None
|
||||||
|
else {}
|
||||||
|
)
|
||||||
manifest_path = tmp_path / "manifest.json"
|
manifest_path = tmp_path / "manifest.json"
|
||||||
manifest_path.write_text(
|
manifest_path.write_text(
|
||||||
json.dumps(
|
json.dumps(
|
||||||
{
|
{
|
||||||
|
**binding,
|
||||||
"tile_set_id": "tiles-fixture",
|
"tile_set_id": "tiles-fixture",
|
||||||
"source_dataset_id": str(uuid4()),
|
"source_dataset_id": binding.get("source_dataset_id", str(uuid4())),
|
||||||
"source_raster_id": str(uuid4()),
|
"source_raster_id": binding.get("source_raster_id", str(uuid4())),
|
||||||
"crs": "EPSG:4326",
|
"crs": "EPSG:4326",
|
||||||
"bounds": [4.0, 51.0, 5.0, 52.0],
|
"bounds": [4.0, 51.0, 5.0, 52.0],
|
||||||
"tile_size": 100,
|
"tile_size": 100,
|
||||||
@@ -514,7 +536,7 @@ def test_yolo_run_fails_closed_before_adapter_load_without_sidecar(tmp_path: Pat
|
|||||||
dataset_id=dataset_id,
|
dataset_id=dataset_id,
|
||||||
model_id="yolo-configured",
|
model_id="yolo-configured",
|
||||||
confidence_threshold=0.5,
|
confidence_threshold=0.5,
|
||||||
tile_manifest_path=str(_manifest(tmp_path)),
|
tile_manifest_path=str(_manifest(tmp_path, dataset=db.get(Dataset, dataset_id))),
|
||||||
settings=settings,
|
settings=settings,
|
||||||
yolo_adapter_class=AvailableAdapter,
|
yolo_adapter_class=AvailableAdapter,
|
||||||
)
|
)
|
||||||
@@ -530,7 +552,7 @@ def test_yolo_run_rejects_manifest_over_tile_limit(tmp_path: Path) -> None:
|
|||||||
model_path.write_bytes(b"local weights")
|
model_path.write_bytes(b"local weights")
|
||||||
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_max_tiles=1)
|
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_max_tiles=1)
|
||||||
_write_model_sidecar(model_path, settings, db=db)
|
_write_model_sidecar(model_path, settings, db=db)
|
||||||
manifest_path = _manifest(tmp_path, tile_count=2)
|
manifest_path = _manifest(tmp_path, tile_count=2, dataset=db.get(Dataset, dataset_id))
|
||||||
|
|
||||||
result = DetectionService.run_detection(
|
result = DetectionService.run_detection(
|
||||||
db=db,
|
db=db,
|
||||||
@@ -609,7 +631,7 @@ def test_yolo_run_rejects_unbound_model_snapshot_before_adapter_load(tmp_path: P
|
|||||||
dataset_id=dataset_id,
|
dataset_id=dataset_id,
|
||||||
model_id="yolo-configured",
|
model_id="yolo-configured",
|
||||||
confidence_threshold=0.5,
|
confidence_threshold=0.5,
|
||||||
tile_manifest_path=str(_manifest(tmp_path)),
|
tile_manifest_path=str(_manifest(tmp_path, dataset=db.get(Dataset, dataset_id))),
|
||||||
settings=settings,
|
settings=settings,
|
||||||
yolo_adapter_class=NeverLoadUnboundModelAdapter,
|
yolo_adapter_class=NeverLoadUnboundModelAdapter,
|
||||||
)
|
)
|
||||||
@@ -638,7 +660,7 @@ def test_yolo_run_persists_mocked_georeferenced_detections(tmp_path: Path) -> No
|
|||||||
model_path.write_bytes(b"local weights")
|
model_path.write_bytes(b"local weights")
|
||||||
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_model_version="local-test")
|
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_model_version="local-test")
|
||||||
_write_model_sidecar(model_path, settings, db=db)
|
_write_model_sidecar(model_path, settings, db=db)
|
||||||
manifest_path = _manifest(tmp_path, tile_count=1)
|
manifest_path = _manifest(tmp_path, tile_count=1, dataset=db.get(Dataset, dataset_id))
|
||||||
|
|
||||||
result = DetectionService.run_detection(
|
result = DetectionService.run_detection(
|
||||||
db=db,
|
db=db,
|
||||||
@@ -678,7 +700,7 @@ def test_yolo_class_filter_is_case_insensitive_and_persists_canonical_class(tmp_
|
|||||||
model_path.write_bytes(b"local weights")
|
model_path.write_bytes(b"local weights")
|
||||||
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
||||||
_write_model_sidecar(model_path, settings, db=db)
|
_write_model_sidecar(model_path, settings, db=db)
|
||||||
manifest_path = _manifest(tmp_path, tile_count=1)
|
manifest_path = _manifest(tmp_path, tile_count=1, dataset=db.get(Dataset, dataset_id))
|
||||||
|
|
||||||
result = DetectionService.run_detection(
|
result = DetectionService.run_detection(
|
||||||
db=db,
|
db=db,
|
||||||
@@ -706,7 +728,7 @@ def test_yolo_run_suppresses_cross_tile_duplicate_detections(tmp_path: Path) ->
|
|||||||
model_path.write_bytes(b"local weights")
|
model_path.write_bytes(b"local weights")
|
||||||
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_duplicate_iou_threshold=0.5)
|
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_duplicate_iou_threshold=0.5)
|
||||||
_write_model_sidecar(model_path, settings, db=db)
|
_write_model_sidecar(model_path, settings, db=db)
|
||||||
manifest_path = _manifest(tmp_path, tile_count=2)
|
manifest_path = _manifest(tmp_path, tile_count=2, dataset=db.get(Dataset, dataset_id))
|
||||||
|
|
||||||
result = DetectionService.run_detection(
|
result = DetectionService.run_detection(
|
||||||
db=db,
|
db=db,
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_feature
|
from tests.frontend_contract import assert_mentions, read_feature
|
||||||
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
|||||||
@@ -1,6 +1,12 @@
|
|||||||
|
import asyncio
|
||||||
|
from hashlib import sha256
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.services.dataset_service import DatasetService
|
||||||
from app.services.storage_service import StorageService
|
from app.services.storage_service import StorageService
|
||||||
|
|
||||||
|
|
||||||
@@ -26,3 +32,111 @@ def test_persist_dataset_file_records_metadata(monkeypatch, tmp_path) -> None:
|
|||||||
assert len(metadata["checksum_sha256"]) == 64
|
assert len(metadata["checksum_sha256"]) == 64
|
||||||
assert Path(metadata["storage_path"]).exists()
|
assert Path(metadata["storage_path"]).exists()
|
||||||
assert str(Path(tmp_path, "uploads", "project-123", "vector", "dataset-456")) in metadata["storage_path"]
|
assert str(Path(tmp_path, "uploads", "project-123", "vector", "dataset-456")) in metadata["storage_path"]
|
||||||
|
|
||||||
|
|
||||||
|
class _ChunkedUpload:
|
||||||
|
def __init__(self, content: bytes) -> None:
|
||||||
|
self.content = content
|
||||||
|
self.requested_sizes: list[int] = []
|
||||||
|
|
||||||
|
async def read(self, size: int) -> bytes:
|
||||||
|
self.requested_sizes.append(size)
|
||||||
|
chunk, self.content = self.content[:size], self.content[size:]
|
||||||
|
return chunk
|
||||||
|
|
||||||
|
|
||||||
|
def test_persist_upload_file_streams_bounded_chunks(monkeypatch, tmp_path) -> None:
|
||||||
|
monkeypatch.setattr(StorageService, "_base_dir", staticmethod(lambda: tmp_path))
|
||||||
|
upload = _ChunkedUpload(b"abcdefghijk")
|
||||||
|
|
||||||
|
metadata = asyncio.run(
|
||||||
|
StorageService.persist_upload_file(
|
||||||
|
project_id="project",
|
||||||
|
dataset_id="dataset",
|
||||||
|
dataset_type="raster",
|
||||||
|
original_filename="source.tif",
|
||||||
|
upload=upload,
|
||||||
|
content_type="image/tiff",
|
||||||
|
max_bytes=32,
|
||||||
|
chunk_size=4,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
stored = Path(metadata["storage_path"])
|
||||||
|
assert upload.requested_sizes == [4, 4, 4, 4]
|
||||||
|
assert stored.read_bytes() == b"abcdefghijk"
|
||||||
|
assert metadata["size_bytes"] == 11
|
||||||
|
assert metadata["checksum_sha256"] == sha256(b"abcdefghijk").hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def test_persist_upload_file_rejects_oversize_and_removes_partial_file(monkeypatch, tmp_path) -> None:
|
||||||
|
monkeypatch.setattr(StorageService, "_base_dir", staticmethod(lambda: tmp_path))
|
||||||
|
upload = _ChunkedUpload(b"0123456789")
|
||||||
|
expected_path = Path(
|
||||||
|
StorageService.dataset_file_path(
|
||||||
|
"project",
|
||||||
|
"dataset",
|
||||||
|
"vector",
|
||||||
|
"source.geojson",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
asyncio.run(
|
||||||
|
StorageService.persist_upload_file(
|
||||||
|
project_id="project",
|
||||||
|
dataset_id="dataset",
|
||||||
|
dataset_type="vector",
|
||||||
|
original_filename="source.geojson",
|
||||||
|
upload=upload,
|
||||||
|
content_type="application/geo+json",
|
||||||
|
max_bytes=8,
|
||||||
|
chunk_size=3,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert exc_info.value.code == "UPLOAD_TOO_LARGE"
|
||||||
|
assert exc_info.value.status_code == 413
|
||||||
|
assert not expected_path.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_vector_staging_uses_lower_in_memory_limit(monkeypatch) -> None:
|
||||||
|
captured = {}
|
||||||
|
|
||||||
|
async def persist_upload_file(**kwargs):
|
||||||
|
captured.update(kwargs)
|
||||||
|
return {"storage_path": "unused"}
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.dataset_service.get_settings",
|
||||||
|
lambda: SimpleNamespace(max_upload_mb=500, max_in_memory_vector_mb=32),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(StorageService, "persist_upload_file", persist_upload_file)
|
||||||
|
|
||||||
|
asyncio.run(
|
||||||
|
DatasetService._stage_upload(
|
||||||
|
project_id="project",
|
||||||
|
dataset_id="dataset",
|
||||||
|
dataset_type="vector",
|
||||||
|
filename="source.geojson",
|
||||||
|
file=SimpleNamespace(content_type="application/geo+json"),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert captured["max_bytes"] == 32 * 1024 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
def test_staged_vector_read_is_bounded_before_loading_file(monkeypatch, tmp_path) -> None:
|
||||||
|
source = tmp_path / "large.geojson"
|
||||||
|
source.write_bytes(b"x" * (1024 * 1024 + 1))
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.dataset_service.get_settings",
|
||||||
|
lambda: SimpleNamespace(max_upload_mb=500, max_in_memory_vector_mb=1),
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
DatasetService._read_staged_vector_bytes({"storage_path": str(source)})
|
||||||
|
|
||||||
|
assert exc_info.value.code == "UPLOAD_TOO_LARGE"
|
||||||
|
assert exc_info.value.status_code == 413
|
||||||
|
assert not source.exists()
|
||||||
|
|||||||
@@ -0,0 +1,210 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from geoalchemy2.shape import from_shape
|
||||||
|
import pytest
|
||||||
|
from shapely.geometry import box
|
||||||
|
|
||||||
|
from app.core.config import Settings
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.models import Area, Dataset, DatasetVersion, SourceRegistry, SourceSnapshot
|
||||||
|
from app.services.tile_manifest_service import TileManifestService, canonical_manifest_json
|
||||||
|
|
||||||
|
|
||||||
|
class FakeSession:
|
||||||
|
def __init__(self, objects=None) -> None:
|
||||||
|
self.objects = objects or {}
|
||||||
|
|
||||||
|
def get(self, model, item_id):
|
||||||
|
return self.objects.get((model, item_id))
|
||||||
|
|
||||||
|
|
||||||
|
def _dataset_and_session(*, with_area: bool = True):
|
||||||
|
project_id = uuid4()
|
||||||
|
dataset_id = uuid4()
|
||||||
|
registry_id = uuid4()
|
||||||
|
snapshot_id = uuid4()
|
||||||
|
checksum = "a" * 64
|
||||||
|
area = None
|
||||||
|
area_id = uuid4() if with_area else None
|
||||||
|
if area_id is not None:
|
||||||
|
area = Area(
|
||||||
|
id=area_id,
|
||||||
|
project_id=project_id,
|
||||||
|
name="Inference AOI",
|
||||||
|
geometry=from_shape(box(4.0, 51.0, 5.0, 52.0), srid=4326),
|
||||||
|
)
|
||||||
|
registry = SourceRegistry(
|
||||||
|
id=registry_id,
|
||||||
|
source_key="governed-test-raster",
|
||||||
|
display_name="Governed test raster",
|
||||||
|
classification="derived",
|
||||||
|
authority_name="GeoIntel",
|
||||||
|
)
|
||||||
|
snapshot = SourceSnapshot(
|
||||||
|
id=snapshot_id,
|
||||||
|
source_registry_id=registry_id,
|
||||||
|
snapshot_key="snapshot-1",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
ingest_status="ingested",
|
||||||
|
freshness_status="current",
|
||||||
|
)
|
||||||
|
dataset = Dataset(
|
||||||
|
id=dataset_id,
|
||||||
|
project_id=project_id,
|
||||||
|
area_id=area_id,
|
||||||
|
name="orthophoto.tif",
|
||||||
|
dataset_type="raster",
|
||||||
|
source="governed-test-raster",
|
||||||
|
source_name="governed-test-raster",
|
||||||
|
storage_path="storage/uploads/orthophoto.tif",
|
||||||
|
size_bytes=123,
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
crs="EPSG:4326",
|
||||||
|
bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0},
|
||||||
|
source_registry_id=registry_id,
|
||||||
|
source_snapshot_id=snapshot_id,
|
||||||
|
data_contract_key="geointel.raster.geotiff",
|
||||||
|
data_contract_version="1.0.0",
|
||||||
|
validation_status="passed",
|
||||||
|
provenance_status="complete",
|
||||||
|
lineage_status="not_applicable",
|
||||||
|
quarantine_status="not_quarantined",
|
||||||
|
status="ready",
|
||||||
|
)
|
||||||
|
dataset.source_registry = registry
|
||||||
|
dataset.source_snapshot = snapshot
|
||||||
|
version = DatasetVersion(
|
||||||
|
id=uuid4(),
|
||||||
|
dataset_id=dataset_id,
|
||||||
|
version=3,
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
)
|
||||||
|
dataset.versions.append(version)
|
||||||
|
objects = {(Dataset, dataset_id): dataset}
|
||||||
|
if area is not None:
|
||||||
|
objects[(Area, area.id)] = area
|
||||||
|
return FakeSession(objects), dataset, area
|
||||||
|
|
||||||
|
|
||||||
|
def _manifest(tmp_path: Path, db: FakeSession, dataset: Dataset) -> Path:
|
||||||
|
tile_path = tmp_path / "tile_0000.tif"
|
||||||
|
tile_path.write_bytes(b"immutable tile")
|
||||||
|
payload = {
|
||||||
|
**TileManifestService.dataset_binding(db, dataset),
|
||||||
|
"tile_set_id": "tile-set-1",
|
||||||
|
"crs": "EPSG:4326",
|
||||||
|
"source_crs": "EPSG:4326",
|
||||||
|
"dataset_crs": "EPSG:4326",
|
||||||
|
"bounds": [4.0, 51.0, 5.0, 52.0],
|
||||||
|
"count": 1,
|
||||||
|
"tiles": [
|
||||||
|
{
|
||||||
|
"path": str(tile_path),
|
||||||
|
"bounds": [4.0, 51.0, 5.0, 52.0],
|
||||||
|
"crs": "EPSG:4326",
|
||||||
|
"index": 0,
|
||||||
|
**TileManifestService.tile_integrity(tile_path),
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
manifest_path = tmp_path / "manifest.json"
|
||||||
|
manifest_path.write_text(canonical_manifest_json(payload), encoding="utf-8")
|
||||||
|
return manifest_path
|
||||||
|
|
||||||
|
|
||||||
|
def _validate(tmp_path: Path, db: FakeSession, dataset: Dataset, *, prefix: str = "DETECTION"):
|
||||||
|
manifest_path = tmp_path / "manifest.json"
|
||||||
|
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
return TileManifestService.validate_for_inference(
|
||||||
|
db,
|
||||||
|
dataset,
|
||||||
|
payload,
|
||||||
|
manifest_path=manifest_path,
|
||||||
|
settings=Settings(_env_file=None, storage_root=str(tmp_path)),
|
||||||
|
error_prefix=prefix,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_versioned_tile_manifest_binds_dataset_version_snapshot_area_and_tile_bytes(tmp_path: Path) -> None:
|
||||||
|
db, dataset, _area = _dataset_and_session()
|
||||||
|
manifest_path = _manifest(tmp_path, db, dataset)
|
||||||
|
|
||||||
|
evidence = _validate(tmp_path, db, dataset)
|
||||||
|
|
||||||
|
assert evidence["manifest_path"] == str(manifest_path.resolve())
|
||||||
|
assert evidence["source_dataset_id"] == str(dataset.id)
|
||||||
|
assert evidence["source_snapshot_id"] == str(dataset.source_snapshot_id)
|
||||||
|
assert evidence["dataset_version_id"] == str(dataset.versions[0].id)
|
||||||
|
assert evidence["source_area_id"] == str(dataset.area_id)
|
||||||
|
assert evidence["tile_count"] == 1
|
||||||
|
assert evidence["tile_union_bounds_epsg4326"] == pytest.approx([4.0, 51.0, 5.0, 52.0])
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("field", "value", "expected_code"),
|
||||||
|
[
|
||||||
|
("source_dataset_id", lambda: str(uuid4()), "DETECTION_TILE_MANIFEST_DATASET_MISMATCH"),
|
||||||
|
("source_dataset_checksum_sha256", lambda: "b" * 64, "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"),
|
||||||
|
("source_snapshot_id", lambda: str(uuid4()), "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"),
|
||||||
|
("dataset_version", lambda: 99, "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"),
|
||||||
|
("source_area_geometry_sha256", lambda: "c" * 64, "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_tile_manifest_rejects_dataset_or_provenance_mismatch(
|
||||||
|
tmp_path: Path,
|
||||||
|
field: str,
|
||||||
|
value,
|
||||||
|
expected_code: str,
|
||||||
|
) -> None:
|
||||||
|
db, dataset, _area = _dataset_and_session()
|
||||||
|
manifest_path = _manifest(tmp_path, db, dataset)
|
||||||
|
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
payload[field] = value()
|
||||||
|
manifest_path.write_text(canonical_manifest_json(payload), encoding="utf-8")
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
_validate(tmp_path, db, dataset)
|
||||||
|
|
||||||
|
assert exc_info.value.code == expected_code
|
||||||
|
|
||||||
|
|
||||||
|
def test_tile_manifest_rejects_tile_bytes_changed_after_manifest_creation(tmp_path: Path) -> None:
|
||||||
|
db, dataset, _area = _dataset_and_session()
|
||||||
|
manifest_path = _manifest(tmp_path, db, dataset)
|
||||||
|
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
Path(payload["tiles"][0]["path"]).write_bytes(b"tampered tile")
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
_validate(tmp_path, db, dataset)
|
||||||
|
|
||||||
|
assert exc_info.value.code == "DETECTION_TILE_MANIFEST_TILE_INTEGRITY_MISMATCH"
|
||||||
|
|
||||||
|
|
||||||
|
def test_tile_manifest_rejects_dataset_without_version_binding(tmp_path: Path) -> None:
|
||||||
|
db, dataset, _area = _dataset_and_session()
|
||||||
|
_manifest(tmp_path, db, dataset)
|
||||||
|
dataset.versions.clear()
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
_validate(tmp_path, db, dataset)
|
||||||
|
|
||||||
|
assert exc_info.value.code == "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"
|
||||||
|
assert "dataset_version_id" in exc_info.value.details["missing_fields"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_segmentation_tile_manifest_rejects_union_outside_dataset_scope(tmp_path: Path) -> None:
|
||||||
|
db, dataset, _area = _dataset_and_session(with_area=False)
|
||||||
|
manifest_path = _manifest(tmp_path, db, dataset)
|
||||||
|
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
payload["bounds"] = [4.0, 51.0, 6.0, 52.0]
|
||||||
|
payload["tiles"][0]["bounds"] = [4.0, 51.0, 6.0, 52.0]
|
||||||
|
manifest_path.write_text(canonical_manifest_json(payload), encoding="utf-8")
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
_validate(tmp_path, db, dataset, prefix="SEGMENTATION")
|
||||||
|
|
||||||
|
assert exc_info.value.code == "SEGMENTATION_TILE_MANIFEST_SCOPE_MISMATCH"
|
||||||
+114
-23
@@ -13,8 +13,8 @@ This document freezes the first API shape. Codex may add implementation details
|
|||||||
uses the canonical `{"data": ...}` envelope. Readiness runs an OpenAPI audit
|
uses the canonical `{"data": ...}` envelope. Readiness runs an OpenAPI audit
|
||||||
that rejects free-form dictionary responses and envelope drift.
|
that rejects free-form dictionary responses and envelope drift.
|
||||||
- The only successful non-envelope responses are `/health`, `/health/live`,
|
- The only successful non-envelope responses are `/health`, `/health/live`,
|
||||||
`/health/ready`, the four documented persisted-raster PNG endpoints and the
|
`/health/ready`, the two documented Authentik redirects, the documented
|
||||||
export artifact download endpoint.
|
persisted-raster PNG endpoints and the export artifact download endpoint.
|
||||||
|
|
||||||
## Shared schemas
|
## Shared schemas
|
||||||
|
|
||||||
@@ -63,7 +63,7 @@ one client/username combination within five minutes temporarily return HTTP
|
|||||||
|
|
||||||
Optional guest access is a configuration-gated demonstration mode. It creates
|
Optional guest access is a configuration-gated demonstration mode. It creates
|
||||||
a shorter signed session with role `guest`, scopes that session to the
|
a shorter signed session with role `guest`, scopes that session to the
|
||||||
idempotently seeded demo project and blocks mutating operator routes. Project
|
idempotently seeded demo project and blocks administrative operator routes. Project
|
||||||
listing is filtered to the bound demo project. The frontend exposes the same
|
listing is filtered to the bound demo project. The frontend exposes the same
|
||||||
exploration, assistant, model-selection, analysis, QA and export workspaces as
|
exploration, assistant, model-selection, analysis, QA and export workspaces as
|
||||||
an operator. Model catalogs are globally readable; every run, result and export
|
an operator. Model catalogs are globally readable; every run, result and export
|
||||||
@@ -73,13 +73,33 @@ administrative mutations remain unavailable. This is deliberately **not**
|
|||||||
a substitute for user accounts, authorization or tenant isolation; expose it
|
a substitute for user accounts, authorization or tenant isolation; expose it
|
||||||
only on a dedicated demo installation without private or operational data.
|
only on a dedicated demo installation without private or operational data.
|
||||||
|
|
||||||
|
The functional demo boundary includes bounded, project-path-scoped official
|
||||||
|
source acquisition, persisted bbox selections, temporal comparisons and change
|
||||||
|
detection. Change detection resolves the source dataset first, requires its
|
||||||
|
project to equal the signed guest-project UUID and validates the target dataset
|
||||||
|
against that same project before the synchronous job starts. Cross-project
|
||||||
|
datasets therefore fail before comparison. Generic uploads, arbitrary
|
||||||
|
clip/buffer/intersect operations, forceful project management and evidence
|
||||||
|
adjudication remain blocked.
|
||||||
|
|
||||||
|
The login/demo seed remains offline and retains its synthetic raster as an
|
||||||
|
explicit `fixture` for UI context and fixture QA only. That raster is excluded
|
||||||
|
from configured detection and segmentation selectors and the production
|
||||||
|
consumption gate continues to reject it. A guest starts real model inference
|
||||||
|
by drawing a bounded map selection; GeoIntel then acquires an official regional
|
||||||
|
orthophoto on demand inside the signed demo project, persists its contextual
|
||||||
|
source registry/snapshot/checksum evidence, and only then tiles and runs the
|
||||||
|
configured model. No network acquisition occurs merely by logging in.
|
||||||
|
|
||||||
### GET `/api/v1/auth/session`
|
### GET `/api/v1/auth/session`
|
||||||
|
|
||||||
Public session probe used by the frontend before it mounts the workbench.
|
Public session probe used by the frontend before it mounts the workbench.
|
||||||
When authentication is disabled, `authenticated` is true and
|
When authentication is disabled, `authenticated` is true and
|
||||||
`authentication_required` is false so local development retains its existing
|
`authentication_required` is false so local development retains its existing
|
||||||
direct workflow. `guest_access_enabled` tells the landing page whether it may
|
direct workflow. `guest_access_enabled` tells the landing page whether it may
|
||||||
show the guest action.
|
show the guest action. `authentik_enabled` indicates that the additive
|
||||||
|
Authentik operator flow is fully configured; the local operator login remains
|
||||||
|
available as a recovery path.
|
||||||
|
|
||||||
```json
|
```json
|
||||||
{
|
{
|
||||||
@@ -90,6 +110,7 @@ show the guest action.
|
|||||||
"expires_at": null,
|
"expires_at": null,
|
||||||
"role": null,
|
"role": null,
|
||||||
"guest_access_enabled": true,
|
"guest_access_enabled": true,
|
||||||
|
"authentik_enabled": false,
|
||||||
"guest_project_id": null
|
"guest_project_id": null
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -112,6 +133,23 @@ Successful login sets the session cookie and returns the authenticated session
|
|||||||
shape. Invalid credentials return HTTP 401 `INVALID_CREDENTIALS`; username
|
shape. Invalid credentials return HTTP 401 `INVALID_CREDENTIALS`; username
|
||||||
existence is not disclosed.
|
existence is not disclosed.
|
||||||
|
|
||||||
|
### GET `/api/v1/auth/authentik/start`
|
||||||
|
|
||||||
|
Starts an authorization-code OIDC flow with PKCE, signed state and nonce when
|
||||||
|
all Authentik settings are present. Discovery, token and JWKS requests are
|
||||||
|
restricted to the configured HTTPS issuer origin, reject redirects and enforce
|
||||||
|
a bounded JSON response size. The flow cookie is HttpOnly, Secure, ten minutes
|
||||||
|
or less and scoped to the Authentik callback path.
|
||||||
|
|
||||||
|
### GET `/api/v1/auth/authentik/callback`
|
||||||
|
|
||||||
|
Validates issuer, audience, signature, expiry, state, nonce and the exact
|
||||||
|
configured verified e-mail address. Success creates the same operator session
|
||||||
|
as local login and clears the one-use flow cookie. Failure clears that cookie
|
||||||
|
and redirects to the landing page with a generic error marker; token or
|
||||||
|
identity details are never returned to the browser. These two redirect routes
|
||||||
|
are the only additional non-envelope authentication responses.
|
||||||
|
|
||||||
### POST `/api/v1/auth/guest`
|
### POST `/api/v1/auth/guest`
|
||||||
|
|
||||||
No request body is required. The endpoint is available only when both
|
No request body is required. The endpoint is available only when both
|
||||||
@@ -130,6 +168,12 @@ is available for that bound demo project. Unscoped analysis routes require the
|
|||||||
same UUID as a `project_id` query parameter; cross-project values fail before
|
same UUID as a `project_id` query parameter; cross-project values fail before
|
||||||
route execution. Coverage resolution additionally verifies the `project_id` in
|
route execution. Coverage resolution additionally verifies the `project_id` in
|
||||||
the request body against the guest-session scope.
|
the request body against the guest-session scope.
|
||||||
|
Raster tiling for an already-persisted raster in the bound demo project is an
|
||||||
|
explicitly allowed preparation step for detection and segmentation. It creates
|
||||||
|
only integrity-bound inference tiles and a job record; uploads, source
|
||||||
|
acquisition, model management and arbitrary derived-dataset writes remain
|
||||||
|
operator-only. Guest tiling is server-capped by the configured inference tile
|
||||||
|
limit before any tile bytes are written.
|
||||||
|
|
||||||
### POST `/api/v1/auth/logout`
|
### POST `/api/v1/auth/logout`
|
||||||
|
|
||||||
@@ -286,11 +330,13 @@ Request:
|
|||||||
|
|
||||||
Backend responsibilities:
|
Backend responsibilities:
|
||||||
|
|
||||||
- Validate geometry.
|
- Accept only a finite, valid `Polygon` or `MultiPolygon` in the declared CRS.
|
||||||
- Repair trivial polygon issues if safe.
|
- Reject unknown/non-2D CRS definitions and geometry outside the Belgium and
|
||||||
- Store geometry in PostGIS.
|
Belgian North Sea workbench domain.
|
||||||
- Calculate area in square meters using projected CRS.
|
- Transform the geometry to EPSG:4326 before PostGIS persistence while
|
||||||
- Store bbox.
|
retaining the exact declared CRS in `original_crs`.
|
||||||
|
- Calculate `area_m2` in Belgian Lambert 72 (`EPSG:31370`) and derive the
|
||||||
|
persisted EPSG:4326 bbox from the normalized geometry.
|
||||||
|
|
||||||
### GET `/api/v1/projects/{project_id}/areas/{area_id}`
|
### GET `/api/v1/projects/{project_id}/areas/{area_id}`
|
||||||
|
|
||||||
@@ -299,8 +345,12 @@ area list endpoint and includes persisted GeoJSON geometry for map display.
|
|||||||
|
|
||||||
### PATCH `/api/v1/projects/{project_id}/areas/{area_id}`
|
### PATCH `/api/v1/projects/{project_id}/areas/{area_id}`
|
||||||
|
|
||||||
Updates the area name and/or geometry. Geometry updates follow the same
|
Updates the area name and/or geometry. A replacement geometry follows the same
|
||||||
validation, repair and metric-calculation rules as area creation.
|
strict CRS validation, EPSG:4326 normalization and metric-calculation rules as
|
||||||
|
area creation and atomically recomputes `geometry`, `bbox`, `area_m2` and
|
||||||
|
`original_crs`. A PATCH containing `crs` without `geometry` fails with
|
||||||
|
`INVALID_AREA_CRS_UPDATE`; omitted CRS on replacement geometry means
|
||||||
|
EPSG:4326.
|
||||||
|
|
||||||
### GET `/api/v1/projects/{project_id}/areas/municipalities`
|
### GET `/api/v1/projects/{project_id}/areas/municipalities`
|
||||||
|
|
||||||
@@ -409,10 +459,22 @@ validation report/status, provenance/lineage status, quarantine status and an
|
|||||||
idempotent ingest key. A malformed or doubtful artifact is retained as
|
idempotent ingest key. A malformed or doubtful artifact is retained as
|
||||||
`status=quarantined`; it is not silently discarded or made ready.
|
`status=quarantined`; it is not silently discarded or made ready.
|
||||||
|
|
||||||
Vector uploads remain stored as original files and are also persisted into
|
The backend streams uploads to governed storage in bounded 8 MiB reads while
|
||||||
`vector_features` as queryable PostGIS state only after their contract passes.
|
calculating size and SHA-256. `GEOINTEL_MAX_UPLOAD_MB` (legacy alias
|
||||||
Non-EPSG:4326 vector coordinates are explicitly transformed before canonical
|
`MAX_UPLOAD_MB`, default 500, allowed range 1–2,048) is enforced by the backend even when a request
|
||||||
feature persistence; relabelling Lambert coordinates as EPSG:4326 is rejected.
|
bypasses the reverse proxy. The first byte beyond the configured limit aborts
|
||||||
|
the ingest, removes the partial file and returns HTTP 413 with
|
||||||
|
`UPLOAD_TOO_LARGE`. Because GeoJSON validation currently requires an in-memory
|
||||||
|
parse, vector uploads have the additional lower
|
||||||
|
`GEOINTEL_MAX_IN_MEMORY_VECTOR_MB` limit (default 64 MiB, maximum 256 MiB).
|
||||||
|
This limit is enforced during streaming and again with a bounded read before
|
||||||
|
parsing; larger vector sources must use a governed batch-import workflow.
|
||||||
|
|
||||||
|
Vector uploads are also persisted into `vector_features` as queryable PostGIS
|
||||||
|
state only after their contract passes. Non-EPSG:4326 source bytes are retained
|
||||||
|
as provenance evidence while the normal Dataset `storage_path` points to the
|
||||||
|
deterministic EPSG:4326 artifact. Relabelling Lambert coordinates as EPSG:4326
|
||||||
|
is rejected.
|
||||||
|
|
||||||
### GET `/api/v1/source-registry`
|
### GET `/api/v1/source-registry`
|
||||||
|
|
||||||
@@ -1053,7 +1115,13 @@ Failure modes:
|
|||||||
|
|
||||||
### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/tile`
|
### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/tile`
|
||||||
|
|
||||||
Generate raster tiles and a manifest for downstream processing. Returns a job payload with `tile_set_id` and manifest metadata.
|
Generate raster tiles and a versioned manifest for downstream processing.
|
||||||
|
Returns a job payload with `tile_set_id` and manifest metadata. Contract
|
||||||
|
`geointel.raster.tile-manifest@2.0.0` binds the tile set to the exact source
|
||||||
|
Dataset, DatasetVersion, source registry/snapshot and checksum values that are
|
||||||
|
available in persistence, plus the source Area and Area-geometry checksum when
|
||||||
|
scoped. Every tile records immutable byte size/SHA-256, explicit CRS and bounds;
|
||||||
|
the manifest records its source extent and tile count.
|
||||||
|
|
||||||
If raster processing dependencies are unavailable:
|
If raster processing dependencies are unavailable:
|
||||||
|
|
||||||
@@ -1594,13 +1662,16 @@ label or model filename.
|
|||||||
|
|
||||||
### GET `/api/v1/detection/model-assets`
|
### GET `/api/v1/detection/model-assets`
|
||||||
|
|
||||||
Returns governed local runtime model files. This is a read-only catalog.
|
Returns configured local runtime model files. This is a read-only catalog;
|
||||||
|
catalog visibility establishes byte identity and runtime availability only.
|
||||||
GeoIntel never downloads, creates, mutates or deletes model weights from this
|
GeoIntel never downloads, creates, mutates or deletes model weights from this
|
||||||
endpoint.
|
endpoint.
|
||||||
|
|
||||||
The backend scans `YOLO_MODELS_DIR` (default `/app/models`). When
|
The backend scans `YOLO_MODELS_DIR` (default `/app/models`). When
|
||||||
`YOLO_MODEL_PATH` resolves to an existing file, production catalog output is
|
`YOLO_MODEL_PATH` resolves to an existing file, production catalog output is
|
||||||
restricted to that explicitly approved active model. When no active model is
|
restricted to that active runtime model. Active/available is a runtime-selection
|
||||||
|
state only: it does not assert governed validation, human review, national
|
||||||
|
coverage or promotion. When no active model is
|
||||||
configured, supported `.pt`, `.onnx` and `.engine` files remain visible for
|
configured, supported `.pt`, `.onnx` and `.engine` files remain visible for
|
||||||
development/operator discovery but cannot make the configured detector ready.
|
development/operator discovery but cannot make the configured detector ready.
|
||||||
|
|
||||||
@@ -1620,8 +1691,12 @@ Response data:
|
|||||||
"size_bytes": 123456,
|
"size_bytes": 123456,
|
||||||
"sha256": "sha256hex",
|
"sha256": "sha256hex",
|
||||||
"active": true,
|
"active": true,
|
||||||
"status": "approved",
|
"runtime_available": true,
|
||||||
"limitation_message": "Approved local runtime model asset. GeoIntel will not download or mutate model weights.",
|
"runtime_status": "active",
|
||||||
|
"governed_validation_status": "not_verified_by_catalog",
|
||||||
|
"promotion_status": "not_verified_by_catalog",
|
||||||
|
"status": "runtime_active",
|
||||||
|
"limitation_message": "Active local runtime model asset. Runtime selection is not evidence of governed validation or promotion.",
|
||||||
"will_download_models": false
|
"will_download_models": false
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
@@ -1732,10 +1807,21 @@ governed, runtime-produced artifact the persistence model requires. Rejection is
|
|||||||
`STORAGE_PATH_OUTSIDE_ROOT`; `GEOINTEL_ALLOW_EXTERNAL_ARTIFACT_PATHS` opts out
|
`STORAGE_PATH_OUTSIDE_ROOT`; `GEOINTEL_ALLOW_EXTERNAL_ARTIFACT_PATHS` opts out
|
||||||
for provisioning workflows that stage tiles before ingest.
|
for provisioning workflows that stage tiles before ingest.
|
||||||
|
|
||||||
|
Configured detection and segmentation revalidate the v2 contract before model
|
||||||
|
loading. The requested Dataset identity, checksum, latest DatasetVersion,
|
||||||
|
source snapshot and Area binding must still match; every tile checksum is
|
||||||
|
recomputed; and the EPSG:4326 tile union must remain inside both the declared
|
||||||
|
manifest extent and persisted raster extent and intersect its Area where one is
|
||||||
|
bound. Validation fails closed with the typed suffixes
|
||||||
|
`TILE_MANIFEST_DATASET_MISMATCH`, `TILE_MANIFEST_PROVENANCE_MISMATCH`,
|
||||||
|
`TILE_MANIFEST_TILE_INTEGRITY_MISMATCH` or
|
||||||
|
`TILE_MANIFEST_SCOPE_MISMATCH`, prefixed with `DETECTION_` or
|
||||||
|
`SEGMENTATION_` for the requesting task.
|
||||||
|
|
||||||
The manifest must carry explicit CRS metadata (`crs`, `source_crs` or
|
The manifest must carry explicit CRS metadata (`crs`, `source_crs` or
|
||||||
`dataset_crs`). A manifest without it fails with
|
`dataset_crs`). A manifest without it fails rather than being georeferenced
|
||||||
`DETECTION_TILE_MANIFEST_INVALID` rather than being georeferenced against an
|
against an assumed EPSG:4326, which would place detections plausibly but
|
||||||
assumed EPSG:4326, which would place detections plausibly but wrongly.
|
wrongly.
|
||||||
|
|
||||||
Tiles are read with rasterio: the visible RGB bands are selected explicitly and
|
Tiles are read with rasterio: the visible RGB bands are selected explicitly and
|
||||||
percentile-stretched to 8-bit, so 16-bit and 4-band (RGB + NIR) orthophotos
|
percentile-stretched to 8-bit, so 16-bit and 4-band (RGB + NIR) orthophotos
|
||||||
@@ -2095,6 +2181,11 @@ to the synchronous route. A configured model requires an existing
|
|||||||
`SEGMENTATION_ACCELERATOR_MISCONFIGURED` when CUDA is required. A valid
|
`SEGMENTATION_ACCELERATOR_MISCONFIGURED` when CUDA is required. A valid
|
||||||
zero-polygon run is shown as an empty model result, never as proof that the AOI
|
zero-polygon run is shown as an empty model result, never as proof that the AOI
|
||||||
contains no relevant objects.
|
contains no relevant objects.
|
||||||
|
When a raster is selected but no manifest exists, the normal frontend flow
|
||||||
|
first requests project-scoped 512 px tiles with 64 px overlap and forwards the
|
||||||
|
server-generated manifest. Inference then applies the complete v2 provenance,
|
||||||
|
integrity and scope validation before model load; manual server-path entry is
|
||||||
|
an operator-only advanced control.
|
||||||
|
|
||||||
Validation errors:
|
Validation errors:
|
||||||
|
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ ALLOWED_NON_ENVELOPE_ENDPOINTS = {
|
|||||||
("GET", "/health"),
|
("GET", "/health"),
|
||||||
("GET", "/health/live"),
|
("GET", "/health/live"),
|
||||||
("GET", "/health/ready"),
|
("GET", "/health/ready"),
|
||||||
|
("GET", "/api/v1/auth/authentik/start"),
|
||||||
|
("GET", "/api/v1/auth/authentik/callback"),
|
||||||
("GET", "/api/v1/exports/{export_id}/download"),
|
("GET", "/api/v1/exports/{export_id}/download"),
|
||||||
("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/image"),
|
("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/image"),
|
||||||
("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/terrain/image"),
|
("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/terrain/image"),
|
||||||
|
|||||||
@@ -1,5 +1,11 @@
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
missing=[d for d in ["contracts/api","contracts/database","contracts/events"] if not (ROOT/d).exists()]
|
missing = [
|
||||||
if missing: raise SystemExit("Missing contract directories: "+", ".join(missing))
|
d
|
||||||
|
for d in ["contracts/api", "contracts/database", "contracts/events"]
|
||||||
|
if not (ROOT / d).exists()
|
||||||
|
]
|
||||||
|
if missing:
|
||||||
|
raise SystemExit("Missing contract directories: " + ", ".join(missing))
|
||||||
print("Contracts smoke OK")
|
print("Contracts smoke OK")
|
||||||
|
|||||||
Reference in New Issue
Block a user