Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Full platform remediation #7

Merged
Jens merged 11 commits from codex/full-platform-remediation into main 2026-08-30 09:30:42 +00:00
Conversation 0 Commits 11 Files Changed 1359
+2335 -312
63 changed files with 2335 additions and 312 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 80a2d1654d - Show all commits
backend/app/api/routes/analysis.py
+5 -1
View File
@@ -1,8 +1,9 @@
from __future__ import annotations
from fastapi import APIRouter, Depends
from fastapi import APIRouter, Depends, Request
from sqlalchemy.orm import Session
from app.api.guest_scope import assert_guest_project_scope
from app.core.errors import AppError
from app.db.session import get_db
from app.models import Dataset
@@ -18,12 +19,15 @@ router = APIRouter(prefix="/analysis", tags=["analysis"])
@router.post("/change-detection", response_model=Envelope[JobRead])
def run_change_detection(
payload: ChangeDetectionRequest,
request: Request,
db: Session = Depends(get_db),
) -> dict:
source_dataset = db.get(Dataset, payload.source_dataset_id)
if not source_dataset:
raise AppError(code="DATASET_NOT_FOUND", message="Source dataset not found", status_code=404)
assert_guest_project_scope(request, source_dataset.project_id)
ChangeDetectionService._get_project_vector_dataset(db, payload.source_dataset_id, source_dataset.project_id, "Source")
ChangeDetectionService._get_project_vector_dataset(db, payload.target_dataset_id, source_dataset.project_id, "Target")
job = JobService.run_sync_job(
db=db,
project_id=source_dataset.project_id,
backend/app/api/routes/datasets.py
+7 -3
View File
@@ -5,13 +5,13 @@ from datetime import datetime
from typing import Any
from uuid import UUID
from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Response
from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, Response
from fastapi import UploadFile
from sqlalchemy.orm import Session
from app.models import Area, Project
from app.core.config import get_settings
from app.core.errors import AppError
from app.db.session import get_db
from app.models import Area, Project
from app.schemas import (
BathymetryPartitionFinalizationResult,
BathymetrySourceProbeRead,
@@ -1172,11 +1172,14 @@ def raster_tile_dataset(
project_id: UUID,
dataset_id: UUID,
payload: RasterTileRequest,
request: Request,
db: Session = Depends(get_db),
):
dataset = DatasetService.get_dataset(db, dataset_id)
if dataset.project_id != project_id:
raise HTTPException(status_code=404, detail="Dataset not found")
principal = getattr(request.state, "auth_principal", None)
guest_max_tiles = get_settings().yolo_max_tiles if getattr(principal, "role", None) == "guest" else None
job = _run_job_sync(
db=db,
project_id=project_id,
@@ -1189,6 +1192,7 @@ def raster_tile_dataset(
tile_size=payload.tile_size,
overlap=payload.overlap,
output_name=payload.output_name,
max_tiles=guest_max_tiles,
),
)
return envelope(job)
backend/app/models.py
+1 -1
View File
@@ -1 +1 @@
from app.models import *
from app.models import * # noqa: F403 - legacy compatibility shim re-exports the package API
backend/app/schemas/area.py
+1
View File
@@ -14,6 +14,7 @@ class AreaCreate(BaseModel):
class AreaUpdate(BaseModel):
name: str | None = None
geometry: dict | None = None
crs: str | None = None
backend/app/schemas/detection.py
+4
View File
@@ -42,6 +42,10 @@ class ModelAssetRead(BaseModel):
size_bytes: int
sha256: str
active: bool
runtime_available: bool
runtime_status: str
governed_validation_status: str
promotion_status: str
status: str
limitation_message: str
will_download_models: bool = False
backend/app/services/aoi_operation_executor.py
+188 -40
View File
@@ -12,7 +12,10 @@ from app.schemas.spw_terrain import SpwTerrainAcquireRequest
from app.schemas.official_vector import OfficialVectorAcquireRequest
from app.schemas.flood_hazard import FloodHazardAcquireRequest
from app.schemas.thematic_raster import ThematicRasterAcquireRequest
from app.schemas.bathymetry import BathymetryProfileAcquireRequest, MdkBathymetryAcquireRequest
from app.schemas.bathymetry import (
BathymetryProfileAcquireRequest,
MdkBathymetryAcquireRequest,
)
from app.schemas.job import JobCreate
from app.schemas.operations import VectorSelectionBBox
from app.schemas.orthophoto import OrthophotoAcquireRequest
@@ -20,12 +23,20 @@ from app.services.aoi_operation_service import AoiOperationService
from app.services.grb_acquisition_service import GrbAcquisitionService
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
from app.services.spw_terrain_service import SpwTerrainService
from app.services.official_vector_acquisition_service import OfficialVectorAcquisitionService
from app.services.official_vector_acquisition_service import (
OfficialVectorAcquisitionService,
)
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
from app.services.thematic_raster_acquisition_service import (
ThematicRasterAcquisitionService,
)
from app.services.walous_land_cover_service import WalousLandCoverService
from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
from app.services.mdk_bathymetry_acquisition_service import MdkBathymetryAcquisitionService
from app.services.bathymetry_profile_acquisition_service import (
BathymetryProfileAcquisitionService,
)
from app.services.mdk_bathymetry_acquisition_service import (
MdkBathymetryAcquisitionService,
)
from app.services.job_service import JobService
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
@@ -40,65 +51,202 @@ class AoiOperationExecutor:
AoiOperationService._refresh_parent(db, operation_id)
return AoiOperationService.read(db, project_id, operation_id)
operation = db.get(AoiOperation, operation_id)
child = JobService.create_job(db, JobCreate(
job_type=f"aoi.{operation.operation_type}.partition",
project_id=project_id,
parameters_json={
"aoi_operation_id": str(operation_id),
"partition_id": str(partition.id),
"partition_key": partition.partition_key,
"provider_key": partition.provider_key,
"product_key": partition.product_key,
},
))
child = JobService.create_job(
db,
JobCreate(
job_type=f"aoi.{operation.operation_type}.partition",
project_id=project_id,
parameters_json={
"aoi_operation_id": str(operation_id),
"partition_id": str(partition.id),
"partition_key": partition.partition_key,
"provider_key": partition.provider_key,
"product_key": partition.product_key,
},
),
)
partition = db.get(AoiOperationPartition, partition.id)
partition.child_job_id = child.id
db.add(partition); db.commit()
db.add(partition)
db.commit()
JobService.mark_running(db, child.id)
try:
result = AoiOperationExecutor._dispatch(db, project_id, operation, partition)
output_id = result.get("output_dataset_id") if isinstance(result, dict) else None
JobService.mark_success(db, child.id, result=result, output_dataset_id=UUID(str(output_id)) if output_id else None)
return AoiOperationService.complete(db, project_id, operation_id, partition.id, result)
result = AoiOperationExecutor._dispatch(
db, project_id, operation, partition
)
output_id = (
result.get("output_dataset_id") if isinstance(result, dict) else None
)
JobService.mark_success(
db,
child.id,
result=result,
output_dataset_id=UUID(str(output_id)) if output_id else None,
)
return AoiOperationService.complete(
db, project_id, operation_id, partition.id, result
)
except AppError as exc:
JobService.mark_failed(db, child.id, exc.message, {"code": exc.code, "details": exc.details})
return AoiOperationService.fail(db, project_id, operation_id, partition.id, exc.message, AoiOperationExecutor._retryable(exc), {"code": exc.code, "details": exc.details})
JobService.mark_failed(
db, child.id, exc.message, {"code": exc.code, "details": exc.details}
)
return AoiOperationService.fail(
db,
project_id,
operation_id,
partition.id,
exc.message,
AoiOperationExecutor._retryable(exc),
{"code": exc.code, "details": exc.details},
)
except Exception:
try:
db.rollback()
JobService.mark_failed(db, child.id, "Unexpected partition execution error", {"code": "AOI_PARTITION_INTERNAL_ERROR"})
JobService.mark_failed(
db,
child.id,
"Unexpected partition execution error",
{"code": "AOI_PARTITION_INTERNAL_ERROR"},
)
finally:
AoiOperationService.fail(db, project_id, operation_id, partition.id, "Unexpected partition execution error", True, {"code": "AOI_PARTITION_INTERNAL_ERROR"})
AoiOperationService.fail(
db,
project_id,
operation_id,
partition.id,
"Unexpected partition execution error",
True,
{"code": "AOI_PARTITION_INTERNAL_ERROR"},
)
raise
@staticmethod
def _dispatch(db, project_id: UUID, operation: AoiOperation, partition: AoiOperationPartition) -> dict:
def _dispatch(
db, project_id: UUID, operation: AoiOperation, partition: AoiOperationPartition
) -> dict:
geometry = to_shape(partition.geometry)
min_x, min_y, max_x, max_y = geometry.bounds
bbox = VectorSelectionBBox(min_x=min_x, min_y=min_y, max_x=max_x, max_y=max_y, crs="EPSG:4326")
force_refresh = bool((operation.request_json or {}).get("parameters_json", {}).get("force_refresh", False))
bbox = VectorSelectionBBox(
min_x=min_x, min_y=min_y, max_x=max_x, max_y=max_y, crs="EPSG:4326"
)
force_refresh = bool(
(operation.request_json or {})
.get("parameters_json", {})
.get("force_refresh", False)
)
if partition.provider_key == "grb":
return GrbAcquisitionService.acquire(db, project_id, GrbAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
return GrbAcquisitionService.acquire(
db,
project_id,
GrbAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "orthophoto":
return OrthophotoAcquisitionService.acquire(db, project_id, OrthophotoAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
return OrthophotoAcquisitionService.acquire(
db,
project_id,
OrthophotoAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "dhmv":
return DhmvAcquisitionService.acquire(db, project_id, DhmvAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
return DhmvAcquisitionService.acquire(
db,
project_id,
DhmvAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "spw_terrain":
return SpwTerrainService.acquire(db, project_id, SpwTerrainAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
return SpwTerrainService.acquire(
db,
project_id,
SpwTerrainAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "official_vector":
return OfficialVectorAcquisitionService.acquire(db, project_id, OfficialVectorAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
return OfficialVectorAcquisitionService.acquire(
db,
project_id,
OfficialVectorAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "flood_hazard":
return FloodHazardAcquisitionService.acquire(db, project_id, FloodHazardAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
return FloodHazardAcquisitionService.acquire(
db,
project_id,
FloodHazardAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "thematic_raster":
return ThematicRasterAcquisitionService.acquire(db, project_id, ThematicRasterAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
return ThematicRasterAcquisitionService.acquire(
db,
project_id,
ThematicRasterAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "walous":
return WalousLandCoverService.acquire(db, project_id, ThematicRasterAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh))
return WalousLandCoverService.acquire(
db,
project_id,
ThematicRasterAcquireRequest(
bbox=bbox,
area_id=operation.area_id,
product_key=partition.product_key,
force_refresh=force_refresh,
),
)
if partition.provider_key == "bathymetry_profiles":
return BathymetryProfileAcquisitionService.acquire(db, project_id, BathymetryProfileAcquireRequest(bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh))
return BathymetryProfileAcquisitionService.acquire(
db,
project_id,
BathymetryProfileAcquireRequest(
bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh
),
)
if partition.provider_key == "mdk_bathymetry":
return MdkBathymetryAcquisitionService.acquire(db, project_id, MdkBathymetryAcquireRequest(bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh))
raise AppError(code="AOI_PROVIDER_UNSUPPORTED", message="No governed AOI executor is registered for this provider", details={"provider_key": partition.provider_key}, status_code=422)
return MdkBathymetryAcquisitionService.acquire(
db,
project_id,
MdkBathymetryAcquireRequest(
bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh
),
)
raise AppError(
code="AOI_PROVIDER_UNSUPPORTED",
message="No governed AOI executor is registered for this provider",
details={"provider_key": partition.provider_key},
status_code=422,
)
@staticmethod
def _retryable(error: AppError) -> bool:
return error.status_code >= 500 or error.code.endswith(("TIMEOUT", "UNAVAILABLE", "TLS_ERROR"))
return error.status_code >= 500 or error.code.endswith(
("TIMEOUT", "UNAVAILABLE", "TLS_ERROR")
)
backend/app/services/aoi_operation_service.py
+273 -68
View File
@@ -22,8 +22,11 @@ class AoiOperationService:
_to_metric = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
_to_wgs84 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
SCOPE_AREA_NAMES = {
"belgium": "Belgium land", "flanders": "Flanders", "wallonia": "Wallonia",
"brussels": "Brussels-Capital Region", "belgian_north_sea": "Belgian part of the North Sea",
"belgium": "Belgium land",
"flanders": "Flanders",
"wallonia": "Wallonia",
"brussels": "Brussels-Capital Region",
"belgian_north_sea": "Belgian part of the North Sea",
"territorial_sea": "Belgian territorial sea (0-12 nautical miles)",
"exclusive_economic_zone": "Belgian exclusive economic zone beyond territorial sea",
"continental_shelf": "Belgian continental shelf beyond territorial sea",
@@ -32,13 +35,19 @@ class AoiOperationService:
@staticmethod
def create(db, project_id: UUID, payload: AoiOperationCreate) -> dict:
if db.get(Project, project_id) is None:
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
raise AppError(
code="PROJECT_NOT_FOUND", message="Project not found", status_code=404
)
geometry = AoiOperationService._resolve_geometry(db, project_id, payload)
if payload.coverage_zone:
geometry = AoiOperationService._clip_to_zone(db, project_id, geometry, payload.coverage_zone)
geometry = AoiOperationService._clip_to_zone(
db, project_id, geometry, payload.coverage_zone
)
geometry = AoiOperationService._as_multipolygon(geometry)
metric_geometry = transform(AoiOperationService._to_metric.transform, geometry)
partition_side_m = AoiOperationService._partition_side(payload.provider_key, payload.max_partition_side_m)
partition_side_m = AoiOperationService._partition_side(
payload.provider_key, payload.max_partition_side_m
)
cells = AoiOperationService._partition(metric_geometry, partition_side_m)
operation_id = uuid4()
now = datetime.now(timezone.utc)
@@ -53,7 +62,9 @@ class AoiOperationService:
plan_json={
"partition_strategy": "epsg31370_square_grid_intersection_v1",
"max_partition_side_m": partition_side_m,
"budget_source": "governed_provider_registry" if payload.max_partition_side_m is None else "stricter_operator_override",
"budget_source": "governed_provider_registry"
if payload.max_partition_side_m is None
else "stricter_operator_override",
"partition_count": len(cells),
"provider_key": payload.provider_key,
"product_key": payload.product_key,
@@ -65,13 +76,21 @@ class AoiOperationService:
wgs84 = transform(AoiOperationService._to_wgs84.transform, cell)
wgs84 = AoiOperationService._as_multipolygon(wgs84)
digest = sha256(wgs84.wkb).hexdigest()[:20]
db.add(AoiOperationPartition(
id=uuid4(), operation_id=operation_id,
partition_key=f"{payload.provider_key}:{payload.product_key}:{ordinal:05d}:{digest}",
provider_key=payload.provider_key, product_key=payload.product_key,
ordinal=ordinal, status="queued", geometry=from_shape(wgs84, srid=4326),
attempt_count=0, max_attempts=payload.max_attempts, created_at=now,
))
db.add(
AoiOperationPartition(
id=uuid4(),
operation_id=operation_id,
partition_key=f"{payload.provider_key}:{payload.product_key}:{ordinal:05d}:{digest}",
provider_key=payload.provider_key,
product_key=payload.product_key,
ordinal=ordinal,
status="queued",
geometry=from_shape(wgs84, srid=4326),
attempt_count=0,
max_attempts=payload.max_attempts,
created_at=now,
)
)
db.commit()
return AoiOperationService.read(db, project_id, operation_id)
@@ -79,13 +98,32 @@ class AoiOperationService:
def _clip_to_zone(db, project_id: UUID, geometry, zone: str):
area_name = AoiOperationService.SCOPE_AREA_NAMES.get(zone)
if area_name is None:
raise AppError(code="AOI_COVERAGE_ZONE_UNSUPPORTED", message="Unknown governed coverage zone", details={"coverage_zone": zone}, status_code=422)
scope = db.query(Area).filter(Area.project_id == project_id, Area.name == area_name).first()
raise AppError(
code="AOI_COVERAGE_ZONE_UNSUPPORTED",
message="Unknown governed coverage zone",
details={"coverage_zone": zone},
status_code=422,
)
scope = (
db.query(Area)
.filter(Area.project_id == project_id, Area.name == area_name)
.first()
)
if scope is None:
raise AppError(code="AOI_COVERAGE_ZONE_NOT_MATERIALIZED", message="The governed coverage-zone geometry is not persisted in this project", details={"coverage_zone": zone}, status_code=409)
raise AppError(
code="AOI_COVERAGE_ZONE_NOT_MATERIALIZED",
message="The governed coverage-zone geometry is not persisted in this project",
details={"coverage_zone": zone},
status_code=409,
)
clipped = geometry.intersection(to_shape(scope.geometry))
if clipped.is_empty:
raise AppError(code="AOI_OUTSIDE_PROVIDER_ZONE", message="The AOI does not intersect the provider coverage zone", details={"coverage_zone": zone}, status_code=422)
raise AppError(
code="AOI_OUTSIDE_PROVIDER_ZONE",
message="The AOI does not intersect the provider coverage zone",
details={"coverage_zone": zone},
status_code=422,
)
return clipped
@staticmethod
@@ -94,19 +132,30 @@ class AoiOperationService:
return MultiPolygon([geometry])
if isinstance(geometry, MultiPolygon):
return geometry
polygons = [part for part in getattr(geometry, "geoms", []) if isinstance(part, Polygon)]
polygons = [
part for part in getattr(geometry, "geoms", []) if isinstance(part, Polygon)
]
if not polygons:
raise AppError(code="AOI_GEOMETRY_EMPTY", message="AOI contains no polygonal area after clipping", status_code=422)
raise AppError(
code="AOI_GEOMETRY_EMPTY",
message="AOI contains no polygonal area after clipping",
status_code=422,
)
return MultiPolygon(polygons)
@staticmethod
def _partition_side(provider_key: str, requested: float | None) -> float:
settings = get_settings()
def raster_side(max_side_m: float, max_pixels: int, resolution_m: float) -> float:
def raster_side(
max_side_m: float, max_pixels: int, resolution_m: float
) -> float:
# Keep every square grid cell within both the provider's spatial
# extent limit and its decoded-pixel budget. The small safety
# margin absorbs ceil/edge rounding in the acquisition services.
pixel_limited_side = math.sqrt(float(max_pixels)) * float(resolution_m) * 0.99
pixel_limited_side = (
math.sqrt(float(max_pixels)) * float(resolution_m) * 0.99
)
return min(float(max_side_m), pixel_limited_side)
budgets = {
@@ -142,22 +191,37 @@ class AoiOperationService:
"mdk_bathymetry": 20_000.0,
}
if provider_key not in budgets:
raise AppError(code="AOI_PROVIDER_UNSUPPORTED", message="No governed partition budget is registered for this provider", details={"provider_key": provider_key}, status_code=422)
raise AppError(
code="AOI_PROVIDER_UNSUPPORTED",
message="No governed partition budget is registered for this provider",
details={"provider_key": provider_key},
status_code=422,
)
governed = budgets[provider_key]
return min(governed, float(requested)) if requested is not None else governed
@staticmethod
def _resolve_geometry(db, project_id: UUID, payload: AoiOperationCreate):
if (payload.area_id is None) == (payload.bbox is None):
raise AppError(code="AOI_SELECTION_REQUIRED", message="Provide exactly one area_id or bbox", status_code=422)
raise AppError(
code="AOI_SELECTION_REQUIRED",
message="Provide exactly one area_id or bbox",
status_code=422,
)
if payload.area_id is not None:
area = db.get(Area, payload.area_id)
if area is None or area.project_id != project_id:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
raise AppError(
code="AREA_NOT_FOUND", message="Area not found", status_code=404
)
return to_shape(area.geometry)
bbox = payload.bbox
if bbox is None or bbox.crs != "EPSG:4326":
raise AppError(code="INVALID_AOI_CRS", message="AOI bbox must use EPSG:4326", status_code=422)
raise AppError(
code="INVALID_AOI_CRS",
message="AOI bbox must use EPSG:4326",
status_code=422,
)
return box(bbox.min_x, bbox.min_y, bbox.max_x, bbox.max_y)
@staticmethod
@@ -166,11 +230,26 @@ class AoiOperationService:
columns = max(1, math.ceil((max_x - min_x) / side_m))
rows = max(1, math.ceil((max_y - min_y) / side_m))
if columns * rows > AoiOperationService.MAX_PARTITIONS:
raise AppError(code="AOI_PARTITION_LIMIT_EXCEEDED", message="AOI requires too many bounded partitions", details={"candidate_count": columns * rows, "max_partitions": AoiOperationService.MAX_PARTITIONS}, status_code=422)
raise AppError(
code="AOI_PARTITION_LIMIT_EXCEEDED",
message="AOI requires too many bounded partitions",
details={
"candidate_count": columns * rows,
"max_partitions": AoiOperationService.MAX_PARTITIONS,
},
status_code=422,
)
partitions = []
for row in range(rows):
for column in range(columns):
clipped = geometry.intersection(box(min_x + column * side_m, min_y + row * side_m, min(min_x + (column + 1) * side_m, max_x), min(min_y + (row + 1) * side_m, max_y)))
clipped = geometry.intersection(
box(
min_x + column * side_m,
min_y + row * side_m,
min(min_x + (column + 1) * side_m, max_x),
min(min_y + (row + 1) * side_m, max_y),
)
)
if not clipped.is_empty and clipped.area > 0:
partitions.append(clipped)
return partitions
@@ -179,98 +258,224 @@ class AoiOperationService:
def read(db, project_id: UUID, operation_id: UUID) -> dict:
operation = db.get(AoiOperation, operation_id)
if operation is None or operation.project_id != project_id:
raise AppError(code="AOI_OPERATION_NOT_FOUND", message="AOI operation not found", status_code=404)
partitions = db.query(AoiOperationPartition).filter(AoiOperationPartition.operation_id == operation_id).order_by(AoiOperationPartition.ordinal).all()
raise AppError(
code="AOI_OPERATION_NOT_FOUND",
message="AOI operation not found",
status_code=404,
)
partitions = (
db.query(AoiOperationPartition)
.filter(AoiOperationPartition.operation_id == operation_id)
.order_by(AoiOperationPartition.ordinal)
.all()
)
counts = Counter(partition.status for partition in partitions)
complete = counts["success"] + counts["skipped"]
return {
"id": operation.id, "project_id": operation.project_id, "area_id": operation.area_id,
"parent_job_id": operation.parent_job_id, "operation_type": operation.operation_type,
"status": operation.status, "request_json": operation.request_json, "plan_json": operation.plan_json,
"result_json": operation.result_json, "error_message": operation.error_message,
"id": operation.id,
"project_id": operation.project_id,
"area_id": operation.area_id,
"parent_job_id": operation.parent_job_id,
"operation_type": operation.operation_type,
"status": operation.status,
"request_json": operation.request_json,
"plan_json": operation.plan_json,
"result_json": operation.result_json,
"error_message": operation.error_message,
"progress": round(complete / len(partitions), 6) if partitions else 0.0,
"partition_counts": dict(counts), "partitions": partitions,
"created_at": operation.created_at, "started_at": operation.started_at, "finished_at": operation.finished_at,
"partition_counts": dict(counts),
"partitions": partitions,
"created_at": operation.created_at,
"started_at": operation.started_at,
"finished_at": operation.finished_at,
}
@staticmethod
def list(db, project_id: UUID, limit: int = 50) -> dict:
rows = db.query(AoiOperation).filter(AoiOperation.project_id == project_id).order_by(AoiOperation.created_at.desc()).limit(limit).all()
return {"items": [AoiOperationService.read(db, project_id, row.id) for row in rows], "total": len(rows)}
rows = (
db.query(AoiOperation)
.filter(AoiOperation.project_id == project_id)
.order_by(AoiOperation.created_at.desc())
.limit(limit)
.all()
)
return {
"items": [AoiOperationService.read(db, project_id, row.id) for row in rows],
"total": len(rows),
}
@staticmethod
def claim_next(db, project_id: UUID, operation_id: UUID):
operation = db.get(AoiOperation, operation_id)
if operation is None or operation.project_id != project_id:
raise AppError(code="AOI_OPERATION_NOT_FOUND", message="AOI operation not found", status_code=404)
partition = db.query(AoiOperationPartition).filter(AoiOperationPartition.operation_id == operation_id, AoiOperationPartition.status == "queued").order_by(AoiOperationPartition.ordinal).with_for_update(skip_locked=True).first()
raise AppError(
code="AOI_OPERATION_NOT_FOUND",
message="AOI operation not found",
status_code=404,
)
partition = (
db.query(AoiOperationPartition)
.filter(
AoiOperationPartition.operation_id == operation_id,
AoiOperationPartition.status == "queued",
)
.order_by(AoiOperationPartition.ordinal)
.with_for_update(skip_locked=True)
.first()
)
if partition is None:
return None
now = datetime.now(timezone.utc)
partition.status = "running"; partition.started_at = now; partition.attempt_count += 1; partition.error_message = None
operation.status = "running"; operation.started_at = operation.started_at or now
db.add(partition); db.add(operation); db.commit(); db.refresh(partition)
partition.status = "running"
partition.started_at = now
partition.attempt_count += 1
partition.error_message = None
operation.status = "running"
operation.started_at = operation.started_at or now
db.add(partition)
db.add(operation)
db.commit()
db.refresh(partition)
return partition
@staticmethod
def checkpoint(db, project_id: UUID, operation_id: UUID, partition_id: UUID, checkpoint: dict):
partition = AoiOperationService._partition_row(db, project_id, operation_id, partition_id)
def checkpoint(
db, project_id: UUID, operation_id: UUID, partition_id: UUID, checkpoint: dict
):
partition = AoiOperationService._partition_row(
db, project_id, operation_id, partition_id
)
if partition.status != "running":
raise AppError(code="AOI_PARTITION_NOT_RUNNING", message="Only a running partition can be checkpointed", status_code=409)
partition.checkpoint_json = checkpoint; db.add(partition); db.commit(); db.refresh(partition)
raise AppError(
code="AOI_PARTITION_NOT_RUNNING",
message="Only a running partition can be checkpointed",
status_code=409,
)
partition.checkpoint_json = checkpoint
db.add(partition)
db.commit()
db.refresh(partition)
return partition
@staticmethod
def complete(db, project_id: UUID, operation_id: UUID, partition_id: UUID, result: dict, skipped: bool = False):
partition = AoiOperationService._partition_row(db, project_id, operation_id, partition_id)
def complete(
db,
project_id: UUID,
operation_id: UUID,
partition_id: UUID,
result: dict,
skipped: bool = False,
):
partition = AoiOperationService._partition_row(
db, project_id, operation_id, partition_id
)
if partition.status == "success" or partition.status == "skipped":
return AoiOperationService.read(db, project_id, operation_id)
if partition.status != "running":
raise AppError(code="AOI_PARTITION_NOT_RUNNING", message="Only a running partition can complete", status_code=409)
partition.status = "skipped" if skipped else "success"; partition.result_json = result; partition.finished_at = datetime.now(timezone.utc)
db.add(partition); db.commit(); AoiOperationService._refresh_parent(db, operation_id)
raise AppError(
code="AOI_PARTITION_NOT_RUNNING",
message="Only a running partition can complete",
status_code=409,
)
partition.status = "skipped" if skipped else "success"
partition.result_json = result
partition.finished_at = datetime.now(timezone.utc)
db.add(partition)
db.commit()
AoiOperationService._refresh_parent(db, operation_id)
return AoiOperationService.read(db, project_id, operation_id)
@staticmethod
def fail(db, project_id: UUID, operation_id: UUID, partition_id: UUID, message: str, retryable: bool, details: dict):
partition = AoiOperationService._partition_row(db, project_id, operation_id, partition_id)
partition.error_message = message; partition.result_json = {"details": details}
partition.status = "queued" if retryable and partition.attempt_count < partition.max_attempts else "failed"
partition.finished_at = None if partition.status == "queued" else datetime.now(timezone.utc)
db.add(partition); db.commit(); AoiOperationService._refresh_parent(db, operation_id)
def fail(
db,
project_id: UUID,
operation_id: UUID,
partition_id: UUID,
message: str,
retryable: bool,
details: dict,
):
partition = AoiOperationService._partition_row(
db, project_id, operation_id, partition_id
)
partition.error_message = message
partition.result_json = {"details": details}
partition.status = (
"queued"
if retryable and partition.attempt_count < partition.max_attempts
else "failed"
)
partition.finished_at = (
None if partition.status == "queued" else datetime.now(timezone.utc)
)
db.add(partition)
db.commit()
AoiOperationService._refresh_parent(db, operation_id)
return AoiOperationService.read(db, project_id, operation_id)
@staticmethod
def _partition_row(db, project_id, operation_id, partition_id):
operation = db.get(AoiOperation, operation_id); partition = db.get(AoiOperationPartition, partition_id)
if operation is None or operation.project_id != project_id or partition is None or partition.operation_id != operation_id:
raise AppError(code="AOI_PARTITION_NOT_FOUND", message="AOI partition not found", status_code=404)
operation = db.get(AoiOperation, operation_id)
partition = db.get(AoiOperationPartition, partition_id)
if (
operation is None
or operation.project_id != project_id
or partition is None
or partition.operation_id != operation_id
):
raise AppError(
code="AOI_PARTITION_NOT_FOUND",
message="AOI partition not found",
status_code=404,
)
return partition
@staticmethod
def _refresh_parent(db, operation_id):
operation = db.get(AoiOperation, operation_id)
partitions = db.query(AoiOperationPartition).filter(AoiOperationPartition.operation_id == operation_id).order_by(AoiOperationPartition.ordinal).all()
partitions = (
db.query(AoiOperationPartition)
.filter(AoiOperationPartition.operation_id == operation_id)
.order_by(AoiOperationPartition.ordinal)
.all()
)
statuses = [partition.status for partition in partitions]
output_dataset_ids = []
for partition in partitions:
output_id = (partition.result_json or {}).get("output_dataset_id") if isinstance(partition.result_json, dict) else None
output_id = (
(partition.result_json or {}).get("output_dataset_id")
if isinstance(partition.result_json, dict)
else None
)
if output_id and str(output_id) not in output_dataset_ids:
output_dataset_ids.append(str(output_id))
operation.result_json = {
"partition_count": len(partitions),
"completed_partition_count": sum(status in {"success", "skipped"} for status in statuses),
"completed_partition_count": sum(
status in {"success", "skipped"} for status in statuses
),
"failed_partition_count": statuses.count("failed"),
"output_dataset_ids": output_dataset_ids,
"merge_contract": "source_aware_spatial_union",
"vector_deduplication": "source_feature_id_then_geometry",
"raster_deduplication": "governed_mosaic_grid",
"complete_coverage": bool(statuses) and all(status in {"success", "skipped"} for status in statuses),
"complete_coverage": bool(statuses)
and all(status in {"success", "skipped"} for status in statuses),
}
now = datetime.now(timezone.utc)
if statuses and all(status in {"success", "skipped"} for status in statuses):
operation.status = "success"; operation.finished_at = now; operation.error_message = None
elif "failed" in statuses and not any(status in {"queued", "running"} for status in statuses):
operation.status = "partial" if any(status in {"success", "skipped"} for status in statuses) else "failed"; operation.finished_at = now
operation.status = "success"
operation.finished_at = now
operation.error_message = None
elif "failed" in statuses and not any(
status in {"queued", "running"} for status in statuses
):
operation.status = (
"partial"
if any(status in {"success", "skipped"} for status in statuses)
else "failed"
)
operation.finished_at = now
operation.error_message = "One or more bounded source partitions failed; inspect partition evidence."
db.add(operation); db.commit()
db.add(operation)
db.commit()
backend/app/services/area_service.py
+28 -7
View File
@@ -9,7 +9,7 @@ from shapely.geometry import mapping
from app.core.errors import AppError
from app.models import Area, Dataset, Project, VectorFeature
from app.schemas.area import AreaCreate, AreaRead, AreaUpdate
from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_to_multipolygon
from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_area_to_epsg4326
class AreaService:
@@ -126,7 +126,11 @@ class AreaService:
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
try:
multipolygon = normalize_to_multipolygon(payload.geometry)
multipolygon, original_crs = normalize_area_to_epsg4326(
payload.geometry,
payload.crs or "EPSG:4326",
)
metric_area = area_m2(multipolygon)
except ValueError as exc:
raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc
@@ -134,8 +138,8 @@ class AreaService:
project_id=project_id,
name=payload.name.strip() or "Unnamed area",
geometry=from_shape(multipolygon, srid=4326),
original_crs=payload.crs or "EPSG:4326",
area_m2=area_m2(multipolygon),
original_crs=original_crs,
area_m2=metric_area,
bbox=from_shape(geometry_bbox_polygon(multipolygon), srid=4326),
)
db.add(area)
@@ -157,11 +161,28 @@ class AreaService:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
changed = False
if payload.name:
if payload.name is not None and payload.name.strip():
area.name = payload.name.strip() or area.name
changed = True
if payload.crs:
area.original_crs = payload.crs
if payload.crs is not None and payload.geometry is None:
raise AppError(
code="INVALID_AREA_CRS_UPDATE",
message="crs can only be supplied together with replacement geometry",
status_code=422,
)
if payload.geometry is not None:
try:
multipolygon, original_crs = normalize_area_to_epsg4326(
payload.geometry,
payload.crs or "EPSG:4326",
)
metric_area = area_m2(multipolygon)
except ValueError as exc:
raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc
area.geometry = from_shape(multipolygon, srid=4326)
area.original_crs = original_crs
area.area_m2 = metric_area
area.bbox = from_shape(geometry_bbox_polygon(multipolygon), srid=4326)
changed = True
if not changed:
raise AppError(code="NO_CHANGES", message="No updatable fields provided", status_code=422)
backend/app/services/bathymetry_profile_acquisition_service.py
+1 -1
View File
@@ -7,7 +7,7 @@ import math
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
from urllib.request import Request
from uuid import UUID
from geoalchemy2.shape import to_shape
backend/app/services/bathymetry_raster_analysis_service.py
-1
View File
@@ -112,7 +112,6 @@ class BathymetryRasterAnalysisService:
try:
import numpy as np
import rasterio
from rasterio.features import geometry_mask
from rasterio.mask import mask
except ImportError as exc:
raise AppError(
backend/app/services/dataset_service.py
+104 -34
View File
@@ -17,6 +17,7 @@ from shapely.geometry import MultiPoint, shape
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.core.config import get_settings
from app.models import Area, Dataset, DatasetVersion, Project
from app.services.data_contract_validation import (
ContractKind,
@@ -476,6 +477,29 @@ class DatasetService:
content_type=content_type,
)
@classmethod
def _persist_vector_source_evidence_from_path(
cls,
*,
project_id: UUID,
dataset_id: UUID,
original_filename: str,
source_path: str | Path,
content_type: str | None,
) -> dict[str, Any]:
safe_filename = StorageService._safe_filename(original_filename)
evidence_path = (
StorageService.dataset_root(str(project_id), str(dataset_id), "vector")
/ "provenance"
/ f"{dataset_id}_source_{safe_filename}"
)
return StorageService.persist_file_from_path(
str(evidence_path),
source_path,
original_filename=safe_filename,
content_type=content_type,
)
@classmethod
def _record_vector_source_evidence(
cls,
@@ -853,6 +877,52 @@ class DatasetService:
raise AppError(code="INVALID_UPLOAD", message="Missing file name", status_code=400)
return filename
@staticmethod
async def _stage_upload(
*,
project_id: UUID,
dataset_id: uuid.UUID,
dataset_type: str,
filename: str,
file: UploadFile,
) -> dict[str, Any]:
settings = get_settings()
max_upload_mb = int(settings.max_upload_mb)
if DatasetService._canonical_dataset_type(dataset_type) == "vector":
max_upload_mb = min(max_upload_mb, int(settings.max_in_memory_vector_mb))
return await StorageService.persist_upload_file(
project_id=str(project_id),
dataset_id=str(dataset_id),
dataset_type=dataset_type,
original_filename=filename,
upload=file,
content_type=file.content_type,
max_bytes=max_upload_mb * 1024 * 1024,
)
@staticmethod
def _read_staged_vector_bytes(storage_info: dict[str, Any]) -> bytes:
settings = get_settings()
max_bytes = min(
int(settings.max_upload_mb),
int(settings.max_in_memory_vector_mb),
) * 1024 * 1024
path = Path(str(storage_info["storage_path"]))
with path.open("rb") as stream:
content = stream.read(max_bytes + 1)
if len(content) > max_bytes:
StorageService.remove_dataset_file(str(path))
raise AppError(
code="UPLOAD_TOO_LARGE",
message="Vector upload exceeds the bounded in-memory parsing limit.",
details={
"max_bytes": max_bytes,
"max_in_memory_vector_mb": max_bytes // (1024 * 1024),
},
status_code=413,
)
return content
@staticmethod
def list_datasets(db: Session, project_id: UUID, limit: int = 50, offset: int = 0) -> tuple[list[DatasetCreateResponse], int]:
total = db.query(Dataset).filter(Dataset.project_id == project_id).count()
@@ -1135,15 +1205,15 @@ class DatasetService:
status_code=415,
)
raw = await file.read()
storage_info = StorageService.persist_dataset_file(
project_id=str(project_id),
dataset_id=str(dataset_id := uuid.uuid4()),
dataset_id = uuid.uuid4()
storage_info = await DatasetService._stage_upload(
project_id=project_id,
dataset_id=dataset_id,
dataset_type=canonical_type,
original_filename=filename,
content=raw,
content_type=file.content_type,
filename=filename,
file=file,
)
raw = DatasetService._read_staged_vector_bytes(storage_info) if canonical_type == "vector" else None
metadata: dict[str, Any] = {}
vector_payload: dict[str, Any] | None = None
@@ -1151,6 +1221,7 @@ class DatasetService:
try:
status = "validating"
if canonical_type == "vector":
assert raw is not None
try:
text = raw.decode("utf-8")
except UnicodeDecodeError as exc:
@@ -1319,8 +1390,15 @@ class DatasetService:
temporal_granularity=temporal_granularity,
source_version=source_version,
)
raw = await file.read()
checksum_sha256 = StorageService.calculate_checksum_sha256(raw)
dataset_id = uuid.uuid4()
storage_info = await DatasetService._stage_upload(
project_id=project_id,
dataset_id=dataset_id,
dataset_type=canonical_type,
filename=filename,
file=file,
)
checksum_sha256 = storage_info["checksum_sha256"]
ingest_key = DatasetService._ingest_key(
project_id=project_id,
source_key="manual",
@@ -1333,6 +1411,7 @@ class DatasetService:
)
existing = DatasetService._find_existing_ingest(db, project_id, ingest_key)
if existing is not None:
StorageService.remove_dataset_file(storage_info["storage_path"])
return DatasetService._to_response(existing)
raw_source_metadata = dict(source_metadata or {})
@@ -1361,8 +1440,7 @@ class DatasetService:
}
)
dataset_id = uuid.uuid4()
storage_info: dict[str, Any] | None = None
raw = DatasetService._read_staged_vector_bytes(storage_info) if canonical_type == "vector" else None
storage_content = raw
source_evidence: dict[str, Any] | None = None
imported_at = datetime.now(timezone.utc)
@@ -1372,6 +1450,7 @@ class DatasetService:
parser_error: tuple[str, str] | None = None
try:
if canonical_type == "vector":
assert raw is not None
try:
payload = json.loads(raw.decode("utf-8"))
except UnicodeDecodeError as exc:
@@ -1394,22 +1473,22 @@ class DatasetService:
)
if DatasetService._vector_storage_requires_canonicalization(source_crs):
storage_content = DatasetService._canonical_vector_storage_bytes(canonical_vector_payload)
source_evidence = DatasetService._persist_vector_source_evidence(
source_evidence = DatasetService._persist_vector_source_evidence_from_path(
project_id=project_id,
dataset_id=dataset_id,
original_filename=filename,
content=raw,
source_path=storage_info["storage_path"],
content_type=file.content_type,
)
storage_info = StorageService.persist_dataset_file(
project_id=str(project_id),
dataset_id=str(dataset_id),
dataset_type=canonical_type,
original_filename=filename,
content=storage_content,
content_type=file.content_type,
)
else:
storage_info = StorageService.persist_dataset_file(
project_id=str(project_id),
dataset_id=str(dataset_id),
dataset_type=canonical_type,
original_filename=filename,
content=raw,
content_type=file.content_type,
)
metadata = extract_raster_metadata(storage_info["storage_path"])
metadata["dataset_type"] = "raster"
source_crs = metadata.get("crs")
@@ -1422,16 +1501,7 @@ class DatasetService:
"processing_code": code,
}
if storage_info is None:
storage_info = StorageService.persist_dataset_file(
project_id=str(project_id),
dataset_id=str(dataset_id),
dataset_type=canonical_type,
original_filename=filename,
content=storage_content,
content_type=file.content_type,
)
computed_storage_checksum_sha256 = StorageService.calculate_checksum_sha256(storage_content)
computed_storage_checksum_sha256 = storage_info["checksum_sha256"]
if source_evidence is not None:
resolved_source_crs = source_crs or DatasetService.CANONICAL_VECTOR_CRS
DatasetService._record_vector_source_evidence(
@@ -1502,7 +1572,7 @@ class DatasetService:
feature_collection=canonical_vector_payload or {"type": "FeatureCollection", "features": []},
checksum_sha256=storage_info["checksum_sha256"],
computed_checksum_sha256=computed_storage_checksum_sha256,
content=storage_content,
content=None,
source_registry_id=str(source_registry.id),
source_snapshot_id=str(source_snapshot.id),
imported_at=imported_at,
@@ -1541,8 +1611,8 @@ class DatasetService:
bounds=DatasetService._extract_raster_bounds_json(metadata),
resolution=resolution,
checksum_sha256=storage_info["checksum_sha256"],
computed_checksum_sha256=checksum_sha256,
content=raw,
computed_checksum_sha256=storage_info["checksum_sha256"],
content=None,
source_registry_id=str(source_registry.id),
source_snapshot_id=str(source_snapshot.id),
imported_at=imported_at,
backend/app/services/detection_review_service.py
-1
View File
@@ -1,7 +1,6 @@
from __future__ import annotations
from collections import Counter
from typing import Any
from uuid import UUID
from sqlalchemy.orm import Session
backend/app/services/detection_service.py
+27
View File
@@ -33,6 +33,7 @@ from app.services.storage_service import StorageService
from app.services.quality_service import QualityService
from app.services.runtime_model_provenance_service import RuntimeModelProvenance, RuntimeModelProvenanceService
from app.services.temporal_compatibility_service import TemporalCompatibilityService
from app.services.tile_manifest_service import TileManifestService
from app.services.yolo_adapter import YoloDetectionAdapter
@@ -968,6 +969,18 @@ class DetectionService:
yolo_adapter_class: Type[YoloDetectionAdapter],
) -> tuple[list[Detection], dict[str, Any]]:
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles, settings)
dataset = db.get(Dataset, dataset_id)
if dataset is None:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
manifest_binding = TileManifestService.validate_for_inference(
db,
dataset,
manifest,
manifest_path=tile_manifest_path or "",
settings=settings,
error_prefix="DETECTION",
)
DetectionService._attach_tile_manifest_binding(analysis_run, job, manifest_binding)
model_path = Path(settings.yolo_model_path or "").expanduser()
runtime_model_provenance = RuntimeModelProvenanceService.validate_for_production_runtime(
db=db,
@@ -1075,9 +1088,23 @@ class DetectionService:
"tile_edge_truncated_count": len(candidates) - len(edge_filtered_candidates),
"duplicate_iou_threshold": float(settings.yolo_duplicate_iou_threshold),
"containment_suppression_threshold": float(settings.yolo_containment_nms_threshold),
"tile_manifest_binding": manifest_binding,
"runtime_model_provenance": runtime_model_provenance.as_dict(),
}
@staticmethod
def _attach_tile_manifest_binding(
analysis_run: AnalysisRun,
job: Job,
binding: dict[str, Any],
) -> None:
analysis_parameters = dict(analysis_run.parameters_json or {})
analysis_parameters["tile_manifest_binding"] = dict(binding)
analysis_run.parameters_json = analysis_parameters
job_parameters = dict(job.parameters_json or {})
job_parameters["tile_manifest_binding"] = dict(binding)
job.parameters_json = job_parameters
@staticmethod
def _attach_runtime_model_provenance(
analysis_run: AnalysisRun,
backend/app/services/dhmv_acquisition_service.py
+1 -1
View File
@@ -12,7 +12,7 @@ from pathlib import Path
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
from urllib.request import Request
from uuid import UUID
from geoalchemy2.shape import to_shape
backend/app/services/flood_hazard_acquisition_service.py
+1 -1
View File
@@ -12,7 +12,7 @@ from pathlib import Path
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
from urllib.request import Request
from uuid import UUID
from xml.etree import ElementTree
backend/app/services/flood_hazard_analysis_service.py
-1
View File
@@ -232,7 +232,6 @@ class FloodHazardAnalysisService:
try:
import numpy as np
import rasterio
from rasterio.features import geometry_mask
from rasterio.mask import mask
except ImportError as exc:
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for flood-hazard analysis", status_code=503) from exc
backend/app/services/mdk_bathymetry_acquisition_service.py
+1 -1
View File
@@ -5,7 +5,7 @@ from datetime import UTC, datetime
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
from urllib.request import Request, urlopen
from urllib.request import Request
from uuid import UUID
from app.core.config import Settings, get_settings
backend/app/services/mdk_bathymetry_probe_service.py
+2 -1
View File
@@ -6,11 +6,12 @@ import ssl
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
from urllib.request import Request, urlopen
from urllib.request import Request
from xml.etree import ElementTree
from app.core.config import Settings, get_settings
from app.schemas.bathymetry import BathymetrySourceProbeRead
from app.services.outbound_request_guard import guarded_opener
class MdkBathymetryProbeService:
backend/app/services/model_asset_catalog_service.py
+7 -3
View File
@@ -79,11 +79,15 @@ class ModelAssetCatalogService:
size_bytes=path.stat().st_size,
sha256=ModelAssetCatalogService._sha256(path),
active=active_model_path == resolved_path,
status="approved" if active_model_path == resolved_path else "available",
runtime_available=True,
runtime_status="active" if active_model_path == resolved_path else "available",
governed_validation_status="not_verified_by_catalog",
promotion_status="not_verified_by_catalog",
status="runtime_active" if active_model_path == resolved_path else "runtime_available",
limitation_message=(
"Approved local runtime model asset. GeoIntel will not download or mutate model weights."
"Active local runtime model asset. Runtime selection is not evidence of governed validation or promotion."
if active_model_path == resolved_path
else "Local development model asset. Configure it explicitly before production use."
else "Local runtime model asset. Governed validation and promotion are not established by this catalog."
),
will_download_models=False,
)
backend/app/services/orthophoto_acquisition_service.py
+1 -1
View File
@@ -11,7 +11,7 @@ from pathlib import Path
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
from urllib.request import Request
from uuid import UUID
from geoalchemy2.shape import to_shape
backend/app/services/raster_operations_service.py
+73 -7
View File
@@ -1,8 +1,8 @@
from __future__ import annotations
import json
import uuid
from datetime import datetime, timezone
from hashlib import sha256
from pathlib import Path
from typing import Any
@@ -16,6 +16,7 @@ from app.models import Area, Dataset, DatasetVersion
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
from app.services.raster_service import extract_raster_metadata
from app.services.storage_service import StorageService
from app.services.tile_manifest_service import TileManifestService, canonical_manifest_json
def _import_rasterio():
@@ -70,6 +71,39 @@ class RasterOperationsService:
raise AppError(code="DATASET_FILE_MISSING", message="Stored raster file missing", status_code=404)
return dataset
@staticmethod
def _validate_storage_checksum(dataset: Dataset) -> None:
"""Refuse tiling pixels that no longer match the governed Dataset row."""
expected_checksum = str(dataset.checksum_sha256 or "").strip().lower()
governed_artifact = bool(getattr(dataset, "data_contract_key", None))
valid_checksum = len(expected_checksum) == 64 and all(character in "0123456789abcdef" for character in expected_checksum)
if not valid_checksum:
if expected_checksum or governed_artifact:
raise AppError(
code="DATASET_STORAGE_CHECKSUM_UNVERIFIABLE",
message="Governed raster storage requires a valid SHA-256 checksum before tiling.",
details={"checksum_sha256": dataset.checksum_sha256},
status_code=409,
)
return
digest = sha256()
with Path(str(dataset.storage_path)).open("rb") as stream:
for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""):
digest.update(chunk)
actual_checksum = digest.hexdigest()
if actual_checksum != expected_checksum:
raise AppError(
code="DATASET_STORAGE_CHECKSUM_MISMATCH",
message="Raster dataset storage no longer matches its validated checksum.",
details={
"expected_checksum_sha256": expected_checksum,
"actual_checksum_sha256": actual_checksum,
},
status_code=409,
)
@staticmethod
def _raster_dependencies() -> tuple[Any, Any]:
try:
@@ -970,14 +1004,17 @@ class RasterOperationsService:
tile_size: int = 512,
overlap: int = 64,
output_name: str | None = None,
max_tiles: int | None = None,
) -> dict[str, Any]:
RasterOperationsService._validate_tile_request(tile_size=tile_size, overlap=overlap)
if max_tiles is not None and max_tiles <= 0:
raise AppError(code="INVALID_PARAMETERS", message="max_tiles must be positive", status_code=400)
dataset = RasterOperationsService._load_dataset(db, dataset_id)
RasterOperationsService._validate_storage_checksum(dataset)
rasterio, _ = RasterOperationsService._raster_dependencies()
tile_set_id = str(uuid.uuid4())
tile_root = StorageService.raster_tiles_root(str(dataset.project_id), str(dataset.id), tile_set_id)
tile_root.mkdir(parents=True, exist_ok=True)
manifest_tiles: list[dict[str, Any]] = []
tile_paths: list[str] = []
@@ -997,9 +1034,23 @@ class RasterOperationsService:
source_width = int(source.width)
source_height = int(source.height)
step = max(1, tile_size - overlap)
x_offsets = RasterOperationsService._tile_offsets(source_width, tile_size, step)
y_offsets = RasterOperationsService._tile_offsets(source_height, tile_size, step)
expected_tile_count = len(x_offsets) * len(y_offsets)
if max_tiles is not None and expected_tile_count > max_tiles:
raise AppError(
code="RASTER_TILE_LIMIT_EXCEEDED",
message="Raster tile generation exceeds the guest analysis limit.",
details={
"expected_tile_count": expected_tile_count,
"max_tiles": max_tiles,
},
status_code=422,
)
tile_root.mkdir(parents=True, exist_ok=True)
tile_index = 0
for yoff in range(0, source_height, step):
for xoff in range(0, source_width, step):
for yoff in y_offsets:
for xoff in x_offsets:
tile_width = min(tile_size, source_width - xoff)
tile_height = min(tile_size, source_height - yoff)
if tile_width <= 0 or tile_height <= 0:
@@ -1022,6 +1073,7 @@ class RasterOperationsService:
tile_dest.write(tile_data)
tile_paths.append(str(tile_path))
tile_integrity = TileManifestService.tile_integrity(tile_path)
manifest_tiles.append(
{
"path": str(tile_path),
@@ -1030,6 +1082,7 @@ class RasterOperationsService:
"transform": [float(item) for item in transform.to_gdal()],
"crs": source_crs,
"index": tile_index,
**tile_integrity,
},
)
tile_index += 1
@@ -1044,9 +1097,8 @@ class RasterOperationsService:
bounds = source_metadata.get("bounds", [0.0, 0.0, 0.0, 0.0])
manifest_crs = source_crs or source_metadata.get("crs") or dataset.crs
manifest_payload = {
**TileManifestService.dataset_binding(db, dataset),
"tile_set_id": tile_set_id,
"source_dataset_id": str(dataset.id),
"source_raster_id": str(dataset.id),
"crs": manifest_crs,
"source_crs": manifest_crs,
"dataset_crs": dataset.crs,
@@ -1066,7 +1118,7 @@ class RasterOperationsService:
"tile_server": None,
}
manifest_path = tile_root / "manifest.json"
manifest_path.write_text(json.dumps(manifest_payload), encoding="utf-8")
manifest_path.write_text(canonical_manifest_json(manifest_payload), encoding="utf-8")
return {
"dataset_id": str(dataset.id),
@@ -1079,3 +1131,17 @@ class RasterOperationsService:
"count": len(manifest_tiles),
"manifest": manifest_payload,
}
@staticmethod
def _tile_offsets(dimension: int, tile_size: int, step: int) -> list[int]:
"""Return full-tile starts plus one unique edge-aligned final start."""
if dimension <= 0 or tile_size <= 0 or step <= 0:
raise AppError(code="INVALID_PARAMETERS", message="Raster tile dimensions must be positive", status_code=400)
if dimension <= tile_size:
return [0]
final_start = dimension - tile_size
offsets = list(range(0, final_start + 1, step))
if offsets[-1] != final_start:
offsets.append(final_start)
return offsets
backend/app/services/raster_partition_analysis_service.py
-2
View File
@@ -8,7 +8,6 @@ from typing import Any
from uuid import UUID
from pyproj import Transformer
from shapely.geometry import mapping
from shapely.ops import transform as shapely_transform
from app.core.errors import AppError
@@ -116,7 +115,6 @@ class RasterPartitionAnalysisService:
try:
import numpy as np
import rasterio
from rasterio.features import geometry_mask
from rasterio.merge import merge
except ImportError as exc:
raise AppError(
backend/app/services/segmentation_service.py
+14
View File
@@ -34,6 +34,7 @@ from app.services.segmentation_adapter import (
SamSegmentationAdapter,
YoloSegmentationAdapter,
)
from app.services.tile_manifest_service import TileManifestService
class SegmentationService:
@@ -749,6 +750,18 @@ class SegmentationService:
sam_adapter_class: type[SamSegmentationAdapter],
) -> tuple[list[Segmentation], dict[str, Any]]:
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles, settings)
dataset = db.get(Dataset, dataset_id)
if dataset is None:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
manifest_binding = TileManifestService.validate_for_inference(
db,
dataset,
manifest,
manifest_path=tile_manifest_path or "",
settings=settings,
error_prefix="SEGMENTATION",
)
DetectionService._attach_tile_manifest_binding(analysis_run, job, manifest_binding)
if model_name == settings.sam_model_id:
model_path = Path(settings.sam_model_path or "").expanduser()
allowed_frameworks = ("ultralytics/sam", "sam", "ultralytics", "pytorch")
@@ -861,6 +874,7 @@ class SegmentationService:
"duplicate_iou_threshold": float(settings.segmentation_duplicate_iou_threshold),
"containment_suppression_threshold": float(settings.segmentation_containment_nms_threshold),
"tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()),
"tile_manifest_binding": manifest_binding,
"runtime_model_provenance": runtime_model_provenance.as_dict(),
}
backend/app/services/source_catalog_probe_service.py
+1 -1
View File
@@ -10,7 +10,7 @@ from threading import Lock
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
from urllib.request import Request, urlopen
from urllib.request import Request
from uuid import UUID
from xml.etree import ElementTree
backend/app/services/storage_service.py
+99
View File
@@ -10,6 +10,8 @@ from app.core.errors import AppError
class StorageService:
UPLOAD_CHUNK_SIZE = 8 * 1024 * 1024
@staticmethod
def _base_dir() -> Path:
return Path(get_settings().storage_root).resolve()
@@ -143,6 +145,75 @@ class StorageService:
}
return metadata
@staticmethod
async def persist_upload_file(
*,
project_id: str,
dataset_id: str,
dataset_type: str,
original_filename: str,
upload: Any,
content_type: str | None,
max_bytes: int,
chunk_size: int | None = None,
) -> dict[str, Any]:
"""Stream an UploadFile to governed storage with a hard byte limit.
The reverse proxy limit is defense in depth. This backend boundary is
authoritative as direct/loopback requests can bypass that proxy.
"""
if max_bytes <= 0:
raise ValueError("max_bytes must be positive")
resolved_chunk_size = chunk_size or StorageService.UPLOAD_CHUNK_SIZE
if resolved_chunk_size <= 0:
raise ValueError("chunk_size must be positive")
normalized_type = StorageService.normalize_dataset_type(dataset_type)
file_path = Path(
StorageService.dataset_file_path(
project_id,
dataset_id,
normalized_type,
original_filename,
)
)
file_path.parent.mkdir(parents=True, exist_ok=True)
digest = hashlib.sha256()
size_bytes = 0
try:
with file_path.open("wb") as stream:
while True:
chunk = await upload.read(resolved_chunk_size)
if not chunk:
break
size_bytes += len(chunk)
if size_bytes > max_bytes:
raise AppError(
code="UPLOAD_TOO_LARGE",
message="Upload exceeds the configured backend size limit.",
details={
"max_bytes": max_bytes,
"max_upload_mb": max_bytes // (1024 * 1024),
},
status_code=413,
)
stream.write(chunk)
digest.update(chunk)
except Exception:
file_path.unlink(missing_ok=True)
parent = file_path.parent
if parent.exists() and parent.is_dir() and not any(parent.iterdir()):
parent.rmdir()
raise
return {
"original_filename": StorageService._safe_filename(original_filename),
"stored_filename": file_path.name,
"content_type": content_type or "application/octet-stream",
"size_bytes": size_bytes,
"checksum_sha256": digest.hexdigest(),
"storage_path": str(file_path),
}
@staticmethod
def persist_dataset_file_from_path(
project_id: str,
@@ -198,6 +269,34 @@ class StorageService:
}
return metadata
@staticmethod
def persist_file_from_path(
storage_path: str,
source_path: str | Path,
original_filename: str,
content_type: str | None,
) -> dict[str, Any]:
source = Path(source_path).resolve()
if not source.is_file():
raise FileNotFoundError(f"Source artifact does not exist: {source}")
target = Path(storage_path)
target.parent.mkdir(parents=True, exist_ok=True)
digest = hashlib.sha256()
size_bytes = 0
with source.open("rb") as input_stream, target.open("wb") as output_stream:
for chunk in iter(lambda: input_stream.read(StorageService.UPLOAD_CHUNK_SIZE), b""):
output_stream.write(chunk)
digest.update(chunk)
size_bytes += len(chunk)
return {
"original_filename": StorageService._safe_filename(original_filename),
"stored_filename": target.name,
"content_type": content_type or "application/octet-stream",
"size_bytes": size_bytes,
"checksum_sha256": digest.hexdigest(),
"storage_path": str(target),
}
@staticmethod
def remove_dataset_file(path: str) -> None:
target = Path(path)
backend/app/services/terrain_analysis_service.py
-1
View File
@@ -111,7 +111,6 @@ class TerrainAnalysisService:
try:
import numpy as np
import rasterio
from rasterio.features import geometry_mask
from rasterio.mask import mask
except ImportError as exc:
raise AppError(
backend/app/services/thematic_raster_acquisition_service.py
+1 -2
View File
@@ -11,7 +11,7 @@ from pathlib import Path
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import urlencode
from urllib.request import Request, urlopen
from urllib.request import Request
from uuid import UUID
from geoalchemy2.shape import to_shape
@@ -407,7 +407,6 @@ class ThematicRasterAcquisitionService:
if len(coverages) == 1:
return coverages[0]
try:
import rasterio
from rasterio.io import MemoryFile
from rasterio.merge import merge
except ImportError as exc:
backend/app/services/thematic_raster_analysis_service.py
-1
View File
@@ -96,7 +96,6 @@ class ThematicRasterAnalysisService:
try:
import numpy as np
import rasterio
from rasterio.features import geometry_mask
from rasterio.mask import mask
except ImportError as exc:
raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for thematic raster analysis", status_code=503) from exc
backend/app/services/tile_manifest_service.py
+485
View File
@@ -0,0 +1,485 @@
from __future__ import annotations
from hashlib import sha256
import json
from math import isfinite
from pathlib import Path
from typing import Any
from geoalchemy2.shape import to_shape
from pyproj import CRS, Transformer
from shapely.geometry import box
from shapely.ops import transform as shapely_transform
from shapely.ops import unary_union
from app.core.errors import AppError
from app.models import Area, Dataset, DatasetVersion
from app.services.storage_service import StorageService
class TileManifestService:
"""Versioned provenance and integrity contract for inference tile sets."""
CONTRACT_KEY = "geointel.raster.tile-manifest"
CONTRACT_VERSION = "2.0.0"
_BINDING_FIELDS = (
"source_dataset_id",
"source_dataset_checksum_sha256",
"source_dataset_size_bytes",
"source_registry_id",
"source_snapshot_id",
"source_snapshot_checksum_sha256",
"data_contract_key",
"data_contract_version",
"source_version",
"dataset_version_id",
"dataset_version",
"dataset_version_checksum_sha256",
"source_area_id",
"source_area_geometry_sha256",
)
_REQUIRED_INFERENCE_BINDING_FIELDS = (
"source_dataset_checksum_sha256",
"source_registry_id",
"source_snapshot_id",
"source_snapshot_checksum_sha256",
"data_contract_key",
"data_contract_version",
"dataset_version_id",
"dataset_version",
"dataset_version_checksum_sha256",
)
_CHECKSUM_FIELDS = (
"source_dataset_checksum_sha256",
"source_snapshot_checksum_sha256",
"dataset_version_checksum_sha256",
)
@staticmethod
def file_sha256(path: str | Path) -> str:
digest = sha256()
with Path(path).open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
@staticmethod
def _latest_dataset_version(dataset: Dataset) -> DatasetVersion | None:
versions = list(dataset.versions or [])
if not versions:
return None
return max(versions, key=lambda item: (int(item.version or 0), str(item.id or "")))
@staticmethod
def _source_snapshot_checksum(dataset: Dataset) -> str | None:
snapshot = dataset.source_snapshot
checksum = getattr(snapshot, "checksum_sha256", None) if snapshot is not None else None
return str(checksum).lower() if checksum else None
@staticmethod
def _area_geometry_binding(db, dataset: Dataset) -> tuple[str | None, str | None]:
if dataset.area_id is None:
return None, None
area = db.get(Area, dataset.area_id)
if area is None or area.geometry is None:
return str(dataset.area_id), None
geometry = to_shape(area.geometry)
return str(dataset.area_id), sha256(geometry.wkb).hexdigest()
@classmethod
def dataset_binding(cls, db, dataset: Dataset) -> dict[str, Any]:
version = cls._latest_dataset_version(dataset)
area_id, area_geometry_sha256 = cls._area_geometry_binding(db, dataset)
return {
"manifest_contract_key": cls.CONTRACT_KEY,
"manifest_contract_version": cls.CONTRACT_VERSION,
"source_dataset_id": str(dataset.id),
"source_raster_id": str(dataset.id),
"source_dataset_checksum_sha256": (
str(dataset.checksum_sha256).lower() if dataset.checksum_sha256 else None
),
"source_dataset_size_bytes": dataset.size_bytes,
"source_registry_id": str(dataset.source_registry_id) if dataset.source_registry_id else None,
"source_snapshot_id": str(dataset.source_snapshot_id) if dataset.source_snapshot_id else None,
"source_snapshot_checksum_sha256": cls._source_snapshot_checksum(dataset),
"data_contract_key": dataset.data_contract_key,
"data_contract_version": dataset.data_contract_version,
"source_version": dataset.source_version,
"dataset_version_id": str(version.id) if version is not None and version.id else None,
"dataset_version": int(version.version) if version is not None and version.version is not None else None,
"dataset_version_checksum_sha256": (
str(version.checksum_sha256).lower()
if version is not None and version.checksum_sha256
else None
),
"source_area_id": area_id,
"source_area_geometry_sha256": area_geometry_sha256,
}
@staticmethod
def tile_integrity(path: str | Path) -> dict[str, Any]:
resolved = Path(path)
return {
"size_bytes": resolved.stat().st_size,
"sha256": TileManifestService.file_sha256(resolved),
}
@staticmethod
def _error(
error_prefix: str,
suffix: str,
message: str,
*,
details: dict[str, Any] | None = None,
) -> AppError:
return AppError(
code=f"{error_prefix}_TILE_MANIFEST_{suffix}",
message=message,
details=details,
status_code=422,
)
@staticmethod
def _bounds_values(value: Any) -> tuple[float, float, float, float] | None:
if isinstance(value, dict):
aliases = (
("min_x", "min_y", "max_x", "max_y"),
("minx", "miny", "maxx", "maxy"),
("left", "bottom", "right", "top"),
)
selected = next(
([value.get(key) for key in keys] for keys in aliases if all(key in value for key in keys)),
None,
)
elif isinstance(value, (list, tuple)) and len(value) == 4:
selected = list(value)
else:
return None
try:
bounds = tuple(float(item) for item in selected) if selected is not None else None
except (TypeError, ValueError):
return None
if bounds is None or not all(isfinite(item) for item in bounds):
return None
if bounds[0] >= bounds[2] or bounds[1] >= bounds[3]:
return None
return bounds
@staticmethod
def _to_epsg4326(bounds: tuple[float, float, float, float], raw_crs: Any):
source_crs = CRS.from_user_input(raw_crs)
geometry = box(*bounds)
if not source_crs.equals(CRS.from_epsg(4326)):
transformer = Transformer.from_crs(source_crs, "EPSG:4326", always_xy=True)
geometry = shapely_transform(transformer.transform, geometry)
if geometry.is_empty or not geometry.is_valid:
raise ValueError("Bounds do not form a valid transformed geometry")
if not all(isfinite(float(value)) for value in geometry.bounds):
raise ValueError("Bounds transform to non-finite coordinates")
return geometry
@classmethod
def _manifest_coverage(cls, manifest: dict[str, Any], *, error_prefix: str):
tiles = manifest.get("tiles")
default_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs")
parts = []
for index, tile in enumerate(tiles if isinstance(tiles, list) else []):
if not isinstance(tile, dict):
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Tile manifest entries must be objects with explicit spatial metadata.",
details={"tile_index": index},
)
bounds = cls._bounds_values(tile.get("bounds"))
raw_crs = tile.get("crs") or default_crs
if bounds is None or not raw_crs:
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Every inference tile requires finite bounds and an explicit CRS.",
details={"tile_index": index},
)
try:
parts.append(cls._to_epsg4326(bounds, raw_crs))
except Exception as exc:
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Inference tile bounds or CRS could not be normalized to EPSG:4326.",
details={"tile_index": index, "reason": str(exc)},
) from exc
coverage = unary_union(parts)
if coverage.is_empty or not coverage.is_valid:
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Inference tile union is empty or invalid.",
)
min_x, min_y, max_x, max_y = coverage.bounds
if min_x < -180 or min_y < -90 or max_x > 180 or max_y > 90:
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Inference tile union falls outside EPSG:4326 bounds.",
details={"bounds": list(coverage.bounds)},
)
return coverage
@classmethod
def _validate_binding(cls, db, dataset: Dataset, manifest: dict[str, Any], *, error_prefix: str) -> dict[str, Any]:
if (
manifest.get("manifest_contract_key") != cls.CONTRACT_KEY
or manifest.get("manifest_contract_version") != cls.CONTRACT_VERSION
):
raise cls._error(
error_prefix,
"PROVENANCE_MISMATCH",
"Inference requires a versioned GeoIntel tile-manifest contract.",
details={
"required_contract": f"{cls.CONTRACT_KEY}@{cls.CONTRACT_VERSION}",
"manifest_contract": (
f"{manifest.get('manifest_contract_key')}@{manifest.get('manifest_contract_version')}"
),
},
)
expected = cls.dataset_binding(db, dataset)
missing = [
field
for field in cls._REQUIRED_INFERENCE_BINDING_FIELDS
if expected.get(field) in {None, ""}
]
invalid_checksums = [
field
for field in cls._CHECKSUM_FIELDS
if len(str(expected.get(field) or "")) != 64
or any(character not in "0123456789abcdef" for character in str(expected.get(field) or "").lower())
]
if missing or invalid_checksums:
raise cls._error(
error_prefix,
"PROVENANCE_MISMATCH",
"The requested Dataset lacks complete immutable provenance for inference tiling.",
details={
"missing_fields": missing,
"invalid_checksum_fields": invalid_checksums,
},
)
manifest_dataset_id = manifest.get("source_dataset_id") or manifest.get("source_raster_id")
if str(manifest_dataset_id or "") != expected["source_dataset_id"]:
raise cls._error(
error_prefix,
"DATASET_MISMATCH",
"Tile manifest belongs to a different raster Dataset.",
details={
"requested_dataset_id": expected["source_dataset_id"],
"manifest_dataset_id": manifest_dataset_id,
},
)
mismatches = {}
for field in cls._BINDING_FIELDS:
expected_value = expected.get(field)
if expected_value is None or field == "source_dataset_id":
continue
observed_value = manifest.get(field)
if str(observed_value) != str(expected_value):
mismatches[field] = {"expected": expected_value, "observed": observed_value}
if mismatches:
raise cls._error(
error_prefix,
"PROVENANCE_MISMATCH",
"Tile manifest provenance no longer matches the requested Dataset snapshot.",
details={"mismatches": mismatches},
)
return expected
@classmethod
def _validate_tile_files(
cls,
manifest: dict[str, Any],
manifest_path: Path,
*,
settings,
error_prefix: str,
) -> list[str]:
resolved_paths: list[str] = []
seen_paths: set[Path] = set()
for index, tile in enumerate(manifest["tiles"]):
raw_path = tile.get("path") if isinstance(tile, dict) else None
if not isinstance(raw_path, str) or not raw_path.strip():
raise cls._error(
error_prefix,
"TILE_INTEGRITY_MISMATCH",
"Every inference tile requires a path and immutable integrity evidence.",
details={"tile_index": index},
)
candidate = Path(raw_path).expanduser()
if not candidate.is_absolute():
candidate = manifest_path.parent / candidate
candidate = StorageService.assert_within_storage_root(
candidate,
label="raster tile",
settings=settings,
)
if not candidate.is_file():
raise cls._error(
error_prefix,
"TILE_INTEGRITY_MISMATCH",
"An inference tile referenced by the manifest does not exist.",
details={"tile_index": index, "tile_path": str(candidate)},
)
if candidate in seen_paths:
raise cls._error(
error_prefix,
"TILE_INTEGRITY_MISMATCH",
"A tile path occurs more than once in the inference manifest.",
details={"tile_index": index, "tile_path": str(candidate)},
)
seen_paths.add(candidate)
observed_size = candidate.stat().st_size
expected_size = tile.get("size_bytes")
expected_checksum = str(tile.get("sha256") or "").strip().lower()
if expected_size != observed_size or len(expected_checksum) != 64:
raise cls._error(
error_prefix,
"TILE_INTEGRITY_MISMATCH",
"Tile size/checksum evidence is missing or no longer matches the staged file.",
details={
"tile_index": index,
"expected_size_bytes": expected_size,
"observed_size_bytes": observed_size,
},
)
observed_checksum = cls.file_sha256(candidate)
if observed_checksum != expected_checksum:
raise cls._error(
error_prefix,
"TILE_INTEGRITY_MISMATCH",
"Tile checksum no longer matches the immutable manifest evidence.",
details={
"tile_index": index,
"expected_sha256": expected_checksum,
"observed_sha256": observed_checksum,
},
)
resolved_paths.append(str(candidate))
declared_count = manifest.get("count")
if declared_count != len(resolved_paths):
raise cls._error(
error_prefix,
"TILE_INTEGRITY_MISMATCH",
"Tile manifest count does not match its tile records.",
details={"declared_count": declared_count, "tile_count": len(resolved_paths)},
)
return resolved_paths
@classmethod
def _validate_scope(
cls,
db,
dataset: Dataset,
manifest: dict[str, Any],
coverage,
*,
error_prefix: str,
) -> None:
manifest_bounds = cls._bounds_values(manifest.get("bounds"))
manifest_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs")
dataset_bounds = cls._bounds_values(dataset.bounds_json)
if dataset_bounds is None and isinstance(dataset.metadata_json, dict):
dataset_bounds = cls._bounds_values(
dataset.metadata_json.get("bounds_json") or dataset.metadata_json.get("bounds")
)
if manifest_bounds is None or not manifest_crs or dataset_bounds is None or not dataset.crs:
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Dataset and tile manifest require explicit CRS and finite bounds for inference.",
)
try:
manifest_extent = cls._to_epsg4326(manifest_bounds, manifest_crs)
dataset_extent = cls._to_epsg4326(dataset_bounds, dataset.crs)
except Exception as exc:
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Dataset or manifest bounds could not be normalized to EPSG:4326.",
details={"reason": str(exc)},
) from exc
tolerance = max(dataset_extent.bounds[2] - dataset_extent.bounds[0], dataset_extent.bounds[3] - dataset_extent.bounds[1]) * 1e-7 + 1e-10
if not manifest_extent.buffer(tolerance).covers(coverage):
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Tile union exceeds the extent declared by its manifest.",
details={"tile_union_bounds": list(coverage.bounds), "manifest_bounds": list(manifest_extent.bounds)},
)
if not dataset_extent.buffer(tolerance).covers(coverage):
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Tile union exceeds the persisted Dataset extent.",
details={"tile_union_bounds": list(coverage.bounds), "dataset_bounds": list(dataset_extent.bounds)},
)
if dataset.area_id is not None:
area = db.get(Area, dataset.area_id)
if area is None or area.geometry is None:
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Dataset references an Area that is unavailable for inference-scope validation.",
details={"area_id": str(dataset.area_id)},
)
area_geometry = to_shape(area.geometry)
if area_geometry.is_empty or not area_geometry.is_valid or not coverage.intersects(area_geometry):
raise cls._error(
error_prefix,
"SCOPE_MISMATCH",
"Tile union does not overlap the persisted Dataset Area.",
details={"area_id": str(dataset.area_id), "tile_union_bounds": list(coverage.bounds)},
)
@classmethod
def validate_for_inference(
cls,
db,
dataset: Dataset,
manifest: dict[str, Any],
*,
manifest_path: str | Path,
settings,
error_prefix: str,
) -> dict[str, Any]:
resolved_manifest_path = StorageService.assert_within_storage_root(
manifest_path,
label="tile manifest",
settings=settings,
)
expected = cls._validate_binding(db, dataset, manifest, error_prefix=error_prefix)
resolved_paths = cls._validate_tile_files(
manifest,
resolved_manifest_path,
settings=settings,
error_prefix=error_prefix,
)
coverage = cls._manifest_coverage(manifest, error_prefix=error_prefix)
cls._validate_scope(db, dataset, manifest, coverage, error_prefix=error_prefix)
return {
"manifest_contract_key": cls.CONTRACT_KEY,
"manifest_contract_version": cls.CONTRACT_VERSION,
"manifest_path": str(resolved_manifest_path),
"manifest_sha256": cls.file_sha256(resolved_manifest_path),
"source_dataset_id": expected["source_dataset_id"],
"source_dataset_checksum_sha256": expected.get("source_dataset_checksum_sha256"),
"source_snapshot_id": expected.get("source_snapshot_id"),
"dataset_version_id": expected.get("dataset_version_id"),
"source_area_id": expected.get("source_area_id"),
"tile_count": len(resolved_paths),
"tile_union_bounds_epsg4326": [float(value) for value in coverage.bounds],
}
def canonical_manifest_json(payload: dict[str, Any]) -> str:
"""Stable serializer shared by the writer and manifest-hash tests."""
return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True)
backend/app/utils/geometry.py
+82 -21
View File
@@ -1,43 +1,101 @@
from __future__ import annotations
from math import isfinite
from numbers import Real
from typing import Any
from pyproj import Transformer
from shapely import force_2d
from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, box, shape
from pyproj import CRS, Transformer
from shapely import force_2d, get_coordinates
from shapely.geometry import MultiPolygon, box, shape
from shapely.ops import transform
from shapely.validation import make_valid
# This is a deliberately broad guard envelope around Belgium and the Belgian
# North Sea. Exact legal/regional clipping remains the responsibility of the
# persisted coverage Areas; this boundary prevents an AOI with valid-looking
# but globally misplaced coordinates from entering the workbench.
BELGIUM_AND_NORTH_SEA_GUARD_BOUNDS = (1.5, 48.5, 7.5, 52.5)
def _raw_coordinates_are_finite(value: Any) -> bool:
if isinstance(value, (list, tuple)):
return bool(value) and all(_raw_coordinates_are_finite(item) for item in value)
return isinstance(value, Real) and not isinstance(value, bool) and isfinite(float(value))
def normalize_to_multipolygon(raw_geometry: dict[str, Any]) -> MultiPolygon:
geom = force_2d(shape(raw_geometry))
if isinstance(raw_geometry, dict) and "coordinates" in raw_geometry:
if not _raw_coordinates_are_finite(raw_geometry["coordinates"]):
raise ValueError("Geometry coordinates must be finite numbers")
try:
geom = force_2d(shape(raw_geometry))
except Exception as exc:
raise ValueError("Geometry is not valid GeoJSON") from exc
coordinates = get_coordinates(geom, include_z=False)
if coordinates.size == 0 or not all(isfinite(float(value)) for row in coordinates for value in row):
raise ValueError("Geometry coordinates must be finite numbers")
if geom.is_empty:
raise ValueError("Geometry is empty")
if not geom.is_valid:
geom = make_valid(geom)
if not geom.is_valid:
raise ValueError("Geometry is invalid and could not be repaired")
raise ValueError("Geometry is invalid")
if geom.geom_type == "Polygon":
return MultiPolygon([geom])
if geom.geom_type == "MultiPolygon":
return MultiPolygon(geom.geoms)
if isinstance(geom, GeometryCollection):
polygons = [g for g in geom.geoms if isinstance(g, Polygon)]
multipolygons = [g for g in geom.geoms if g.geom_type == "MultiPolygon"]
if not polygons and not multipolygons:
raise ValueError("Only polygon geometries are supported for AOI")
normalized = []
normalized.extend(polygons)
for mp in multipolygons:
normalized.extend(mp.geoms)
return MultiPolygon(normalized)
raise ValueError("Only Polygon or MultiPolygon geometries are accepted")
def normalize_area_to_epsg4326(
raw_geometry: dict[str, Any],
source_crs: str,
) -> tuple[MultiPolygon, str]:
"""Validate an AOI and normalize its declared CRS to canonical WGS84.
The returned CRS string preserves the caller's declaration for provenance;
the returned geometry is always finite, polygonal and stored as EPSG:4326.
"""
declared_crs = str(source_crs or "").strip()
if not declared_crs:
raise ValueError("Area CRS is required")
try:
parsed_crs = CRS.from_user_input(declared_crs)
except Exception as exc:
raise ValueError("Area CRS is unknown or invalid") from exc
if not (parsed_crs.is_geographic or parsed_crs.is_projected):
raise ValueError("Area CRS must be a geographic or projected two-dimensional CRS")
if len(parsed_crs.axis_info) != 2:
raise ValueError("Area CRS must have exactly two spatial axes")
geometry = normalize_to_multipolygon(raw_geometry)
target_crs = CRS.from_epsg(4326)
if not parsed_crs.equals(target_crs):
try:
transformer = Transformer.from_crs(parsed_crs, target_crs, always_xy=True)
geometry = normalize_to_multipolygon(
transform(transformer.transform, geometry).__geo_interface__
)
except ValueError:
raise
except Exception as exc:
raise ValueError("Area geometry could not be transformed to EPSG:4326") from exc
min_x, min_y, max_x, max_y = geometry.bounds
if not all(isfinite(value) for value in (min_x, min_y, max_x, max_y)):
raise ValueError("Transformed area geometry contains non-finite coordinates")
world_bounds = (-180.0, -90.0, 180.0, 90.0)
if min_x < world_bounds[0] or min_y < world_bounds[1] or max_x > world_bounds[2] or max_y > world_bounds[3]:
raise ValueError("Transformed area geometry falls outside the EPSG:4326 coordinate domain")
guard = box(*BELGIUM_AND_NORTH_SEA_GUARD_BOUNDS)
if not guard.intersects(geometry):
raise ValueError("Area geometry falls outside Belgium and the Belgian North Sea workbench domain")
if not guard.covers(geometry):
raise ValueError("Area geometry must remain within the Belgium and Belgian North Sea workbench domain")
return geometry, declared_crs
def area_bounds_multipolygon(geom: MultiPolygon):
return {
"min_x": float(geom.bounds[0]),
@@ -52,7 +110,10 @@ def area_m2(geom: MultiPolygon) -> float:
Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True).transform,
geom,
)
return float(projected.area)
result = float(projected.area)
if not isfinite(result) or result <= 0:
raise ValueError("Area geometry must have a finite positive surface")
return result
def geometry_bbox_polygon(geom: MultiPolygon):
backend/scripts/cleanup_demo_artifacts.py
+7 -4
View File
@@ -14,10 +14,13 @@ SCRIPTS_ROOT = REPOSITORY_ROOT / "scripts"
if SCRIPTS_ROOT.is_dir():
sys.path.insert(0, str(SCRIPTS_ROOT))
from app.core.config import get_settings
from app.db.session import SessionLocal
from app.models import Export, Project
from release_backup_guard import require_confirmation, verify_current_backup
from app.core.config import get_settings # noqa: E402 - imported after backend path bootstrap
from app.db.session import SessionLocal # noqa: E402 - imported after backend path bootstrap
from app.models import Export, Project # noqa: E402 - imported after backend path bootstrap
from release_backup_guard import ( # noqa: E402 - imported after scripts path bootstrap
require_confirmation,
verify_current_backup,
)
DEMO_PROJECT_NAME = "GeoIntel Demo - Building QA"
backend/tests/test_analysis_job_queue.py
+1 -1
View File
@@ -14,7 +14,7 @@ from uuid import uuid4
import pytest
from app.core.errors import AppError
from app.models import AnalysisRun, Detection, Job
from app.models import Job
from app.services.analysis_job_worker import AnalysisJobWorker
from app.services.detection_service import DetectionService
backend/tests/test_area_crs_semantics.py
+159
View File
@@ -0,0 +1,159 @@
from __future__ import annotations
from uuid import uuid4
from geoalchemy2.shape import from_shape, to_shape
from pyproj import Transformer
import pytest
from shapely.geometry import Polygon, mapping
from shapely.ops import transform
from app.core.errors import AppError
from app.models import Area, Project
from app.schemas.area import AreaCreate, AreaUpdate
from app.services.area_service import AreaService
from app.utils.geometry import area_m2, normalize_area_to_epsg4326
class FakeSession:
def __init__(self, objects=None) -> None:
self.objects = objects or {}
self.added = []
self.commits = 0
self.refreshes = []
def get(self, model, item_id):
return self.objects.get((model, item_id))
def add(self, item) -> None:
self.added.append(item)
def commit(self) -> None:
self.commits += 1
def refresh(self, item) -> None:
self.refreshes.append(item)
def _wgs84_polygon(offset: float = 0.0) -> Polygon:
return Polygon(
[
(5.00 + offset, 51.00),
(5.01 + offset, 51.00),
(5.01 + offset, 51.01),
(5.00 + offset, 51.01),
(5.00 + offset, 51.00),
]
)
def _to_lambert(geometry: Polygon) -> Polygon:
transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
return transform(transformer.transform, geometry)
def test_create_area_transforms_declared_lambert_geometry_before_storage() -> None:
project_id = uuid4()
db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")})
source = _wgs84_polygon()
area = AreaService.create_area(
db,
project_id,
AreaCreate(name="Lambert AOI", geometry=mapping(_to_lambert(source)), crs="EPSG:31370"),
)
stored = to_shape(area.geometry)
assert stored.bounds == pytest.approx(source.bounds, abs=1e-7)
assert area.original_crs == "EPSG:31370"
assert area.area_m2 == pytest.approx(area_m2(normalize_area_to_epsg4326(mapping(source), "EPSG:4326")[0]))
assert area.area_m2 and area.area_m2 > 0
assert to_shape(area.bbox).bounds == pytest.approx(source.bounds, abs=1e-7)
def test_patch_area_replaces_geometry_and_recomputes_all_spatial_fields() -> None:
area_id = uuid4()
project_id = uuid4()
original = _wgs84_polygon()
normalized, _ = normalize_area_to_epsg4326(mapping(original), "EPSG:4326")
area = Area(
id=area_id,
project_id=project_id,
name="Original",
geometry=from_shape(normalized, srid=4326),
bbox=from_shape(normalized.envelope, srid=4326),
original_crs="EPSG:4326",
area_m2=area_m2(normalized),
)
db = FakeSession({(Area, area_id): area})
replacement = _wgs84_polygon(offset=0.05)
updated = AreaService.update_area(
db,
area_id,
AreaUpdate(
name="Replacement",
geometry=mapping(_to_lambert(replacement)),
crs="EPSG:31370",
),
)
assert updated.name == "Replacement"
assert updated.original_crs == "EPSG:31370"
assert to_shape(updated.geometry).bounds == pytest.approx(replacement.bounds, abs=1e-7)
assert to_shape(updated.bbox).bounds == pytest.approx(replacement.bounds, abs=1e-7)
assert updated.area_m2 and updated.area_m2 > 0
assert db.commits == 1
@pytest.mark.parametrize(
("geometry", "crs", "message_fragment"),
[
(mapping(_wgs84_polygon()), "EPSG:not-real", "unknown or invalid"),
(mapping(_wgs84_polygon()), "EPSG:4979", "exactly two spatial axes"),
(
{
"type": "Polygon",
"coordinates": [[[5.0, 51.0], [float("nan"), 51.0], [5.1, 51.1], [5.0, 51.0]]],
},
"EPSG:4326",
"finite",
),
(mapping(Polygon([(10.0, 51.0), (10.1, 51.0), (10.1, 51.1), (10.0, 51.0)])), "EPSG:4326", "workbench domain"),
(
{
"type": "Polygon",
"coordinates": [[[5.0, 51.0], [5.1, 51.1], [5.1, 51.0], [5.0, 51.1], [5.0, 51.0]]],
},
"EPSG:4326",
"invalid",
),
({"type": "Point", "coordinates": [5.0, 51.0]}, "EPSG:4326", "Polygon or MultiPolygon"),
],
)
def test_create_area_rejects_invalid_crs_nonfinite_and_out_of_domain_geometry(
geometry: dict,
crs: str,
message_fragment: str,
) -> None:
project_id = uuid4()
db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")})
with pytest.raises(AppError) as exc_info:
AreaService.create_area(db, project_id, AreaCreate(name="Invalid", geometry=geometry, crs=crs))
assert exc_info.value.code == "INVALID_GEOMETRY"
assert message_fragment in exc_info.value.message
assert db.commits == 0
def test_patch_area_rejects_crs_without_replacement_geometry() -> None:
area_id = uuid4()
area = Area(id=area_id, project_id=uuid4(), name="AOI", original_crs="EPSG:4326")
db = FakeSession({(Area, area_id): area})
with pytest.raises(AppError) as exc_info:
AreaService.update_area(db, area_id, AreaUpdate(crs="EPSG:31370"))
assert exc_info.value.code == "INVALID_AREA_CRS_UPDATE"
assert db.commits == 0
backend/tests/test_dataset_consumption_gate.py
+11
View File
@@ -190,6 +190,17 @@ def test_passed_manual_or_experimental_dataset_cannot_cross_production_boundary(
assert "experimental_source_not_allowed_for_purpose" in exc_info.value.details["reasons"]
def test_fully_governed_demo_fixture_still_cannot_enter_production_inference() -> None:
fixture = _governed_dataset(source_key="fixture", classification="experimental")
fixture.source_metadata = {"fixture": True, "usage": "offline demo raster workflow only"}
with pytest.raises(AppError) as exc_info:
DatasetConsumptionGate.assert_eligible(fixture, purpose="production_inference")
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert "experimental_source_not_allowed_for_purpose" in exc_info.value.details["reasons"]
def test_reference_validation_requires_authoritative_ground_truth_reference() -> None:
reference = _governed_dataset()
reference.dataset_type = "vector"
backend/tests/test_flood_hazard_selection_data_coverage.py
+3 -1
View File
@@ -17,7 +17,9 @@ import pytest
np = pytest.importorskip("numpy")
from app.services.flood_hazard_analysis_service import FloodHazardCellStatistics
from app.services.flood_hazard_analysis_service import ( # noqa: E402 - optional NumPy gate precedes service import
FloodHazardCellStatistics,
)
NODATA = -9999.0
backend/tests/test_model_asset_catalog.py
+33 -5
View File
@@ -11,10 +11,20 @@ from fastapi.testclient import TestClient
from app.core.config import Settings
from app.core.errors import AppError
from app.main import app
from app.models import AnalysisRun, Dataset, Detection, Job, Project, SourceRegistry, SourceSnapshot
from app.models import (
AnalysisRun,
Dataset,
DatasetVersion,
Detection,
Job,
Project,
SourceRegistry,
SourceSnapshot,
)
from app.services.detection_service import DetectionService
from app.services.model_asset_catalog_service import ModelAssetCatalogService
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
from app.services.tile_manifest_service import TileManifestService
class FakeSession:
@@ -98,6 +108,8 @@ def _project_and_raster_dataset():
source_name="test-derived-raster",
storage_path="storage/uploads/source.tif",
checksum_sha256=checksum,
crs="EPSG:4326",
bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0},
source_registry_id=source_registry_id,
source_snapshot_id=source_snapshot_id,
data_contract_key="geointel.raster.geotiff",
@@ -110,17 +122,22 @@ def _project_and_raster_dataset():
)
dataset.source_registry = source_registry
dataset.source_snapshot = source_snapshot
dataset.versions.append(
DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1, checksum_sha256=checksum)
)
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
return db, project_id, dataset_id
def _manifest(tmp_path: Path) -> Path:
def _manifest(tmp_path: Path, db: FakeSession, dataset: Dataset) -> Path:
tile_path = tmp_path / "tile_0000.tif"
tile_path.write_bytes(b"tile")
binding = TileManifestService.dataset_binding(db, dataset)
manifest_path = tmp_path / "manifest.json"
manifest_path.write_text(
json.dumps(
{
**binding,
"tile_set_id": "tiles-fixture",
"count": 1,
"crs": "EPSG:4326",
@@ -133,6 +150,7 @@ def _manifest(tmp_path: Path) -> Path:
"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01],
"crs": "EPSG:4326",
"index": 0,
**TileManifestService.tile_integrity(tile_path),
}
],
}
@@ -226,7 +244,11 @@ def test_model_asset_catalog_lists_supported_local_model_files(tmp_path: Path) -
assert asset.size_bytes == len(b"local model")
assert len(asset.sha256) == 64
assert asset.active is True
assert asset.status == "approved"
assert asset.runtime_available is True
assert asset.runtime_status == "active"
assert asset.governed_validation_status == "not_verified_by_catalog"
assert asset.promotion_status == "not_verified_by_catalog"
assert asset.status == "runtime_active"
assert asset.will_download_models is False
@@ -257,7 +279,10 @@ def test_model_asset_catalog_only_exposes_explicit_active_asset_in_runtime(tmp_p
assert response.total == 1
assert response.items[0].filename == active_file.name
assert response.items[0].active is True
assert response.items[0].status == "approved"
assert response.items[0].runtime_status == "active"
assert response.items[0].governed_validation_status == "not_verified_by_catalog"
assert response.items[0].promotion_status == "not_verified_by_catalog"
assert response.items[0].status == "runtime_active"
def test_model_asset_catalog_rejects_unknown_asset(tmp_path: Path) -> None:
@@ -284,6 +309,9 @@ def test_model_assets_api_returns_canonical_envelope(monkeypatch, tmp_path: Path
assert payload["data"]["total"] == 1
assert payload["data"]["items"][0]["model_asset_id"] == "building-detector-pt"
assert payload["data"]["items"][0]["active"] is True
assert payload["data"]["items"][0]["runtime_status"] == "active"
assert payload["data"]["items"][0]["governed_validation_status"] == "not_verified_by_catalog"
assert payload["data"]["items"][0]["promotion_status"] == "not_verified_by_catalog"
assert payload["data"]["items"][0]["will_download_models"] is False
@@ -309,7 +337,7 @@ def test_detection_run_persists_selected_model_asset_parameters(tmp_path, monkey
model_id="yolo-configured",
model_asset_id="building-detector-pt",
confidence_threshold=0.5,
tile_manifest_path=str(_manifest(tmp_path)),
tile_manifest_path=str(_manifest(tmp_path, db, db.get(Dataset, dataset_id))),
settings=settings,
yolo_adapter_class=MockYoloAdapter,
)
backend/tests/test_raster_cell_selection.py
+3 -3
View File
@@ -18,10 +18,10 @@ import pytest
np = pytest.importorskip("numpy")
rasterio = pytest.importorskip("rasterio")
from rasterio.transform import from_origin
from shapely.geometry import box
from rasterio.transform import from_origin # noqa: E402 - optional rasterio gate precedes imports
from shapely.geometry import box # noqa: E402 - optional rasterio gate precedes imports
from app.services.raster_cell_selection import select_cells
from app.services.raster_cell_selection import select_cells # noqa: E402 - optional rasterio gate precedes service import
# 100 m cells, origin at the top-left corner of a 3x3 grid.
backend/tests/test_raster_operations_service.py
+101 -2
View File
@@ -1,14 +1,16 @@
from __future__ import annotations
from types import ModuleType, SimpleNamespace
from types import SimpleNamespace
from uuid import uuid4
from pathlib import Path
import importlib
from hashlib import sha256
from geoalchemy2.shape import from_shape
from app.core.errors import AppError
from app.models import Area, Dataset, DatasetVersion
from app.services.raster_operations_service import RasterOperationsService
from app.services.storage_service import StorageService
from app.api.routes.datasets import _run_job_sync
from shapely.geometry import box
import pytest
@@ -688,6 +690,104 @@ def test_raster_tile_returns_manifest_payload(monkeypatch, tmp_path) -> None:
assert payload["manifest"]["tile_server"] is None
@pytest.mark.parametrize(
("dimension", "expected"),
[
(512, [0]),
(513, [0, 1]),
(960, [0, 448]),
(961, [0, 448, 449]),
],
)
def test_raster_tile_offsets_use_full_tiles_and_one_unique_edge_start(dimension, expected) -> None:
assert RasterOperationsService._tile_offsets(dimension, tile_size=512, step=448) == expected
def test_raster_tile_rejects_limit_before_creating_output(monkeypatch, tmp_path) -> None:
project_id = uuid4()
dataset_id = uuid4()
source = tmp_path / "large-raster.tif"
source.write_bytes(b"source")
dataset = Dataset(
id=dataset_id,
project_id=project_id,
name="large-raster.tif",
dataset_type="raster",
source="user_upload",
storage_path=str(source),
original_filename="large-raster.tif",
stored_filename="large-raster.tif",
content_type="image/tiff",
size_bytes=6,
)
db = FakeSession([dataset])
class FakeSource:
width = 2048
height = 2048
count = 1
crs = None
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return None
fake_rasterio = SimpleNamespace(open=lambda _path: FakeSource())
monkeypatch.setattr(
"app.services.raster_operations_service._import_rasterio",
lambda: (fake_rasterio, SimpleNamespace()),
)
tile_root = tmp_path / "tiles-that-must-not-exist"
monkeypatch.setattr(
StorageService,
"raster_tiles_root",
staticmethod(lambda *_args: tile_root),
)
with pytest.raises(AppError) as error:
RasterOperationsService.tile(db, dataset_id, tile_size=512, overlap=64, max_tiles=1)
assert error.value.code == "RASTER_TILE_LIMIT_EXCEEDED"
assert error.value.details == {"expected_tile_count": 25, "max_tiles": 1}
assert not tile_root.exists()
def test_raster_tile_rejects_changed_source_bytes_before_creating_output(monkeypatch, tmp_path) -> None:
project_id = uuid4()
dataset_id = uuid4()
source = tmp_path / "changed-raster.tif"
source.write_bytes(b"changed")
dataset = Dataset(
id=dataset_id,
project_id=project_id,
name="changed-raster.tif",
dataset_type="raster",
source="user_upload",
storage_path=str(source),
original_filename="changed-raster.tif",
stored_filename="changed-raster.tif",
content_type="image/tiff",
size_bytes=7,
checksum_sha256=sha256(b"original").hexdigest(),
data_contract_key="raster.generic",
)
db = FakeSession([dataset])
tile_root = tmp_path / "tiles-that-must-not-exist"
monkeypatch.setattr(
StorageService,
"raster_tiles_root",
staticmethod(lambda *_args: tile_root),
)
with pytest.raises(AppError) as error:
RasterOperationsService.tile(db, dataset_id)
assert error.value.code == "DATASET_STORAGE_CHECKSUM_MISMATCH"
assert not tile_root.exists()
def test_raster_clip_persists_derived_dataset(monkeypatch, tmp_path) -> None:
project_id = uuid4()
dataset_id = uuid4()
@@ -1393,4 +1493,3 @@ def test_run_job_sync_serializes_index_job_output_dataset_id(monkeypatch) -> Non
assert result["job_type"] == "raster.ndvi"
assert result["output_dataset_id"] == str(output_dataset_id)
assert result["result_json"]["output_dataset_id"] == str(output_dataset_id)
backend/tests/test_segmentation_configured_models.py
+26 -10
View File
@@ -9,11 +9,12 @@ import pytest
from geoalchemy2.shape import to_shape
from app.core.config import Settings
from app.models import Dataset, Project, Segmentation, SourceRegistry, SourceSnapshot
from app.models import Dataset, DatasetVersion, Project, Segmentation, SourceRegistry, SourceSnapshot
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
from app.services.model_registry_service import ModelRegistryService
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
from app.services.segmentation_service import SegmentationService
from app.services.tile_manifest_service import TileManifestService
ROOT = Path(__file__).resolve().parents[2]
@@ -122,6 +123,8 @@ def _project_and_dataset(dataset_type: str = "raster"):
source_name="digitaal_vlaanderen_orthophoto",
storage_path="storage/uploads/ortho.tif",
checksum_sha256=checksum,
crs="EPSG:4326",
bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0},
source_registry_id=source_id,
source_snapshot_id=snapshot_id,
data_contract_key="geointel.raster.geotiff",
@@ -134,6 +137,9 @@ def _project_and_dataset(dataset_type: str = "raster"):
)
dataset.source_registry = source
dataset.source_snapshot = snapshot
dataset.versions.append(
DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1, checksum_sha256=checksum)
)
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
return db, project_id, dataset_id
@@ -249,28 +255,36 @@ def _write_configured_model_sidecars(
)
def _manifest(tmp_path: Path, tile_count: int = 1) -> Path:
def _manifest(
tmp_path: Path,
tile_count: int = 1,
*,
db: FakeSession | None = None,
dataset: Dataset | None = None,
) -> Path:
tiles = []
for index in range(tile_count):
tile_path = tmp_path / f"tile_{index:04d}.tif"
tile_path.write_bytes(b"fixture")
tiles.append(
{
tile = {
"path": str(tile_path),
"pixel_window": [0, 0, 100, 100],
"bounds": [4.0, 51.0, 5.0, 52.0],
"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01],
"crs": "EPSG:4326",
"index": index,
**TileManifestService.tile_integrity(tile_path),
}
)
tiles.append(tile)
binding = TileManifestService.dataset_binding(db or FakeSession(), dataset) if dataset is not None else {}
manifest_path = tmp_path / "manifest.json"
manifest_path.write_text(
json.dumps(
{
**binding,
"tile_set_id": "tiles-fixture",
"source_dataset_id": str(uuid4()),
"source_raster_id": str(uuid4()),
"source_dataset_id": binding.get("source_dataset_id", str(uuid4())),
"source_raster_id": binding.get("source_raster_id", str(uuid4())),
"crs": "EPSG:4326",
"bounds": [4.0, 51.0, 5.0, 52.0],
"tile_size": 100,
@@ -424,7 +438,9 @@ def test_configured_segmentation_rejects_unbound_model_snapshot_before_adapter_l
dataset_id=dataset_id,
model_id="yolo-seg-configured",
confidence_threshold=0.5,
tile_manifest_path=str(_manifest(tmp_path)),
tile_manifest_path=str(
_manifest(tmp_path, db=db, dataset=db.get(Dataset, dataset_id))
),
settings=settings,
yolo_seg_adapter_class=NeverLoadSegAdapter,
sam_adapter_class=ClassAgnosticSamAdapter,
@@ -440,7 +456,7 @@ def test_configured_yolo_seg_run_persists_georeferenced_masks(tmp_path: Path) ->
db, project_id, dataset_id = _project_and_dataset()
settings = _settings(tmp_path)
_write_configured_model_sidecars(tmp_path, settings, include_sam=False, db=db)
manifest_path = _manifest(tmp_path)
manifest_path = _manifest(tmp_path, db=db, dataset=db.get(Dataset, dataset_id))
response = SegmentationService.run_segmentation(
db=db,
@@ -479,7 +495,7 @@ def test_configured_sam_run_is_class_agnostic(tmp_path: Path) -> None:
db, project_id, dataset_id = _project_and_dataset()
settings = _settings(tmp_path)
_write_configured_model_sidecars(tmp_path, settings, include_yolo=False, db=db)
manifest_path = _manifest(tmp_path)
manifest_path = _manifest(tmp_path, db=db, dataset=db.get(Dataset, dataset_id))
response = SegmentationService.run_segmentation(
db=db,
backend/tests/test_small_selection_raster_analysis.py
+11 -7
View File
@@ -15,14 +15,18 @@ import pytest
np = pytest.importorskip("numpy")
rasterio = pytest.importorskip("rasterio")
from pyproj import Transformer
from rasterio.transform import from_origin
from pyproj import Transformer # noqa: E402 - optional rasterio gate precedes geospatial imports
from rasterio.transform import from_origin # noqa: E402 - optional rasterio gate precedes geospatial imports
from app.core.config import Settings
from app.models import Dataset
from app.schemas.flood_hazard import FloodHazardSelectionRequest
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
from app.core.config import Settings # noqa: E402 - optional rasterio gate precedes app imports
from app.models import Dataset # noqa: E402 - optional rasterio gate precedes app imports
from app.schemas.flood_hazard import FloodHazardSelectionRequest # noqa: E402 - optional rasterio gate precedes app imports
from app.services.flood_hazard_acquisition_service import ( # noqa: E402 - optional rasterio gate precedes app imports
FloodHazardAcquisitionService,
)
from app.services.flood_hazard_analysis_service import ( # noqa: E402 - optional rasterio gate precedes app imports
FloodHazardAnalysisService,
)
TO_4326 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
backend/tests/test_sprint100_segmentation_manifest_handoff.py
+4 -2
View File
@@ -14,7 +14,10 @@ def test_raster_tile_manifest_can_handoff_to_segmentation_lab() -> None:
assert "segmentationTileManifestPath" in hook
assert "setSegmentationTileManifestPath" in hook
assert "tile_manifest_path: segmentationTileManifestPath.trim() || null" in hook
assert "let manifestPath = segmentationTileManifestPath.trim()" in hook
assert "datasetsApi.rasterInspect(projectId, datasetId)" in hook
assert "datasetsApi.rasterTile(projectId, datasetId" in hook
assert "tile_manifest_path: manifestPath" in hook
assert "segmentationTileManifestPath={segmentationTileManifestPath}" in app
assert "onSetTileManifestPath={setSegmentationTileManifestPath}" in app
assert "onUseTileManifestForSegmentation: useRasterTileManifestForSegmentation" in app
@@ -24,4 +27,3 @@ def test_raster_tile_manifest_can_handoff_to_segmentation_lab() -> None:
assert "Gebruik voor segmentatie" in raster_controls
assert "disabled={!latestRasterTileManifestPath}" in raster_controls
assert "Beeldtegelmanifest" in segmentation_lab
assert "Beeldtegelmanifest" in segmentation_lab
backend/tests/test_sprint106_map_bbox_extract.py
+1 -2
View File
@@ -1,6 +1,5 @@
from __future__ import annotations
import re
import uuid
from pathlib import Path
from types import SimpleNamespace
@@ -11,7 +10,7 @@ from shapely.geometry import Polygon, box
from app.core.errors import AppError
from app.models import Dataset, VectorFeature
from app.services.vector_feature_service import VectorFeatureService
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace, read_feature
from tests.frontend_contract import assert_calls, assert_wired, read_map_workspace, read_feature
ROOT = Path(__file__).resolve().parents[2]
backend/tests/test_sprint122_raster_upload_metadata_mapping.py
+13 -6
View File
@@ -12,8 +12,12 @@ class FakeUploadFile:
filename = "real-orthophoto.tif"
content_type = "image/tiff"
async def read(self) -> bytes:
return b"fake-raster"
def __init__(self) -> None:
self._content = b"fake-raster"
async def read(self, size: int) -> bytes:
chunk, self._content = self._content[:size], self._content[size:]
return chunk
class FakeSession:
@@ -40,16 +44,19 @@ def test_raster_upload_maps_metadata_bounds_resolution_and_bands(monkeypatch) ->
project_id = uuid4()
db = FakeSession(project_id)
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_dataset_file",
lambda **_: {
async def persist_upload_file(**_kwargs):
return {
"storage_path": "/tmp/real-orthophoto.tif",
"original_filename": "real-orthophoto.tif",
"stored_filename": "real-orthophoto.tif",
"content_type": "image/tiff",
"size_bytes": 11,
"checksum_sha256": "checksum",
},
}
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_upload_file",
persist_upload_file,
)
monkeypatch.setattr(
"app.services.dataset_service.extract_raster_metadata",
backend/tests/test_sprint186_map_first_geographic_explorer.py
+1 -2
View File
@@ -1,9 +1,8 @@
from __future__ import annotations
import re
from pathlib import Path
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace, read_feature
from tests.frontend_contract import assert_calls, assert_wired, read_map_workspace, read_feature
ROOT = Path(__file__).resolve().parents[2]
backend/tests/test_sprint192_regional_map_state.py
+1 -1
View File
@@ -1,5 +1,5 @@
from pathlib import Path
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace, read_feature
from tests.frontend_contract import assert_calls, read_map_workspace, read_feature
ROOT = Path(__file__).resolve().parents[2]
backend/tests/test_sprint194_regional_timeseries.py
+2 -1
View File
@@ -251,7 +251,8 @@ def test_end_user_dataset_sources_are_human_readable() -> None:
assert "statbel: 'Statbel'" in display
assert "getDatasetSourceDisplayName(activeThemeDataset)" in workspace
assert "getDatasetSourceDisplayName(resultDataset)" in workspace
assert "Zoek optioneel een gemeente" in workspace
assert 'aria-label="Optioneel een gemeente zoeken"' in workspace
assert 'placeholder="Gemeentenaam of NIS-code"' in workspace
assert "latestDatasetBySeries" in catalog
assert "Historische meetmomenten" in catalog
assert "getDatasetSourceDisplayName(dataset)" in catalog
backend/tests/test_sprint196_map_orthophoto_analysis.py
+4
View File
@@ -20,6 +20,7 @@ from app.db.session import get_db
from app.main import app
from app.models import Area, Dataset, DatasetVersion, Job, Project, SourceRegistry, SourceSnapshot
from app.schemas.orthophoto import OrthophotoAcquireRequest
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
from tests.frontend_contract import read_feature
@@ -262,6 +263,9 @@ def test_regional_orthophoto_products_bind_provider_and_governed_scope(
assert snapshot.checksum_sha256 == dataset.checksum_sha256
assert snapshot.ingest_status == "ingested"
assert snapshot.freshness_status == "current"
dataset.source_registry = source
dataset.source_snapshot = snapshot
assert DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference").eligible is True
prepared = OrthophotoAcquisitionService._prepared_request(_selection_payload(product_key="1971"), settings)
assert prepared["params"]["LAYERS"] == "OKZPAN71VL"
backend/tests/test_sprint200_temporal_explorer_handoff.py
+1 -1
View File
@@ -1,5 +1,5 @@
from pathlib import Path
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace
from tests.frontend_contract import assert_wired, read_map_workspace
ROOT = Path(__file__).resolve().parents[2]
backend/tests/test_sprint205_agricultural_parcel_history.py
-1
View File
@@ -1,7 +1,6 @@
from __future__ import annotations
import importlib.util
import json
import sys
import zipfile
from pathlib import Path
backend/tests/test_sprint213_thematic_rasters.py
-1
View File
@@ -7,7 +7,6 @@ from uuid import uuid4
import numpy as np
import pytest
import rasterio
from fastapi.testclient import TestClient
from pyproj import Transformer
from rasterio.io import MemoryFile
backend/tests/test_sprint223_governed_grb_refresh.py
-2
View File
@@ -252,7 +252,6 @@ def test_refresh_api_and_frontend_remain_explicit_only() -> None:
def test_map_theme_ranking_prefers_newer_observation_over_feature_count() -> None:
root = Path(__file__).resolve().parents[2]
workspace = read_map_workspace()
observed_sort = workspace.index("const observedAtDifference")
feature_tiebreaker = workspace.index("right.feature_count", observed_sort)
@@ -262,7 +261,6 @@ def test_map_theme_ranking_prefers_newer_observation_over_feature_count() -> Non
def test_map_workspace_restores_theme_from_selected_dataset() -> None:
root = Path(__file__).resolve().parents[2]
workspace = read_map_workspace()
assert "function themeIdForDataset(" in workspace
assert "useState<DataThemeId>(() =>" in workspace
backend/tests/test_sprint236_bathymetry_expansion.py
+27 -2
View File
@@ -1,7 +1,6 @@
from __future__ import annotations
from datetime import UTC, datetime
import json
from pathlib import Path
import ssl
import sys
@@ -31,7 +30,7 @@ if str(SCRIPTS) not in sys.path:
sys.path.insert(0, str(SCRIPTS))
import provision_flanders_geographic_scope as flanders_scope # noqa: E402
from tests.frontend_contract import read_map_workspace, read_feature
from tests.frontend_contract import read_map_workspace, read_feature # noqa: E402
class BinaryResponse:
@@ -136,6 +135,32 @@ def test_mdk_probe_parses_capabilities_without_enabling_acquisition() -> None:
assert seen["timeout"] == 20
def test_mdk_probe_default_opener_uses_guarded_strict_tls_path(monkeypatch) -> None:
seen = {}
def guarded_factory(expected_url):
seen["expected_url"] = expected_url
def open_request(request, timeout):
seen["request_url"] = request.full_url
seen["timeout"] = timeout
return BinaryResponse(capabilities_xml())
return open_request
monkeypatch.setattr(
"app.services.mdk_bathymetry_probe_service.guarded_opener",
guarded_factory,
)
result = MdkBathymetryProbeService.probe(settings=Settings(_env_file=None))
assert result["status"] == "reachable"
assert seen["expected_url"] == seen["request_url"]
assert seen["expected_url"].startswith("https://")
assert seen["timeout"] == 20
def test_mdk_probe_reports_tls_failure_and_never_uses_insecure_fallback() -> None:
calls = 0
backend/tests/test_sprint237_flanders_thematic_on_demand.py
+1 -1
View File
@@ -1,5 +1,5 @@
from pathlib import Path
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace, read_feature
from tests.frontend_contract import assert_wired, read_map_workspace, read_feature
ROOT = Path(__file__).resolve().parents[2]
backend/tests/test_sprint239_bounded_grb_acquisition.py
+1 -1
View File
@@ -19,7 +19,7 @@ from app.models import Area, Dataset, Job, Project
from app.schemas.grb import GrbAcquireRequest
from app.services.dataset_service import DatasetService
from app.services.grb_acquisition_service import GrbAcquisitionService
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace, read_feature
from tests.frontend_contract import assert_wired, read_map_workspace, read_feature
ROOT = Path(__file__).resolve().parents[2]
backend/tests/test_sprint241_spw_bathymetry_raster.py
-1
View File
@@ -1,7 +1,6 @@
from __future__ import annotations
import importlib.util
import io
from pathlib import Path
import sys
import zipfile
backend/tests/test_sprint7a_persistence_foundation.py
+33 -15
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
import json
from pathlib import Path
from uuid import uuid4
@@ -187,23 +188,31 @@ def test_dataset_upload_persists_vector_features(monkeypatch, tmp_path) -> None:
filename = "reference.geojson"
content_type = "application/geo+json"
async def read(self) -> bytes:
def __init__(self) -> None:
import json
return json.dumps(payload).encode("utf-8")
self._content = json.dumps(payload).encode("utf-8")
async def read(self, size: int) -> bytes:
chunk, self._content = self._content[:size], self._content[size:]
return chunk
storage_path = tmp_path / "reference.geojson"
storage_path.write_text("{}", encoding="utf-8")
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_dataset_file",
lambda **_kwargs: {
storage_path.write_text(json.dumps(payload), encoding="utf-8")
async def persist_upload_file(**_kwargs):
return {
"storage_path": str(storage_path),
"original_filename": "reference.geojson",
"stored_filename": "reference.geojson",
"content_type": "application/geo+json",
"size_bytes": 2,
"size_bytes": storage_path.stat().st_size,
"checksum_sha256": "0" * 64,
},
}
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_upload_file",
persist_upload_file,
)
result = asyncio.run(
@@ -236,19 +245,28 @@ def test_dataset_upload_rolls_back_dataset_and_file_when_vector_indexing_fails(m
filename = "invalid.geojson"
content_type = "application/geo+json"
async def read(self) -> bytes:
return b'{"type":"FeatureCollection","features":[]}'
def __init__(self) -> None:
self._content = b'{"type":"FeatureCollection","features":[]}'
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_dataset_file",
lambda **_kwargs: {
async def read(self, size: int) -> bytes:
chunk, self._content = self._content[:size], self._content[size:]
return chunk
storage_path.write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8")
async def persist_upload_file(**_kwargs):
return {
"storage_path": str(storage_path),
"original_filename": "invalid.geojson",
"stored_filename": "invalid.geojson",
"content_type": "application/geo+json",
"size_bytes": 2,
"size_bytes": storage_path.stat().st_size,
"checksum_sha256": "0" * 64,
},
}
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_upload_file",
persist_upload_file,
)
monkeypatch.setattr(
VectorFeatureService,
backend/tests/test_sprint8b_yolo_foundation.py
+35 -13
View File
@@ -13,12 +13,23 @@ from shapely.geometry import box, mapping
from app.core.config import Settings
from app.core.errors import AppError
from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project, SourceRegistry, SourceSnapshot
from app.models import (
AnalysisRun,
Area,
Dataset,
DatasetVersion,
Detection,
Job,
Project,
SourceRegistry,
SourceSnapshot,
)
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
from app.services.detection_service import DetectionService
from app.services.model_registry_service import ModelRegistryService
from app.services.model_validation_scope_service import ModelValidationScopeService
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
from app.services.tile_manifest_service import TileManifestService
from app.services.yolo_adapter import YoloDetectionAdapter
ROOT = Path(__file__).resolve().parents[2]
@@ -189,6 +200,8 @@ def _project_and_dataset(dataset_type: str = "raster"):
source_name="test-derived-raster",
storage_path="storage/uploads/source.tif",
checksum_sha256=checksum,
crs="EPSG:4326",
bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0},
source_registry_id=source_registry_id,
source_snapshot_id=source_snapshot_id,
data_contract_key="geointel.raster.geotiff",
@@ -201,6 +214,9 @@ def _project_and_dataset(dataset_type: str = "raster"):
)
dataset.source_registry = source_registry
dataset.source_snapshot = source_snapshot
dataset.versions.append(
DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1, checksum_sha256=checksum)
)
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
return db, project_id, dataset_id
@@ -309,28 +325,34 @@ def _write_model_sidecar(
)
def _manifest(tmp_path: Path, tile_count: int = 1) -> Path:
def _manifest(tmp_path: Path, tile_count: int = 1, dataset: Dataset | None = None) -> Path:
tiles = []
for index in range(tile_count):
tile_path = tmp_path / f"tile_{index:04d}.tif"
tile_path.write_bytes(b"fixture")
tiles.append(
{
tile = {
"path": str(tile_path),
"pixel_window": [0, 0, 100, 100],
"bounds": [4.0, 51.0, 5.0, 52.0],
"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01],
"crs": "EPSG:4326",
"index": index,
**TileManifestService.tile_integrity(tile_path),
}
)
tiles.append(tile)
binding = (
TileManifestService.dataset_binding(SimpleNamespace(get=lambda *_args: None), dataset)
if dataset is not None
else {}
)
manifest_path = tmp_path / "manifest.json"
manifest_path.write_text(
json.dumps(
{
**binding,
"tile_set_id": "tiles-fixture",
"source_dataset_id": str(uuid4()),
"source_raster_id": str(uuid4()),
"source_dataset_id": binding.get("source_dataset_id", str(uuid4())),
"source_raster_id": binding.get("source_raster_id", str(uuid4())),
"crs": "EPSG:4326",
"bounds": [4.0, 51.0, 5.0, 52.0],
"tile_size": 100,
@@ -514,7 +536,7 @@ def test_yolo_run_fails_closed_before_adapter_load_without_sidecar(tmp_path: Pat
dataset_id=dataset_id,
model_id="yolo-configured",
confidence_threshold=0.5,
tile_manifest_path=str(_manifest(tmp_path)),
tile_manifest_path=str(_manifest(tmp_path, dataset=db.get(Dataset, dataset_id))),
settings=settings,
yolo_adapter_class=AvailableAdapter,
)
@@ -530,7 +552,7 @@ def test_yolo_run_rejects_manifest_over_tile_limit(tmp_path: Path) -> None:
model_path.write_bytes(b"local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_max_tiles=1)
_write_model_sidecar(model_path, settings, db=db)
manifest_path = _manifest(tmp_path, tile_count=2)
manifest_path = _manifest(tmp_path, tile_count=2, dataset=db.get(Dataset, dataset_id))
result = DetectionService.run_detection(
db=db,
@@ -609,7 +631,7 @@ def test_yolo_run_rejects_unbound_model_snapshot_before_adapter_load(tmp_path: P
dataset_id=dataset_id,
model_id="yolo-configured",
confidence_threshold=0.5,
tile_manifest_path=str(_manifest(tmp_path)),
tile_manifest_path=str(_manifest(tmp_path, dataset=db.get(Dataset, dataset_id))),
settings=settings,
yolo_adapter_class=NeverLoadUnboundModelAdapter,
)
@@ -638,7 +660,7 @@ def test_yolo_run_persists_mocked_georeferenced_detections(tmp_path: Path) -> No
model_path.write_bytes(b"local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_model_version="local-test")
_write_model_sidecar(model_path, settings, db=db)
manifest_path = _manifest(tmp_path, tile_count=1)
manifest_path = _manifest(tmp_path, tile_count=1, dataset=db.get(Dataset, dataset_id))
result = DetectionService.run_detection(
db=db,
@@ -678,7 +700,7 @@ def test_yolo_class_filter_is_case_insensitive_and_persists_canonical_class(tmp_
model_path.write_bytes(b"local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path))
_write_model_sidecar(model_path, settings, db=db)
manifest_path = _manifest(tmp_path, tile_count=1)
manifest_path = _manifest(tmp_path, tile_count=1, dataset=db.get(Dataset, dataset_id))
result = DetectionService.run_detection(
db=db,
@@ -706,7 +728,7 @@ def test_yolo_run_suppresses_cross_tile_duplicate_detections(tmp_path: Path) ->
model_path.write_bytes(b"local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_duplicate_iou_threshold=0.5)
_write_model_sidecar(model_path, settings, db=db)
manifest_path = _manifest(tmp_path, tile_count=2)
manifest_path = _manifest(tmp_path, tile_count=2, dataset=db.get(Dataset, dataset_id))
result = DetectionService.run_detection(
db=db,
backend/tests/test_sprint93_export_handoff_completion.py
+1 -1
View File
@@ -1,5 +1,5 @@
from pathlib import Path
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_feature
from tests.frontend_contract import assert_mentions, read_feature
ROOT = Path(__file__).resolve().parents[2]
backend/tests/test_storage_service.py
+114
View File
@@ -1,6 +1,12 @@
import asyncio
from hashlib import sha256
from pathlib import Path
from types import SimpleNamespace
import pytest
from app.core.errors import AppError
from app.services.dataset_service import DatasetService
from app.services.storage_service import StorageService
@@ -26,3 +32,111 @@ def test_persist_dataset_file_records_metadata(monkeypatch, tmp_path) -> None:
assert len(metadata["checksum_sha256"]) == 64
assert Path(metadata["storage_path"]).exists()
assert str(Path(tmp_path, "uploads", "project-123", "vector", "dataset-456")) in metadata["storage_path"]
class _ChunkedUpload:
def __init__(self, content: bytes) -> None:
self.content = content
self.requested_sizes: list[int] = []
async def read(self, size: int) -> bytes:
self.requested_sizes.append(size)
chunk, self.content = self.content[:size], self.content[size:]
return chunk
def test_persist_upload_file_streams_bounded_chunks(monkeypatch, tmp_path) -> None:
monkeypatch.setattr(StorageService, "_base_dir", staticmethod(lambda: tmp_path))
upload = _ChunkedUpload(b"abcdefghijk")
metadata = asyncio.run(
StorageService.persist_upload_file(
project_id="project",
dataset_id="dataset",
dataset_type="raster",
original_filename="source.tif",
upload=upload,
content_type="image/tiff",
max_bytes=32,
chunk_size=4,
)
)
stored = Path(metadata["storage_path"])
assert upload.requested_sizes == [4, 4, 4, 4]
assert stored.read_bytes() == b"abcdefghijk"
assert metadata["size_bytes"] == 11
assert metadata["checksum_sha256"] == sha256(b"abcdefghijk").hexdigest()
def test_persist_upload_file_rejects_oversize_and_removes_partial_file(monkeypatch, tmp_path) -> None:
monkeypatch.setattr(StorageService, "_base_dir", staticmethod(lambda: tmp_path))
upload = _ChunkedUpload(b"0123456789")
expected_path = Path(
StorageService.dataset_file_path(
"project",
"dataset",
"vector",
"source.geojson",
)
)
with pytest.raises(AppError) as exc_info:
asyncio.run(
StorageService.persist_upload_file(
project_id="project",
dataset_id="dataset",
dataset_type="vector",
original_filename="source.geojson",
upload=upload,
content_type="application/geo+json",
max_bytes=8,
chunk_size=3,
)
)
assert exc_info.value.code == "UPLOAD_TOO_LARGE"
assert exc_info.value.status_code == 413
assert not expected_path.exists()
def test_vector_staging_uses_lower_in_memory_limit(monkeypatch) -> None:
captured = {}
async def persist_upload_file(**kwargs):
captured.update(kwargs)
return {"storage_path": "unused"}
monkeypatch.setattr(
"app.services.dataset_service.get_settings",
lambda: SimpleNamespace(max_upload_mb=500, max_in_memory_vector_mb=32),
)
monkeypatch.setattr(StorageService, "persist_upload_file", persist_upload_file)
asyncio.run(
DatasetService._stage_upload(
project_id="project",
dataset_id="dataset",
dataset_type="vector",
filename="source.geojson",
file=SimpleNamespace(content_type="application/geo+json"),
)
)
assert captured["max_bytes"] == 32 * 1024 * 1024
def test_staged_vector_read_is_bounded_before_loading_file(monkeypatch, tmp_path) -> None:
source = tmp_path / "large.geojson"
source.write_bytes(b"x" * (1024 * 1024 + 1))
monkeypatch.setattr(
"app.services.dataset_service.get_settings",
lambda: SimpleNamespace(max_upload_mb=500, max_in_memory_vector_mb=1),
)
with pytest.raises(AppError) as exc_info:
DatasetService._read_staged_vector_bytes({"storage_path": str(source)})
assert exc_info.value.code == "UPLOAD_TOO_LARGE"
assert exc_info.value.status_code == 413
assert not source.exists()
backend/tests/test_tile_manifest_binding.py
+210
View File
@@ -0,0 +1,210 @@
from __future__ import annotations
import json
from pathlib import Path
from uuid import uuid4
from geoalchemy2.shape import from_shape
import pytest
from shapely.geometry import box
from app.core.config import Settings
from app.core.errors import AppError
from app.models import Area, Dataset, DatasetVersion, SourceRegistry, SourceSnapshot
from app.services.tile_manifest_service import TileManifestService, canonical_manifest_json
class FakeSession:
def __init__(self, objects=None) -> None:
self.objects = objects or {}
def get(self, model, item_id):
return self.objects.get((model, item_id))
def _dataset_and_session(*, with_area: bool = True):
project_id = uuid4()
dataset_id = uuid4()
registry_id = uuid4()
snapshot_id = uuid4()
checksum = "a" * 64
area = None
area_id = uuid4() if with_area else None
if area_id is not None:
area = Area(
id=area_id,
project_id=project_id,
name="Inference AOI",
geometry=from_shape(box(4.0, 51.0, 5.0, 52.0), srid=4326),
)
registry = SourceRegistry(
id=registry_id,
source_key="governed-test-raster",
display_name="Governed test raster",
classification="derived",
authority_name="GeoIntel",
)
snapshot = SourceSnapshot(
id=snapshot_id,
source_registry_id=registry_id,
snapshot_key="snapshot-1",
checksum_sha256=checksum,
ingest_status="ingested",
freshness_status="current",
)
dataset = Dataset(
id=dataset_id,
project_id=project_id,
area_id=area_id,
name="orthophoto.tif",
dataset_type="raster",
source="governed-test-raster",
source_name="governed-test-raster",
storage_path="storage/uploads/orthophoto.tif",
size_bytes=123,
checksum_sha256=checksum,
crs="EPSG:4326",
bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0},
source_registry_id=registry_id,
source_snapshot_id=snapshot_id,
data_contract_key="geointel.raster.geotiff",
data_contract_version="1.0.0",
validation_status="passed",
provenance_status="complete",
lineage_status="not_applicable",
quarantine_status="not_quarantined",
status="ready",
)
dataset.source_registry = registry
dataset.source_snapshot = snapshot
version = DatasetVersion(
id=uuid4(),
dataset_id=dataset_id,
version=3,
checksum_sha256=checksum,
)
dataset.versions.append(version)
objects = {(Dataset, dataset_id): dataset}
if area is not None:
objects[(Area, area.id)] = area
return FakeSession(objects), dataset, area
def _manifest(tmp_path: Path, db: FakeSession, dataset: Dataset) -> Path:
tile_path = tmp_path / "tile_0000.tif"
tile_path.write_bytes(b"immutable tile")
payload = {
**TileManifestService.dataset_binding(db, dataset),
"tile_set_id": "tile-set-1",
"crs": "EPSG:4326",
"source_crs": "EPSG:4326",
"dataset_crs": "EPSG:4326",
"bounds": [4.0, 51.0, 5.0, 52.0],
"count": 1,
"tiles": [
{
"path": str(tile_path),
"bounds": [4.0, 51.0, 5.0, 52.0],
"crs": "EPSG:4326",
"index": 0,
**TileManifestService.tile_integrity(tile_path),
}
],
}
manifest_path = tmp_path / "manifest.json"
manifest_path.write_text(canonical_manifest_json(payload), encoding="utf-8")
return manifest_path
def _validate(tmp_path: Path, db: FakeSession, dataset: Dataset, *, prefix: str = "DETECTION"):
manifest_path = tmp_path / "manifest.json"
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
return TileManifestService.validate_for_inference(
db,
dataset,
payload,
manifest_path=manifest_path,
settings=Settings(_env_file=None, storage_root=str(tmp_path)),
error_prefix=prefix,
)
def test_versioned_tile_manifest_binds_dataset_version_snapshot_area_and_tile_bytes(tmp_path: Path) -> None:
db, dataset, _area = _dataset_and_session()
manifest_path = _manifest(tmp_path, db, dataset)
evidence = _validate(tmp_path, db, dataset)
assert evidence["manifest_path"] == str(manifest_path.resolve())
assert evidence["source_dataset_id"] == str(dataset.id)
assert evidence["source_snapshot_id"] == str(dataset.source_snapshot_id)
assert evidence["dataset_version_id"] == str(dataset.versions[0].id)
assert evidence["source_area_id"] == str(dataset.area_id)
assert evidence["tile_count"] == 1
assert evidence["tile_union_bounds_epsg4326"] == pytest.approx([4.0, 51.0, 5.0, 52.0])
@pytest.mark.parametrize(
("field", "value", "expected_code"),
[
("source_dataset_id", lambda: str(uuid4()), "DETECTION_TILE_MANIFEST_DATASET_MISMATCH"),
("source_dataset_checksum_sha256", lambda: "b" * 64, "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"),
("source_snapshot_id", lambda: str(uuid4()), "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"),
("dataset_version", lambda: 99, "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"),
("source_area_geometry_sha256", lambda: "c" * 64, "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"),
],
)
def test_tile_manifest_rejects_dataset_or_provenance_mismatch(
tmp_path: Path,
field: str,
value,
expected_code: str,
) -> None:
db, dataset, _area = _dataset_and_session()
manifest_path = _manifest(tmp_path, db, dataset)
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
payload[field] = value()
manifest_path.write_text(canonical_manifest_json(payload), encoding="utf-8")
with pytest.raises(AppError) as exc_info:
_validate(tmp_path, db, dataset)
assert exc_info.value.code == expected_code
def test_tile_manifest_rejects_tile_bytes_changed_after_manifest_creation(tmp_path: Path) -> None:
db, dataset, _area = _dataset_and_session()
manifest_path = _manifest(tmp_path, db, dataset)
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
Path(payload["tiles"][0]["path"]).write_bytes(b"tampered tile")
with pytest.raises(AppError) as exc_info:
_validate(tmp_path, db, dataset)
assert exc_info.value.code == "DETECTION_TILE_MANIFEST_TILE_INTEGRITY_MISMATCH"
def test_tile_manifest_rejects_dataset_without_version_binding(tmp_path: Path) -> None:
db, dataset, _area = _dataset_and_session()
_manifest(tmp_path, db, dataset)
dataset.versions.clear()
with pytest.raises(AppError) as exc_info:
_validate(tmp_path, db, dataset)
assert exc_info.value.code == "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"
assert "dataset_version_id" in exc_info.value.details["missing_fields"]
def test_segmentation_tile_manifest_rejects_union_outside_dataset_scope(tmp_path: Path) -> None:
db, dataset, _area = _dataset_and_session(with_area=False)
manifest_path = _manifest(tmp_path, db, dataset)
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
payload["bounds"] = [4.0, 51.0, 6.0, 52.0]
payload["tiles"][0]["bounds"] = [4.0, 51.0, 6.0, 52.0]
manifest_path.write_text(canonical_manifest_json(payload), encoding="utf-8")
with pytest.raises(AppError) as exc_info:
_validate(tmp_path, db, dataset, prefix="SEGMENTATION")
assert exc_info.value.code == "SEGMENTATION_TILE_MANIFEST_SCOPE_MISMATCH"
docs/API_CONTRACTS.md
+114 -23
View File
@@ -13,8 +13,8 @@ This document freezes the first API shape. Codex may add implementation details
uses the canonical `{"data": ...}` envelope. Readiness runs an OpenAPI audit
that rejects free-form dictionary responses and envelope drift.
- The only successful non-envelope responses are `/health`, `/health/live`,
`/health/ready`, the four documented persisted-raster PNG endpoints and the
export artifact download endpoint.
`/health/ready`, the two documented Authentik redirects, the documented
persisted-raster PNG endpoints and the export artifact download endpoint.
## Shared schemas
@@ -63,7 +63,7 @@ one client/username combination within five minutes temporarily return HTTP
Optional guest access is a configuration-gated demonstration mode. It creates
a shorter signed session with role `guest`, scopes that session to the
idempotently seeded demo project and blocks mutating operator routes. Project
idempotently seeded demo project and blocks administrative operator routes. Project
listing is filtered to the bound demo project. The frontend exposes the same
exploration, assistant, model-selection, analysis, QA and export workspaces as
an operator. Model catalogs are globally readable; every run, result and export
@@ -73,13 +73,33 @@ administrative mutations remain unavailable. This is deliberately **not**
a substitute for user accounts, authorization or tenant isolation; expose it
only on a dedicated demo installation without private or operational data.
The functional demo boundary includes bounded, project-path-scoped official
source acquisition, persisted bbox selections, temporal comparisons and change
detection. Change detection resolves the source dataset first, requires its
project to equal the signed guest-project UUID and validates the target dataset
against that same project before the synchronous job starts. Cross-project
datasets therefore fail before comparison. Generic uploads, arbitrary
clip/buffer/intersect operations, forceful project management and evidence
adjudication remain blocked.
The login/demo seed remains offline and retains its synthetic raster as an
explicit `fixture` for UI context and fixture QA only. That raster is excluded
from configured detection and segmentation selectors and the production
consumption gate continues to reject it. A guest starts real model inference
by drawing a bounded map selection; GeoIntel then acquires an official regional
orthophoto on demand inside the signed demo project, persists its contextual
source registry/snapshot/checksum evidence, and only then tiles and runs the
configured model. No network acquisition occurs merely by logging in.
### GET `/api/v1/auth/session`
Public session probe used by the frontend before it mounts the workbench.
When authentication is disabled, `authenticated` is true and
`authentication_required` is false so local development retains its existing
direct workflow. `guest_access_enabled` tells the landing page whether it may
show the guest action.
show the guest action. `authentik_enabled` indicates that the additive
Authentik operator flow is fully configured; the local operator login remains
available as a recovery path.
```json
{
@@ -90,6 +110,7 @@ show the guest action.
"expires_at": null,
"role": null,
"guest_access_enabled": true,
"authentik_enabled": false,
"guest_project_id": null
}
}
@@ -112,6 +133,23 @@ Successful login sets the session cookie and returns the authenticated session
shape. Invalid credentials return HTTP 401 `INVALID_CREDENTIALS`; username
existence is not disclosed.
### GET `/api/v1/auth/authentik/start`
Starts an authorization-code OIDC flow with PKCE, signed state and nonce when
all Authentik settings are present. Discovery, token and JWKS requests are
restricted to the configured HTTPS issuer origin, reject redirects and enforce
a bounded JSON response size. The flow cookie is HttpOnly, Secure, ten minutes
or less and scoped to the Authentik callback path.
### GET `/api/v1/auth/authentik/callback`
Validates issuer, audience, signature, expiry, state, nonce and the exact
configured verified e-mail address. Success creates the same operator session
as local login and clears the one-use flow cookie. Failure clears that cookie
and redirects to the landing page with a generic error marker; token or
identity details are never returned to the browser. These two redirect routes
are the only additional non-envelope authentication responses.
### POST `/api/v1/auth/guest`
No request body is required. The endpoint is available only when both
@@ -130,6 +168,12 @@ is available for that bound demo project. Unscoped analysis routes require the
same UUID as a `project_id` query parameter; cross-project values fail before
route execution. Coverage resolution additionally verifies the `project_id` in
the request body against the guest-session scope.
Raster tiling for an already-persisted raster in the bound demo project is an
explicitly allowed preparation step for detection and segmentation. It creates
only integrity-bound inference tiles and a job record; uploads, source
acquisition, model management and arbitrary derived-dataset writes remain
operator-only. Guest tiling is server-capped by the configured inference tile
limit before any tile bytes are written.
### POST `/api/v1/auth/logout`
@@ -286,11 +330,13 @@ Request:
Backend responsibilities:
- Validate geometry.
- Repair trivial polygon issues if safe.
- Store geometry in PostGIS.
- Calculate area in square meters using projected CRS.
- Store bbox.
- Accept only a finite, valid `Polygon` or `MultiPolygon` in the declared CRS.
- Reject unknown/non-2D CRS definitions and geometry outside the Belgium and
Belgian North Sea workbench domain.
- Transform the geometry to EPSG:4326 before PostGIS persistence while
retaining the exact declared CRS in `original_crs`.
- Calculate `area_m2` in Belgian Lambert 72 (`EPSG:31370`) and derive the
persisted EPSG:4326 bbox from the normalized geometry.
### GET `/api/v1/projects/{project_id}/areas/{area_id}`
@@ -299,8 +345,12 @@ area list endpoint and includes persisted GeoJSON geometry for map display.
### PATCH `/api/v1/projects/{project_id}/areas/{area_id}`
Updates the area name and/or geometry. Geometry updates follow the same
validation, repair and metric-calculation rules as area creation.
Updates the area name and/or geometry. A replacement geometry follows the same
strict CRS validation, EPSG:4326 normalization and metric-calculation rules as
area creation and atomically recomputes `geometry`, `bbox`, `area_m2` and
`original_crs`. A PATCH containing `crs` without `geometry` fails with
`INVALID_AREA_CRS_UPDATE`; omitted CRS on replacement geometry means
EPSG:4326.
### GET `/api/v1/projects/{project_id}/areas/municipalities`
@@ -409,10 +459,22 @@ validation report/status, provenance/lineage status, quarantine status and an
idempotent ingest key. A malformed or doubtful artifact is retained as
`status=quarantined`; it is not silently discarded or made ready.
Vector uploads remain stored as original files and are also persisted into
`vector_features` as queryable PostGIS state only after their contract passes.
Non-EPSG:4326 vector coordinates are explicitly transformed before canonical
feature persistence; relabelling Lambert coordinates as EPSG:4326 is rejected.
The backend streams uploads to governed storage in bounded 8 MiB reads while
calculating size and SHA-256. `GEOINTEL_MAX_UPLOAD_MB` (legacy alias
`MAX_UPLOAD_MB`, default 500, allowed range 1–2,048) is enforced by the backend even when a request
bypasses the reverse proxy. The first byte beyond the configured limit aborts
the ingest, removes the partial file and returns HTTP 413 with
`UPLOAD_TOO_LARGE`. Because GeoJSON validation currently requires an in-memory
parse, vector uploads have the additional lower
`GEOINTEL_MAX_IN_MEMORY_VECTOR_MB` limit (default 64 MiB, maximum 256 MiB).
This limit is enforced during streaming and again with a bounded read before
parsing; larger vector sources must use a governed batch-import workflow.
Vector uploads are also persisted into `vector_features` as queryable PostGIS
state only after their contract passes. Non-EPSG:4326 source bytes are retained
as provenance evidence while the normal Dataset `storage_path` points to the
deterministic EPSG:4326 artifact. Relabelling Lambert coordinates as EPSG:4326
is rejected.
### GET `/api/v1/source-registry`
@@ -1053,7 +1115,13 @@ Failure modes:
### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/tile`
Generate raster tiles and a manifest for downstream processing. Returns a job payload with `tile_set_id` and manifest metadata.
Generate raster tiles and a versioned manifest for downstream processing.
Returns a job payload with `tile_set_id` and manifest metadata. Contract
`geointel.raster.tile-manifest@2.0.0` binds the tile set to the exact source
Dataset, DatasetVersion, source registry/snapshot and checksum values that are
available in persistence, plus the source Area and Area-geometry checksum when
scoped. Every tile records immutable byte size/SHA-256, explicit CRS and bounds;
the manifest records its source extent and tile count.
If raster processing dependencies are unavailable:
@@ -1594,13 +1662,16 @@ label or model filename.
### GET `/api/v1/detection/model-assets`
Returns governed local runtime model files. This is a read-only catalog.
Returns configured local runtime model files. This is a read-only catalog;
catalog visibility establishes byte identity and runtime availability only.
GeoIntel never downloads, creates, mutates or deletes model weights from this
endpoint.
The backend scans `YOLO_MODELS_DIR` (default `/app/models`). When
`YOLO_MODEL_PATH` resolves to an existing file, production catalog output is
restricted to that explicitly approved active model. When no active model is
restricted to that active runtime model. Active/available is a runtime-selection
state only: it does not assert governed validation, human review, national
coverage or promotion. When no active model is
configured, supported `.pt`, `.onnx` and `.engine` files remain visible for
development/operator discovery but cannot make the configured detector ready.
@@ -1620,8 +1691,12 @@ Response data:
"size_bytes": 123456,
"sha256": "sha256hex",
"active": true,
"status": "approved",
"limitation_message": "Approved local runtime model asset. GeoIntel will not download or mutate model weights.",
"runtime_available": true,
"runtime_status": "active",
"governed_validation_status": "not_verified_by_catalog",
"promotion_status": "not_verified_by_catalog",
"status": "runtime_active",
"limitation_message": "Active local runtime model asset. Runtime selection is not evidence of governed validation or promotion.",
"will_download_models": false
}
],
@@ -1732,10 +1807,21 @@ governed, runtime-produced artifact the persistence model requires. Rejection is
`STORAGE_PATH_OUTSIDE_ROOT`; `GEOINTEL_ALLOW_EXTERNAL_ARTIFACT_PATHS` opts out
for provisioning workflows that stage tiles before ingest.
Configured detection and segmentation revalidate the v2 contract before model
loading. The requested Dataset identity, checksum, latest DatasetVersion,
source snapshot and Area binding must still match; every tile checksum is
recomputed; and the EPSG:4326 tile union must remain inside both the declared
manifest extent and persisted raster extent and intersect its Area where one is
bound. Validation fails closed with the typed suffixes
`TILE_MANIFEST_DATASET_MISMATCH`, `TILE_MANIFEST_PROVENANCE_MISMATCH`,
`TILE_MANIFEST_TILE_INTEGRITY_MISMATCH` or
`TILE_MANIFEST_SCOPE_MISMATCH`, prefixed with `DETECTION_` or
`SEGMENTATION_` for the requesting task.
The manifest must carry explicit CRS metadata (`crs`, `source_crs` or
`dataset_crs`). A manifest without it fails with
`DETECTION_TILE_MANIFEST_INVALID` rather than being georeferenced against an
assumed EPSG:4326, which would place detections plausibly but wrongly.
`dataset_crs`). A manifest without it fails rather than being georeferenced
against an assumed EPSG:4326, which would place detections plausibly but
wrongly.
Tiles are read with rasterio: the visible RGB bands are selected explicitly and
percentile-stretched to 8-bit, so 16-bit and 4-band (RGB + NIR) orthophotos
@@ -2095,6 +2181,11 @@ to the synchronous route. A configured model requires an existing
`SEGMENTATION_ACCELERATOR_MISCONFIGURED` when CUDA is required. A valid
zero-polygon run is shown as an empty model result, never as proof that the AOI
contains no relevant objects.
When a raster is selected but no manifest exists, the normal frontend flow
first requests project-scoped 512 px tiles with 64 px overlap and forwards the
server-generated manifest. Inference then applies the complete v2 provenance,
integrity and scope validation before model load; manual server-path entry is
an operator-only advanced control.
Validation errors:
scripts/audit_api_contracts.py
+2
View File
@@ -13,6 +13,8 @@ ALLOWED_NON_ENVELOPE_ENDPOINTS = {
("GET", "/health"),
("GET", "/health/live"),
("GET", "/health/ready"),
("GET", "/api/v1/auth/authentik/start"),
("GET", "/api/v1/auth/authentik/callback"),
("GET", "/api/v1/exports/{export_id}/download"),
("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/image"),
("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/terrain/image"),
scripts/smoke_contracts.py
+8 -2
View File
@@ -1,5 +1,11 @@
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
missing=[d for d in ["contracts/api","contracts/database","contracts/events"] if not (ROOT/d).exists()]
if missing: raise SystemExit("Missing contract directories: "+", ".join(missing))
missing = [
d
for d in ["contracts/api", "contracts/database", "contracts/events"]
if not (ROOT / d).exists()
]
if missing:
raise SystemExit("Missing contract directories: " + ", ".join(missing))
print("Contracts smoke OK")