Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(platform): govern geospatial analysis and raster handoffs

Browse Source
This commit is contained in:
Jens
2026-08-30 06:00:15 +02:00
parent 96f90373dc
commit 80a2d1654d
63 changed files with 2335 additions and 312 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/services/aoi_operation_service.py
+273 -68
View File
@@ -22,8 +22,11 @@ class AoiOperationService:
_to_metric = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
_to_wgs84 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
SCOPE_AREA_NAMES = {
"belgium": "Belgium land", "flanders": "Flanders", "wallonia": "Wallonia",
"brussels": "Brussels-Capital Region", "belgian_north_sea": "Belgian part of the North Sea",
"belgium": "Belgium land",
"flanders": "Flanders",
"wallonia": "Wallonia",
"brussels": "Brussels-Capital Region",
"belgian_north_sea": "Belgian part of the North Sea",
"territorial_sea": "Belgian territorial sea (0-12 nautical miles)",
"exclusive_economic_zone": "Belgian exclusive economic zone beyond territorial sea",
"continental_shelf": "Belgian continental shelf beyond territorial sea",
@@ -32,13 +35,19 @@ class AoiOperationService:
@staticmethod
def create(db, project_id: UUID, payload: AoiOperationCreate) -> dict:
if db.get(Project, project_id) is None:
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
raise AppError(
code="PROJECT_NOT_FOUND", message="Project not found", status_code=404
)
geometry = AoiOperationService._resolve_geometry(db, project_id, payload)
if payload.coverage_zone:
geometry = AoiOperationService._clip_to_zone(db, project_id, geometry, payload.coverage_zone)
geometry = AoiOperationService._clip_to_zone(
db, project_id, geometry, payload.coverage_zone
)
geometry = AoiOperationService._as_multipolygon(geometry)
metric_geometry = transform(AoiOperationService._to_metric.transform, geometry)
partition_side_m = AoiOperationService._partition_side(payload.provider_key, payload.max_partition_side_m)
partition_side_m = AoiOperationService._partition_side(
payload.provider_key, payload.max_partition_side_m
)
cells = AoiOperationService._partition(metric_geometry, partition_side_m)
operation_id = uuid4()
now = datetime.now(timezone.utc)
@@ -53,7 +62,9 @@ class AoiOperationService:
plan_json={
"partition_strategy": "epsg31370_square_grid_intersection_v1",
"max_partition_side_m": partition_side_m,
"budget_source": "governed_provider_registry" if payload.max_partition_side_m is None else "stricter_operator_override",
"budget_source": "governed_provider_registry"
if payload.max_partition_side_m is None
else "stricter_operator_override",
"partition_count": len(cells),
"provider_key": payload.provider_key,
"product_key": payload.product_key,
@@ -65,13 +76,21 @@ class AoiOperationService:
wgs84 = transform(AoiOperationService._to_wgs84.transform, cell)
wgs84 = AoiOperationService._as_multipolygon(wgs84)
digest = sha256(wgs84.wkb).hexdigest()[:20]
db.add(AoiOperationPartition(
id=uuid4(), operation_id=operation_id,
partition_key=f"{payload.provider_key}:{payload.product_key}:{ordinal:05d}:{digest}",
provider_key=payload.provider_key, product_key=payload.product_key,
ordinal=ordinal, status="queued", geometry=from_shape(wgs84, srid=4326),
attempt_count=0, max_attempts=payload.max_attempts, created_at=now,
))
db.add(
AoiOperationPartition(
id=uuid4(),
operation_id=operation_id,
partition_key=f"{payload.provider_key}:{payload.product_key}:{ordinal:05d}:{digest}",
provider_key=payload.provider_key,
product_key=payload.product_key,
ordinal=ordinal,
status="queued",
geometry=from_shape(wgs84, srid=4326),
attempt_count=0,
max_attempts=payload.max_attempts,
created_at=now,
)
)
db.commit()
return AoiOperationService.read(db, project_id, operation_id)
@@ -79,13 +98,32 @@ class AoiOperationService:
def _clip_to_zone(db, project_id: UUID, geometry, zone: str):
area_name = AoiOperationService.SCOPE_AREA_NAMES.get(zone)
if area_name is None:
raise AppError(code="AOI_COVERAGE_ZONE_UNSUPPORTED", message="Unknown governed coverage zone", details={"coverage_zone": zone}, status_code=422)
scope = db.query(Area).filter(Area.project_id == project_id, Area.name == area_name).first()
raise AppError(
code="AOI_COVERAGE_ZONE_UNSUPPORTED",
message="Unknown governed coverage zone",
details={"coverage_zone": zone},
status_code=422,
)
scope = (
db.query(Area)
.filter(Area.project_id == project_id, Area.name == area_name)
.first()
)
if scope is None:
raise AppError(code="AOI_COVERAGE_ZONE_NOT_MATERIALIZED", message="The governed coverage-zone geometry is not persisted in this project", details={"coverage_zone": zone}, status_code=409)
raise AppError(
code="AOI_COVERAGE_ZONE_NOT_MATERIALIZED",
message="The governed coverage-zone geometry is not persisted in this project",
details={"coverage_zone": zone},
status_code=409,
)
clipped = geometry.intersection(to_shape(scope.geometry))
if clipped.is_empty:
raise AppError(code="AOI_OUTSIDE_PROVIDER_ZONE", message="The AOI does not intersect the provider coverage zone", details={"coverage_zone": zone}, status_code=422)
raise AppError(
code="AOI_OUTSIDE_PROVIDER_ZONE",
message="The AOI does not intersect the provider coverage zone",
details={"coverage_zone": zone},
status_code=422,
)
return clipped
@staticmethod
@@ -94,19 +132,30 @@ class AoiOperationService:
return MultiPolygon([geometry])
if isinstance(geometry, MultiPolygon):
return geometry
polygons = [part for part in getattr(geometry, "geoms", []) if isinstance(part, Polygon)]
polygons = [
part for part in getattr(geometry, "geoms", []) if isinstance(part, Polygon)
]
if not polygons:
raise AppError(code="AOI_GEOMETRY_EMPTY", message="AOI contains no polygonal area after clipping", status_code=422)
raise AppError(
code="AOI_GEOMETRY_EMPTY",
message="AOI contains no polygonal area after clipping",
status_code=422,
)
return MultiPolygon(polygons)
@staticmethod
def _partition_side(provider_key: str, requested: float | None) -> float:
settings = get_settings()
def raster_side(max_side_m: float, max_pixels: int, resolution_m: float) -> float:
def raster_side(
max_side_m: float, max_pixels: int, resolution_m: float
) -> float:
# Keep every square grid cell within both the provider's spatial
# extent limit and its decoded-pixel budget. The small safety
# margin absorbs ceil/edge rounding in the acquisition services.
pixel_limited_side = math.sqrt(float(max_pixels)) * float(resolution_m) * 0.99
pixel_limited_side = (
math.sqrt(float(max_pixels)) * float(resolution_m) * 0.99
)
return min(float(max_side_m), pixel_limited_side)
budgets = {
@@ -142,22 +191,37 @@ class AoiOperationService:
"mdk_bathymetry": 20_000.0,
}
if provider_key not in budgets:
raise AppError(code="AOI_PROVIDER_UNSUPPORTED", message="No governed partition budget is registered for this provider", details={"provider_key": provider_key}, status_code=422)
raise AppError(
code="AOI_PROVIDER_UNSUPPORTED",
message="No governed partition budget is registered for this provider",
details={"provider_key": provider_key},
status_code=422,
)
governed = budgets[provider_key]
return min(governed, float(requested)) if requested is not None else governed
@staticmethod
def _resolve_geometry(db, project_id: UUID, payload: AoiOperationCreate):
if (payload.area_id is None) == (payload.bbox is None):
raise AppError(code="AOI_SELECTION_REQUIRED", message="Provide exactly one area_id or bbox", status_code=422)
raise AppError(
code="AOI_SELECTION_REQUIRED",
message="Provide exactly one area_id or bbox",
status_code=422,
)
if payload.area_id is not None:
area = db.get(Area, payload.area_id)
if area is None or area.project_id != project_id:
raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404)
raise AppError(
code="AREA_NOT_FOUND", message="Area not found", status_code=404
)
return to_shape(area.geometry)
bbox = payload.bbox
if bbox is None or bbox.crs != "EPSG:4326":
raise AppError(code="INVALID_AOI_CRS", message="AOI bbox must use EPSG:4326", status_code=422)
raise AppError(
code="INVALID_AOI_CRS",
message="AOI bbox must use EPSG:4326",
status_code=422,
)
return box(bbox.min_x, bbox.min_y, bbox.max_x, bbox.max_y)
@staticmethod
@@ -166,11 +230,26 @@ class AoiOperationService:
columns = max(1, math.ceil((max_x - min_x) / side_m))
rows = max(1, math.ceil((max_y - min_y) / side_m))
if columns * rows > AoiOperationService.MAX_PARTITIONS:
raise AppError(code="AOI_PARTITION_LIMIT_EXCEEDED", message="AOI requires too many bounded partitions", details={"candidate_count": columns * rows, "max_partitions": AoiOperationService.MAX_PARTITIONS}, status_code=422)
raise AppError(
code="AOI_PARTITION_LIMIT_EXCEEDED",
message="AOI requires too many bounded partitions",
details={
"candidate_count": columns * rows,
"max_partitions": AoiOperationService.MAX_PARTITIONS,
},
status_code=422,
)
partitions = []
for row in range(rows):
for column in range(columns):
clipped = geometry.intersection(box(min_x + column * side_m, min_y + row * side_m, min(min_x + (column + 1) * side_m, max_x), min(min_y + (row + 1) * side_m, max_y)))
clipped = geometry.intersection(
box(
min_x + column * side_m,
min_y + row * side_m,
min(min_x + (column + 1) * side_m, max_x),
min(min_y + (row + 1) * side_m, max_y),
)
)
if not clipped.is_empty and clipped.area > 0:
partitions.append(clipped)
return partitions
@@ -179,98 +258,224 @@ class AoiOperationService:
def read(db, project_id: UUID, operation_id: UUID) -> dict:
operation = db.get(AoiOperation, operation_id)
if operation is None or operation.project_id != project_id:
raise AppError(code="AOI_OPERATION_NOT_FOUND", message="AOI operation not found", status_code=404)
partitions = db.query(AoiOperationPartition).filter(AoiOperationPartition.operation_id == operation_id).order_by(AoiOperationPartition.ordinal).all()
raise AppError(
code="AOI_OPERATION_NOT_FOUND",
message="AOI operation not found",
status_code=404,
)
partitions = (
db.query(AoiOperationPartition)
.filter(AoiOperationPartition.operation_id == operation_id)
.order_by(AoiOperationPartition.ordinal)
.all()
)
counts = Counter(partition.status for partition in partitions)
complete = counts["success"] + counts["skipped"]
return {
"id": operation.id, "project_id": operation.project_id, "area_id": operation.area_id,
"parent_job_id": operation.parent_job_id, "operation_type": operation.operation_type,
"status": operation.status, "request_json": operation.request_json, "plan_json": operation.plan_json,
"result_json": operation.result_json, "error_message": operation.error_message,
"id": operation.id,
"project_id": operation.project_id,
"area_id": operation.area_id,
"parent_job_id": operation.parent_job_id,
"operation_type": operation.operation_type,
"status": operation.status,
"request_json": operation.request_json,
"plan_json": operation.plan_json,
"result_json": operation.result_json,
"error_message": operation.error_message,
"progress": round(complete / len(partitions), 6) if partitions else 0.0,
"partition_counts": dict(counts), "partitions": partitions,
"created_at": operation.created_at, "started_at": operation.started_at, "finished_at": operation.finished_at,
"partition_counts": dict(counts),
"partitions": partitions,
"created_at": operation.created_at,
"started_at": operation.started_at,
"finished_at": operation.finished_at,
}
@staticmethod
def list(db, project_id: UUID, limit: int = 50) -> dict:
rows = db.query(AoiOperation).filter(AoiOperation.project_id == project_id).order_by(AoiOperation.created_at.desc()).limit(limit).all()
return {"items": [AoiOperationService.read(db, project_id, row.id) for row in rows], "total": len(rows)}
rows = (
db.query(AoiOperation)
.filter(AoiOperation.project_id == project_id)
.order_by(AoiOperation.created_at.desc())
.limit(limit)
.all()
)
return {
"items": [AoiOperationService.read(db, project_id, row.id) for row in rows],
"total": len(rows),
}
@staticmethod
def claim_next(db, project_id: UUID, operation_id: UUID):
operation = db.get(AoiOperation, operation_id)
if operation is None or operation.project_id != project_id:
raise AppError(code="AOI_OPERATION_NOT_FOUND", message="AOI operation not found", status_code=404)
partition = db.query(AoiOperationPartition).filter(AoiOperationPartition.operation_id == operation_id, AoiOperationPartition.status == "queued").order_by(AoiOperationPartition.ordinal).with_for_update(skip_locked=True).first()
raise AppError(
code="AOI_OPERATION_NOT_FOUND",
message="AOI operation not found",
status_code=404,
)
partition = (
db.query(AoiOperationPartition)
.filter(
AoiOperationPartition.operation_id == operation_id,
AoiOperationPartition.status == "queued",
)
.order_by(AoiOperationPartition.ordinal)
.with_for_update(skip_locked=True)
.first()
)
if partition is None:
return None
now = datetime.now(timezone.utc)
partition.status = "running"; partition.started_at = now; partition.attempt_count += 1; partition.error_message = None
operation.status = "running"; operation.started_at = operation.started_at or now
db.add(partition); db.add(operation); db.commit(); db.refresh(partition)
partition.status = "running"
partition.started_at = now
partition.attempt_count += 1
partition.error_message = None
operation.status = "running"
operation.started_at = operation.started_at or now
db.add(partition)
db.add(operation)
db.commit()
db.refresh(partition)
return partition
@staticmethod
def checkpoint(db, project_id: UUID, operation_id: UUID, partition_id: UUID, checkpoint: dict):
partition = AoiOperationService._partition_row(db, project_id, operation_id, partition_id)
def checkpoint(
db, project_id: UUID, operation_id: UUID, partition_id: UUID, checkpoint: dict
):
partition = AoiOperationService._partition_row(
db, project_id, operation_id, partition_id
)
if partition.status != "running":
raise AppError(code="AOI_PARTITION_NOT_RUNNING", message="Only a running partition can be checkpointed", status_code=409)
partition.checkpoint_json = checkpoint; db.add(partition); db.commit(); db.refresh(partition)
raise AppError(
code="AOI_PARTITION_NOT_RUNNING",
message="Only a running partition can be checkpointed",
status_code=409,
)
partition.checkpoint_json = checkpoint
db.add(partition)
db.commit()
db.refresh(partition)
return partition
@staticmethod
def complete(db, project_id: UUID, operation_id: UUID, partition_id: UUID, result: dict, skipped: bool = False):
partition = AoiOperationService._partition_row(db, project_id, operation_id, partition_id)
def complete(
db,
project_id: UUID,
operation_id: UUID,
partition_id: UUID,
result: dict,
skipped: bool = False,
):
partition = AoiOperationService._partition_row(
db, project_id, operation_id, partition_id
)
if partition.status == "success" or partition.status == "skipped":
return AoiOperationService.read(db, project_id, operation_id)
if partition.status != "running":
raise AppError(code="AOI_PARTITION_NOT_RUNNING", message="Only a running partition can complete", status_code=409)
partition.status = "skipped" if skipped else "success"; partition.result_json = result; partition.finished_at = datetime.now(timezone.utc)
db.add(partition); db.commit(); AoiOperationService._refresh_parent(db, operation_id)
raise AppError(
code="AOI_PARTITION_NOT_RUNNING",
message="Only a running partition can complete",
status_code=409,
)
partition.status = "skipped" if skipped else "success"
partition.result_json = result
partition.finished_at = datetime.now(timezone.utc)
db.add(partition)
db.commit()
AoiOperationService._refresh_parent(db, operation_id)
return AoiOperationService.read(db, project_id, operation_id)
@staticmethod
def fail(db, project_id: UUID, operation_id: UUID, partition_id: UUID, message: str, retryable: bool, details: dict):
partition = AoiOperationService._partition_row(db, project_id, operation_id, partition_id)
partition.error_message = message; partition.result_json = {"details": details}
partition.status = "queued" if retryable and partition.attempt_count < partition.max_attempts else "failed"
partition.finished_at = None if partition.status == "queued" else datetime.now(timezone.utc)
db.add(partition); db.commit(); AoiOperationService._refresh_parent(db, operation_id)
def fail(
db,
project_id: UUID,
operation_id: UUID,
partition_id: UUID,
message: str,
retryable: bool,
details: dict,
):
partition = AoiOperationService._partition_row(
db, project_id, operation_id, partition_id
)
partition.error_message = message
partition.result_json = {"details": details}
partition.status = (
"queued"
if retryable and partition.attempt_count < partition.max_attempts
else "failed"
)
partition.finished_at = (
None if partition.status == "queued" else datetime.now(timezone.utc)
)
db.add(partition)
db.commit()
AoiOperationService._refresh_parent(db, operation_id)
return AoiOperationService.read(db, project_id, operation_id)
@staticmethod
def _partition_row(db, project_id, operation_id, partition_id):
operation = db.get(AoiOperation, operation_id); partition = db.get(AoiOperationPartition, partition_id)
if operation is None or operation.project_id != project_id or partition is None or partition.operation_id != operation_id:
raise AppError(code="AOI_PARTITION_NOT_FOUND", message="AOI partition not found", status_code=404)
operation = db.get(AoiOperation, operation_id)
partition = db.get(AoiOperationPartition, partition_id)
if (
operation is None
or operation.project_id != project_id
or partition is None
or partition.operation_id != operation_id
):
raise AppError(
code="AOI_PARTITION_NOT_FOUND",
message="AOI partition not found",
status_code=404,
)
return partition
@staticmethod
def _refresh_parent(db, operation_id):
operation = db.get(AoiOperation, operation_id)
partitions = db.query(AoiOperationPartition).filter(AoiOperationPartition.operation_id == operation_id).order_by(AoiOperationPartition.ordinal).all()
partitions = (
db.query(AoiOperationPartition)
.filter(AoiOperationPartition.operation_id == operation_id)
.order_by(AoiOperationPartition.ordinal)
.all()
)
statuses = [partition.status for partition in partitions]
output_dataset_ids = []
for partition in partitions:
output_id = (partition.result_json or {}).get("output_dataset_id") if isinstance(partition.result_json, dict) else None
output_id = (
(partition.result_json or {}).get("output_dataset_id")
if isinstance(partition.result_json, dict)
else None
)
if output_id and str(output_id) not in output_dataset_ids:
output_dataset_ids.append(str(output_id))
operation.result_json = {
"partition_count": len(partitions),
"completed_partition_count": sum(status in {"success", "skipped"} for status in statuses),
"completed_partition_count": sum(
status in {"success", "skipped"} for status in statuses
),
"failed_partition_count": statuses.count("failed"),
"output_dataset_ids": output_dataset_ids,
"merge_contract": "source_aware_spatial_union",
"vector_deduplication": "source_feature_id_then_geometry",
"raster_deduplication": "governed_mosaic_grid",
"complete_coverage": bool(statuses) and all(status in {"success", "skipped"} for status in statuses),
"complete_coverage": bool(statuses)
and all(status in {"success", "skipped"} for status in statuses),
}
now = datetime.now(timezone.utc)
if statuses and all(status in {"success", "skipped"} for status in statuses):
operation.status = "success"; operation.finished_at = now; operation.error_message = None
elif "failed" in statuses and not any(status in {"queued", "running"} for status in statuses):
operation.status = "partial" if any(status in {"success", "skipped"} for status in statuses) else "failed"; operation.finished_at = now
operation.status = "success"
operation.finished_at = now
operation.error_message = None
elif "failed" in statuses and not any(
status in {"queued", "running"} for status in statuses
):
operation.status = (
"partial"
if any(status in {"success", "skipped"} for status in statuses)
else "failed"
)
operation.finished_at = now
operation.error_message = "One or more bounded source partitions failed; inspect partition evidence."
db.add(operation); db.commit()
db.add(operation)
db.commit()