From 80a2d1654d0af41c35305cd0134ad248dc1924ea Mon Sep 17 00:00:00 2001 From: Jens Date: Sun, 30 Aug 2026 06:00:15 +0200 Subject: [PATCH] fix(platform): govern geospatial analysis and raster handoffs --- backend/app/api/routes/analysis.py | 6 +- backend/app/api/routes/datasets.py | 10 +- backend/app/models.py | 2 +- backend/app/schemas/area.py | 1 + backend/app/schemas/detection.py | 4 + .../app/services/aoi_operation_executor.py | 228 ++++++-- backend/app/services/aoi_operation_service.py | 341 +++++++++--- backend/app/services/area_service.py | 35 +- .../bathymetry_profile_acquisition_service.py | 2 +- .../bathymetry_raster_analysis_service.py | 1 - backend/app/services/dataset_service.py | 138 +++-- .../app/services/detection_review_service.py | 1 - backend/app/services/detection_service.py | 27 + .../app/services/dhmv_acquisition_service.py | 2 +- .../flood_hazard_acquisition_service.py | 2 +- .../services/flood_hazard_analysis_service.py | 1 - .../mdk_bathymetry_acquisition_service.py | 2 +- .../services/mdk_bathymetry_probe_service.py | 3 +- .../services/model_asset_catalog_service.py | 10 +- .../orthophoto_acquisition_service.py | 2 +- .../app/services/raster_operations_service.py | 80 ++- .../raster_partition_analysis_service.py | 2 - backend/app/services/segmentation_service.py | 14 + .../services/source_catalog_probe_service.py | 2 +- backend/app/services/storage_service.py | 99 ++++ .../app/services/terrain_analysis_service.py | 1 - .../thematic_raster_acquisition_service.py | 3 +- .../thematic_raster_analysis_service.py | 1 - backend/app/services/tile_manifest_service.py | 485 ++++++++++++++++++ backend/app/utils/geometry.py | 103 +++- backend/scripts/cleanup_demo_artifacts.py | 11 +- backend/tests/test_analysis_job_queue.py | 2 +- backend/tests/test_area_crs_semantics.py | 159 ++++++ .../tests/test_dataset_consumption_gate.py | 11 + ...st_flood_hazard_selection_data_coverage.py | 4 +- backend/tests/test_model_asset_catalog.py | 38 +- backend/tests/test_raster_cell_selection.py | 6 +- .../tests/test_raster_operations_service.py | 103 +++- .../test_segmentation_configured_models.py | 36 +- .../test_small_selection_raster_analysis.py | 18 +- ...sprint100_segmentation_manifest_handoff.py | 6 +- .../tests/test_sprint106_map_bbox_extract.py | 3 +- ...print122_raster_upload_metadata_mapping.py | 19 +- ...sprint186_map_first_geographic_explorer.py | 3 +- .../test_sprint192_regional_map_state.py | 2 +- .../test_sprint194_regional_timeseries.py | 3 +- .../test_sprint196_map_orthophoto_analysis.py | 4 + ...est_sprint200_temporal_explorer_handoff.py | 2 +- ...t_sprint205_agricultural_parcel_history.py | 1 - .../tests/test_sprint213_thematic_rasters.py | 1 - .../test_sprint223_governed_grb_refresh.py | 2 - .../test_sprint236_bathymetry_expansion.py | 29 +- ...t_sprint237_flanders_thematic_on_demand.py | 2 +- .../test_sprint239_bounded_grb_acquisition.py | 2 +- .../test_sprint241_spw_bathymetry_raster.py | 1 - .../test_sprint7a_persistence_foundation.py | 48 +- .../tests/test_sprint8b_yolo_foundation.py | 48 +- ...test_sprint93_export_handoff_completion.py | 2 +- backend/tests/test_storage_service.py | 114 ++++ backend/tests/test_tile_manifest_binding.py | 210 ++++++++ docs/API_CONTRACTS.md | 137 ++++- scripts/audit_api_contracts.py | 2 + scripts/smoke_contracts.py | 10 +- 63 files changed, 2335 insertions(+), 312 deletions(-) create mode 100644 backend/app/services/tile_manifest_service.py create mode 100644 backend/tests/test_area_crs_semantics.py create mode 100644 backend/tests/test_tile_manifest_binding.py diff --git a/backend/app/api/routes/analysis.py b/backend/app/api/routes/analysis.py index 6eb771dc..cd3ad881 100644 --- a/backend/app/api/routes/analysis.py +++ b/backend/app/api/routes/analysis.py @@ -1,8 +1,9 @@ from __future__ import annotations -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Request from sqlalchemy.orm import Session +from app.api.guest_scope import assert_guest_project_scope from app.core.errors import AppError from app.db.session import get_db from app.models import Dataset @@ -18,12 +19,15 @@ router = APIRouter(prefix="/analysis", tags=["analysis"]) @router.post("/change-detection", response_model=Envelope[JobRead]) def run_change_detection( payload: ChangeDetectionRequest, + request: Request, db: Session = Depends(get_db), ) -> dict: source_dataset = db.get(Dataset, payload.source_dataset_id) if not source_dataset: raise AppError(code="DATASET_NOT_FOUND", message="Source dataset not found", status_code=404) + assert_guest_project_scope(request, source_dataset.project_id) ChangeDetectionService._get_project_vector_dataset(db, payload.source_dataset_id, source_dataset.project_id, "Source") + ChangeDetectionService._get_project_vector_dataset(db, payload.target_dataset_id, source_dataset.project_id, "Target") job = JobService.run_sync_job( db=db, project_id=source_dataset.project_id, diff --git a/backend/app/api/routes/datasets.py b/backend/app/api/routes/datasets.py index 5df2f053..677097a3 100644 --- a/backend/app/api/routes/datasets.py +++ b/backend/app/api/routes/datasets.py @@ -5,13 +5,13 @@ from datetime import datetime from typing import Any from uuid import UUID -from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Response +from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, Response from fastapi import UploadFile from sqlalchemy.orm import Session -from app.models import Area, Project - +from app.core.config import get_settings from app.core.errors import AppError from app.db.session import get_db +from app.models import Area, Project from app.schemas import ( BathymetryPartitionFinalizationResult, BathymetrySourceProbeRead, @@ -1172,11 +1172,14 @@ def raster_tile_dataset( project_id: UUID, dataset_id: UUID, payload: RasterTileRequest, + request: Request, db: Session = Depends(get_db), ): dataset = DatasetService.get_dataset(db, dataset_id) if dataset.project_id != project_id: raise HTTPException(status_code=404, detail="Dataset not found") + principal = getattr(request.state, "auth_principal", None) + guest_max_tiles = get_settings().yolo_max_tiles if getattr(principal, "role", None) == "guest" else None job = _run_job_sync( db=db, project_id=project_id, @@ -1189,6 +1192,7 @@ def raster_tile_dataset( tile_size=payload.tile_size, overlap=payload.overlap, output_name=payload.output_name, + max_tiles=guest_max_tiles, ), ) return envelope(job) diff --git a/backend/app/models.py b/backend/app/models.py index 5bf5a522..8fd4bbf1 100644 --- a/backend/app/models.py +++ b/backend/app/models.py @@ -1 +1 @@ -from app.models import * +from app.models import * # noqa: F403 - legacy compatibility shim re-exports the package API diff --git a/backend/app/schemas/area.py b/backend/app/schemas/area.py index 084e0102..5f5e02b1 100644 --- a/backend/app/schemas/area.py +++ b/backend/app/schemas/area.py @@ -14,6 +14,7 @@ class AreaCreate(BaseModel): class AreaUpdate(BaseModel): name: str | None = None + geometry: dict | None = None crs: str | None = None diff --git a/backend/app/schemas/detection.py b/backend/app/schemas/detection.py index add75ca0..22889ead 100644 --- a/backend/app/schemas/detection.py +++ b/backend/app/schemas/detection.py @@ -42,6 +42,10 @@ class ModelAssetRead(BaseModel): size_bytes: int sha256: str active: bool + runtime_available: bool + runtime_status: str + governed_validation_status: str + promotion_status: str status: str limitation_message: str will_download_models: bool = False diff --git a/backend/app/services/aoi_operation_executor.py b/backend/app/services/aoi_operation_executor.py index dc521549..12cf55f1 100644 --- a/backend/app/services/aoi_operation_executor.py +++ b/backend/app/services/aoi_operation_executor.py @@ -12,7 +12,10 @@ from app.schemas.spw_terrain import SpwTerrainAcquireRequest from app.schemas.official_vector import OfficialVectorAcquireRequest from app.schemas.flood_hazard import FloodHazardAcquireRequest from app.schemas.thematic_raster import ThematicRasterAcquireRequest -from app.schemas.bathymetry import BathymetryProfileAcquireRequest, MdkBathymetryAcquireRequest +from app.schemas.bathymetry import ( + BathymetryProfileAcquireRequest, + MdkBathymetryAcquireRequest, +) from app.schemas.job import JobCreate from app.schemas.operations import VectorSelectionBBox from app.schemas.orthophoto import OrthophotoAcquireRequest @@ -20,12 +23,20 @@ from app.services.aoi_operation_service import AoiOperationService from app.services.grb_acquisition_service import GrbAcquisitionService from app.services.dhmv_acquisition_service import DhmvAcquisitionService from app.services.spw_terrain_service import SpwTerrainService -from app.services.official_vector_acquisition_service import OfficialVectorAcquisitionService +from app.services.official_vector_acquisition_service import ( + OfficialVectorAcquisitionService, +) from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService -from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService +from app.services.thematic_raster_acquisition_service import ( + ThematicRasterAcquisitionService, +) from app.services.walous_land_cover_service import WalousLandCoverService -from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService -from app.services.mdk_bathymetry_acquisition_service import MdkBathymetryAcquisitionService +from app.services.bathymetry_profile_acquisition_service import ( + BathymetryProfileAcquisitionService, +) +from app.services.mdk_bathymetry_acquisition_service import ( + MdkBathymetryAcquisitionService, +) from app.services.job_service import JobService from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService @@ -40,65 +51,202 @@ class AoiOperationExecutor: AoiOperationService._refresh_parent(db, operation_id) return AoiOperationService.read(db, project_id, operation_id) operation = db.get(AoiOperation, operation_id) - child = JobService.create_job(db, JobCreate( - job_type=f"aoi.{operation.operation_type}.partition", - project_id=project_id, - parameters_json={ - "aoi_operation_id": str(operation_id), - "partition_id": str(partition.id), - "partition_key": partition.partition_key, - "provider_key": partition.provider_key, - "product_key": partition.product_key, - }, - )) + child = JobService.create_job( + db, + JobCreate( + job_type=f"aoi.{operation.operation_type}.partition", + project_id=project_id, + parameters_json={ + "aoi_operation_id": str(operation_id), + "partition_id": str(partition.id), + "partition_key": partition.partition_key, + "provider_key": partition.provider_key, + "product_key": partition.product_key, + }, + ), + ) partition = db.get(AoiOperationPartition, partition.id) partition.child_job_id = child.id - db.add(partition); db.commit() + db.add(partition) + db.commit() JobService.mark_running(db, child.id) try: - result = AoiOperationExecutor._dispatch(db, project_id, operation, partition) - output_id = result.get("output_dataset_id") if isinstance(result, dict) else None - JobService.mark_success(db, child.id, result=result, output_dataset_id=UUID(str(output_id)) if output_id else None) - return AoiOperationService.complete(db, project_id, operation_id, partition.id, result) + result = AoiOperationExecutor._dispatch( + db, project_id, operation, partition + ) + output_id = ( + result.get("output_dataset_id") if isinstance(result, dict) else None + ) + JobService.mark_success( + db, + child.id, + result=result, + output_dataset_id=UUID(str(output_id)) if output_id else None, + ) + return AoiOperationService.complete( + db, project_id, operation_id, partition.id, result + ) except AppError as exc: - JobService.mark_failed(db, child.id, exc.message, {"code": exc.code, "details": exc.details}) - return AoiOperationService.fail(db, project_id, operation_id, partition.id, exc.message, AoiOperationExecutor._retryable(exc), {"code": exc.code, "details": exc.details}) + JobService.mark_failed( + db, child.id, exc.message, {"code": exc.code, "details": exc.details} + ) + return AoiOperationService.fail( + db, + project_id, + operation_id, + partition.id, + exc.message, + AoiOperationExecutor._retryable(exc), + {"code": exc.code, "details": exc.details}, + ) except Exception: try: db.rollback() - JobService.mark_failed(db, child.id, "Unexpected partition execution error", {"code": "AOI_PARTITION_INTERNAL_ERROR"}) + JobService.mark_failed( + db, + child.id, + "Unexpected partition execution error", + {"code": "AOI_PARTITION_INTERNAL_ERROR"}, + ) finally: - AoiOperationService.fail(db, project_id, operation_id, partition.id, "Unexpected partition execution error", True, {"code": "AOI_PARTITION_INTERNAL_ERROR"}) + AoiOperationService.fail( + db, + project_id, + operation_id, + partition.id, + "Unexpected partition execution error", + True, + {"code": "AOI_PARTITION_INTERNAL_ERROR"}, + ) raise @staticmethod - def _dispatch(db, project_id: UUID, operation: AoiOperation, partition: AoiOperationPartition) -> dict: + def _dispatch( + db, project_id: UUID, operation: AoiOperation, partition: AoiOperationPartition + ) -> dict: geometry = to_shape(partition.geometry) min_x, min_y, max_x, max_y = geometry.bounds - bbox = VectorSelectionBBox(min_x=min_x, min_y=min_y, max_x=max_x, max_y=max_y, crs="EPSG:4326") - force_refresh = bool((operation.request_json or {}).get("parameters_json", {}).get("force_refresh", False)) + bbox = VectorSelectionBBox( + min_x=min_x, min_y=min_y, max_x=max_x, max_y=max_y, crs="EPSG:4326" + ) + force_refresh = bool( + (operation.request_json or {}) + .get("parameters_json", {}) + .get("force_refresh", False) + ) if partition.provider_key == "grb": - return GrbAcquisitionService.acquire(db, project_id, GrbAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh)) + return GrbAcquisitionService.acquire( + db, + project_id, + GrbAcquireRequest( + bbox=bbox, + area_id=operation.area_id, + product_key=partition.product_key, + force_refresh=force_refresh, + ), + ) if partition.provider_key == "orthophoto": - return OrthophotoAcquisitionService.acquire(db, project_id, OrthophotoAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh)) + return OrthophotoAcquisitionService.acquire( + db, + project_id, + OrthophotoAcquireRequest( + bbox=bbox, + area_id=operation.area_id, + product_key=partition.product_key, + force_refresh=force_refresh, + ), + ) if partition.provider_key == "dhmv": - return DhmvAcquisitionService.acquire(db, project_id, DhmvAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh)) + return DhmvAcquisitionService.acquire( + db, + project_id, + DhmvAcquireRequest( + bbox=bbox, + area_id=operation.area_id, + product_key=partition.product_key, + force_refresh=force_refresh, + ), + ) if partition.provider_key == "spw_terrain": - return SpwTerrainService.acquire(db, project_id, SpwTerrainAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh)) + return SpwTerrainService.acquire( + db, + project_id, + SpwTerrainAcquireRequest( + bbox=bbox, + area_id=operation.area_id, + product_key=partition.product_key, + force_refresh=force_refresh, + ), + ) if partition.provider_key == "official_vector": - return OfficialVectorAcquisitionService.acquire(db, project_id, OfficialVectorAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh)) + return OfficialVectorAcquisitionService.acquire( + db, + project_id, + OfficialVectorAcquireRequest( + bbox=bbox, + area_id=operation.area_id, + product_key=partition.product_key, + force_refresh=force_refresh, + ), + ) if partition.provider_key == "flood_hazard": - return FloodHazardAcquisitionService.acquire(db, project_id, FloodHazardAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh)) + return FloodHazardAcquisitionService.acquire( + db, + project_id, + FloodHazardAcquireRequest( + bbox=bbox, + area_id=operation.area_id, + product_key=partition.product_key, + force_refresh=force_refresh, + ), + ) if partition.provider_key == "thematic_raster": - return ThematicRasterAcquisitionService.acquire(db, project_id, ThematicRasterAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh)) + return ThematicRasterAcquisitionService.acquire( + db, + project_id, + ThematicRasterAcquireRequest( + bbox=bbox, + area_id=operation.area_id, + product_key=partition.product_key, + force_refresh=force_refresh, + ), + ) if partition.provider_key == "walous": - return WalousLandCoverService.acquire(db, project_id, ThematicRasterAcquireRequest(bbox=bbox, area_id=operation.area_id, product_key=partition.product_key, force_refresh=force_refresh)) + return WalousLandCoverService.acquire( + db, + project_id, + ThematicRasterAcquireRequest( + bbox=bbox, + area_id=operation.area_id, + product_key=partition.product_key, + force_refresh=force_refresh, + ), + ) if partition.provider_key == "bathymetry_profiles": - return BathymetryProfileAcquisitionService.acquire(db, project_id, BathymetryProfileAcquireRequest(bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh)) + return BathymetryProfileAcquisitionService.acquire( + db, + project_id, + BathymetryProfileAcquireRequest( + bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh + ), + ) if partition.provider_key == "mdk_bathymetry": - return MdkBathymetryAcquisitionService.acquire(db, project_id, MdkBathymetryAcquireRequest(bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh)) - raise AppError(code="AOI_PROVIDER_UNSUPPORTED", message="No governed AOI executor is registered for this provider", details={"provider_key": partition.provider_key}, status_code=422) + return MdkBathymetryAcquisitionService.acquire( + db, + project_id, + MdkBathymetryAcquireRequest( + bbox=bbox, area_id=operation.area_id, force_refresh=force_refresh + ), + ) + raise AppError( + code="AOI_PROVIDER_UNSUPPORTED", + message="No governed AOI executor is registered for this provider", + details={"provider_key": partition.provider_key}, + status_code=422, + ) @staticmethod def _retryable(error: AppError) -> bool: - return error.status_code >= 500 or error.code.endswith(("TIMEOUT", "UNAVAILABLE", "TLS_ERROR")) + return error.status_code >= 500 or error.code.endswith( + ("TIMEOUT", "UNAVAILABLE", "TLS_ERROR") + ) diff --git a/backend/app/services/aoi_operation_service.py b/backend/app/services/aoi_operation_service.py index 16fa3290..88f7cd6d 100644 --- a/backend/app/services/aoi_operation_service.py +++ b/backend/app/services/aoi_operation_service.py @@ -22,8 +22,11 @@ class AoiOperationService: _to_metric = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True) _to_wgs84 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True) SCOPE_AREA_NAMES = { - "belgium": "Belgium land", "flanders": "Flanders", "wallonia": "Wallonia", - "brussels": "Brussels-Capital Region", "belgian_north_sea": "Belgian part of the North Sea", + "belgium": "Belgium land", + "flanders": "Flanders", + "wallonia": "Wallonia", + "brussels": "Brussels-Capital Region", + "belgian_north_sea": "Belgian part of the North Sea", "territorial_sea": "Belgian territorial sea (0-12 nautical miles)", "exclusive_economic_zone": "Belgian exclusive economic zone beyond territorial sea", "continental_shelf": "Belgian continental shelf beyond territorial sea", @@ -32,13 +35,19 @@ class AoiOperationService: @staticmethod def create(db, project_id: UUID, payload: AoiOperationCreate) -> dict: if db.get(Project, project_id) is None: - raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + raise AppError( + code="PROJECT_NOT_FOUND", message="Project not found", status_code=404 + ) geometry = AoiOperationService._resolve_geometry(db, project_id, payload) if payload.coverage_zone: - geometry = AoiOperationService._clip_to_zone(db, project_id, geometry, payload.coverage_zone) + geometry = AoiOperationService._clip_to_zone( + db, project_id, geometry, payload.coverage_zone + ) geometry = AoiOperationService._as_multipolygon(geometry) metric_geometry = transform(AoiOperationService._to_metric.transform, geometry) - partition_side_m = AoiOperationService._partition_side(payload.provider_key, payload.max_partition_side_m) + partition_side_m = AoiOperationService._partition_side( + payload.provider_key, payload.max_partition_side_m + ) cells = AoiOperationService._partition(metric_geometry, partition_side_m) operation_id = uuid4() now = datetime.now(timezone.utc) @@ -53,7 +62,9 @@ class AoiOperationService: plan_json={ "partition_strategy": "epsg31370_square_grid_intersection_v1", "max_partition_side_m": partition_side_m, - "budget_source": "governed_provider_registry" if payload.max_partition_side_m is None else "stricter_operator_override", + "budget_source": "governed_provider_registry" + if payload.max_partition_side_m is None + else "stricter_operator_override", "partition_count": len(cells), "provider_key": payload.provider_key, "product_key": payload.product_key, @@ -65,13 +76,21 @@ class AoiOperationService: wgs84 = transform(AoiOperationService._to_wgs84.transform, cell) wgs84 = AoiOperationService._as_multipolygon(wgs84) digest = sha256(wgs84.wkb).hexdigest()[:20] - db.add(AoiOperationPartition( - id=uuid4(), operation_id=operation_id, - partition_key=f"{payload.provider_key}:{payload.product_key}:{ordinal:05d}:{digest}", - provider_key=payload.provider_key, product_key=payload.product_key, - ordinal=ordinal, status="queued", geometry=from_shape(wgs84, srid=4326), - attempt_count=0, max_attempts=payload.max_attempts, created_at=now, - )) + db.add( + AoiOperationPartition( + id=uuid4(), + operation_id=operation_id, + partition_key=f"{payload.provider_key}:{payload.product_key}:{ordinal:05d}:{digest}", + provider_key=payload.provider_key, + product_key=payload.product_key, + ordinal=ordinal, + status="queued", + geometry=from_shape(wgs84, srid=4326), + attempt_count=0, + max_attempts=payload.max_attempts, + created_at=now, + ) + ) db.commit() return AoiOperationService.read(db, project_id, operation_id) @@ -79,13 +98,32 @@ class AoiOperationService: def _clip_to_zone(db, project_id: UUID, geometry, zone: str): area_name = AoiOperationService.SCOPE_AREA_NAMES.get(zone) if area_name is None: - raise AppError(code="AOI_COVERAGE_ZONE_UNSUPPORTED", message="Unknown governed coverage zone", details={"coverage_zone": zone}, status_code=422) - scope = db.query(Area).filter(Area.project_id == project_id, Area.name == area_name).first() + raise AppError( + code="AOI_COVERAGE_ZONE_UNSUPPORTED", + message="Unknown governed coverage zone", + details={"coverage_zone": zone}, + status_code=422, + ) + scope = ( + db.query(Area) + .filter(Area.project_id == project_id, Area.name == area_name) + .first() + ) if scope is None: - raise AppError(code="AOI_COVERAGE_ZONE_NOT_MATERIALIZED", message="The governed coverage-zone geometry is not persisted in this project", details={"coverage_zone": zone}, status_code=409) + raise AppError( + code="AOI_COVERAGE_ZONE_NOT_MATERIALIZED", + message="The governed coverage-zone geometry is not persisted in this project", + details={"coverage_zone": zone}, + status_code=409, + ) clipped = geometry.intersection(to_shape(scope.geometry)) if clipped.is_empty: - raise AppError(code="AOI_OUTSIDE_PROVIDER_ZONE", message="The AOI does not intersect the provider coverage zone", details={"coverage_zone": zone}, status_code=422) + raise AppError( + code="AOI_OUTSIDE_PROVIDER_ZONE", + message="The AOI does not intersect the provider coverage zone", + details={"coverage_zone": zone}, + status_code=422, + ) return clipped @staticmethod @@ -94,19 +132,30 @@ class AoiOperationService: return MultiPolygon([geometry]) if isinstance(geometry, MultiPolygon): return geometry - polygons = [part for part in getattr(geometry, "geoms", []) if isinstance(part, Polygon)] + polygons = [ + part for part in getattr(geometry, "geoms", []) if isinstance(part, Polygon) + ] if not polygons: - raise AppError(code="AOI_GEOMETRY_EMPTY", message="AOI contains no polygonal area after clipping", status_code=422) + raise AppError( + code="AOI_GEOMETRY_EMPTY", + message="AOI contains no polygonal area after clipping", + status_code=422, + ) return MultiPolygon(polygons) @staticmethod def _partition_side(provider_key: str, requested: float | None) -> float: settings = get_settings() - def raster_side(max_side_m: float, max_pixels: int, resolution_m: float) -> float: + + def raster_side( + max_side_m: float, max_pixels: int, resolution_m: float + ) -> float: # Keep every square grid cell within both the provider's spatial # extent limit and its decoded-pixel budget. The small safety # margin absorbs ceil/edge rounding in the acquisition services. - pixel_limited_side = math.sqrt(float(max_pixels)) * float(resolution_m) * 0.99 + pixel_limited_side = ( + math.sqrt(float(max_pixels)) * float(resolution_m) * 0.99 + ) return min(float(max_side_m), pixel_limited_side) budgets = { @@ -142,22 +191,37 @@ class AoiOperationService: "mdk_bathymetry": 20_000.0, } if provider_key not in budgets: - raise AppError(code="AOI_PROVIDER_UNSUPPORTED", message="No governed partition budget is registered for this provider", details={"provider_key": provider_key}, status_code=422) + raise AppError( + code="AOI_PROVIDER_UNSUPPORTED", + message="No governed partition budget is registered for this provider", + details={"provider_key": provider_key}, + status_code=422, + ) governed = budgets[provider_key] return min(governed, float(requested)) if requested is not None else governed @staticmethod def _resolve_geometry(db, project_id: UUID, payload: AoiOperationCreate): if (payload.area_id is None) == (payload.bbox is None): - raise AppError(code="AOI_SELECTION_REQUIRED", message="Provide exactly one area_id or bbox", status_code=422) + raise AppError( + code="AOI_SELECTION_REQUIRED", + message="Provide exactly one area_id or bbox", + status_code=422, + ) if payload.area_id is not None: area = db.get(Area, payload.area_id) if area is None or area.project_id != project_id: - raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) + raise AppError( + code="AREA_NOT_FOUND", message="Area not found", status_code=404 + ) return to_shape(area.geometry) bbox = payload.bbox if bbox is None or bbox.crs != "EPSG:4326": - raise AppError(code="INVALID_AOI_CRS", message="AOI bbox must use EPSG:4326", status_code=422) + raise AppError( + code="INVALID_AOI_CRS", + message="AOI bbox must use EPSG:4326", + status_code=422, + ) return box(bbox.min_x, bbox.min_y, bbox.max_x, bbox.max_y) @staticmethod @@ -166,11 +230,26 @@ class AoiOperationService: columns = max(1, math.ceil((max_x - min_x) / side_m)) rows = max(1, math.ceil((max_y - min_y) / side_m)) if columns * rows > AoiOperationService.MAX_PARTITIONS: - raise AppError(code="AOI_PARTITION_LIMIT_EXCEEDED", message="AOI requires too many bounded partitions", details={"candidate_count": columns * rows, "max_partitions": AoiOperationService.MAX_PARTITIONS}, status_code=422) + raise AppError( + code="AOI_PARTITION_LIMIT_EXCEEDED", + message="AOI requires too many bounded partitions", + details={ + "candidate_count": columns * rows, + "max_partitions": AoiOperationService.MAX_PARTITIONS, + }, + status_code=422, + ) partitions = [] for row in range(rows): for column in range(columns): - clipped = geometry.intersection(box(min_x + column * side_m, min_y + row * side_m, min(min_x + (column + 1) * side_m, max_x), min(min_y + (row + 1) * side_m, max_y))) + clipped = geometry.intersection( + box( + min_x + column * side_m, + min_y + row * side_m, + min(min_x + (column + 1) * side_m, max_x), + min(min_y + (row + 1) * side_m, max_y), + ) + ) if not clipped.is_empty and clipped.area > 0: partitions.append(clipped) return partitions @@ -179,98 +258,224 @@ class AoiOperationService: def read(db, project_id: UUID, operation_id: UUID) -> dict: operation = db.get(AoiOperation, operation_id) if operation is None or operation.project_id != project_id: - raise AppError(code="AOI_OPERATION_NOT_FOUND", message="AOI operation not found", status_code=404) - partitions = db.query(AoiOperationPartition).filter(AoiOperationPartition.operation_id == operation_id).order_by(AoiOperationPartition.ordinal).all() + raise AppError( + code="AOI_OPERATION_NOT_FOUND", + message="AOI operation not found", + status_code=404, + ) + partitions = ( + db.query(AoiOperationPartition) + .filter(AoiOperationPartition.operation_id == operation_id) + .order_by(AoiOperationPartition.ordinal) + .all() + ) counts = Counter(partition.status for partition in partitions) complete = counts["success"] + counts["skipped"] return { - "id": operation.id, "project_id": operation.project_id, "area_id": operation.area_id, - "parent_job_id": operation.parent_job_id, "operation_type": operation.operation_type, - "status": operation.status, "request_json": operation.request_json, "plan_json": operation.plan_json, - "result_json": operation.result_json, "error_message": operation.error_message, + "id": operation.id, + "project_id": operation.project_id, + "area_id": operation.area_id, + "parent_job_id": operation.parent_job_id, + "operation_type": operation.operation_type, + "status": operation.status, + "request_json": operation.request_json, + "plan_json": operation.plan_json, + "result_json": operation.result_json, + "error_message": operation.error_message, "progress": round(complete / len(partitions), 6) if partitions else 0.0, - "partition_counts": dict(counts), "partitions": partitions, - "created_at": operation.created_at, "started_at": operation.started_at, "finished_at": operation.finished_at, + "partition_counts": dict(counts), + "partitions": partitions, + "created_at": operation.created_at, + "started_at": operation.started_at, + "finished_at": operation.finished_at, } @staticmethod def list(db, project_id: UUID, limit: int = 50) -> dict: - rows = db.query(AoiOperation).filter(AoiOperation.project_id == project_id).order_by(AoiOperation.created_at.desc()).limit(limit).all() - return {"items": [AoiOperationService.read(db, project_id, row.id) for row in rows], "total": len(rows)} + rows = ( + db.query(AoiOperation) + .filter(AoiOperation.project_id == project_id) + .order_by(AoiOperation.created_at.desc()) + .limit(limit) + .all() + ) + return { + "items": [AoiOperationService.read(db, project_id, row.id) for row in rows], + "total": len(rows), + } @staticmethod def claim_next(db, project_id: UUID, operation_id: UUID): operation = db.get(AoiOperation, operation_id) if operation is None or operation.project_id != project_id: - raise AppError(code="AOI_OPERATION_NOT_FOUND", message="AOI operation not found", status_code=404) - partition = db.query(AoiOperationPartition).filter(AoiOperationPartition.operation_id == operation_id, AoiOperationPartition.status == "queued").order_by(AoiOperationPartition.ordinal).with_for_update(skip_locked=True).first() + raise AppError( + code="AOI_OPERATION_NOT_FOUND", + message="AOI operation not found", + status_code=404, + ) + partition = ( + db.query(AoiOperationPartition) + .filter( + AoiOperationPartition.operation_id == operation_id, + AoiOperationPartition.status == "queued", + ) + .order_by(AoiOperationPartition.ordinal) + .with_for_update(skip_locked=True) + .first() + ) if partition is None: return None now = datetime.now(timezone.utc) - partition.status = "running"; partition.started_at = now; partition.attempt_count += 1; partition.error_message = None - operation.status = "running"; operation.started_at = operation.started_at or now - db.add(partition); db.add(operation); db.commit(); db.refresh(partition) + partition.status = "running" + partition.started_at = now + partition.attempt_count += 1 + partition.error_message = None + operation.status = "running" + operation.started_at = operation.started_at or now + db.add(partition) + db.add(operation) + db.commit() + db.refresh(partition) return partition @staticmethod - def checkpoint(db, project_id: UUID, operation_id: UUID, partition_id: UUID, checkpoint: dict): - partition = AoiOperationService._partition_row(db, project_id, operation_id, partition_id) + def checkpoint( + db, project_id: UUID, operation_id: UUID, partition_id: UUID, checkpoint: dict + ): + partition = AoiOperationService._partition_row( + db, project_id, operation_id, partition_id + ) if partition.status != "running": - raise AppError(code="AOI_PARTITION_NOT_RUNNING", message="Only a running partition can be checkpointed", status_code=409) - partition.checkpoint_json = checkpoint; db.add(partition); db.commit(); db.refresh(partition) + raise AppError( + code="AOI_PARTITION_NOT_RUNNING", + message="Only a running partition can be checkpointed", + status_code=409, + ) + partition.checkpoint_json = checkpoint + db.add(partition) + db.commit() + db.refresh(partition) return partition @staticmethod - def complete(db, project_id: UUID, operation_id: UUID, partition_id: UUID, result: dict, skipped: bool = False): - partition = AoiOperationService._partition_row(db, project_id, operation_id, partition_id) + def complete( + db, + project_id: UUID, + operation_id: UUID, + partition_id: UUID, + result: dict, + skipped: bool = False, + ): + partition = AoiOperationService._partition_row( + db, project_id, operation_id, partition_id + ) if partition.status == "success" or partition.status == "skipped": return AoiOperationService.read(db, project_id, operation_id) if partition.status != "running": - raise AppError(code="AOI_PARTITION_NOT_RUNNING", message="Only a running partition can complete", status_code=409) - partition.status = "skipped" if skipped else "success"; partition.result_json = result; partition.finished_at = datetime.now(timezone.utc) - db.add(partition); db.commit(); AoiOperationService._refresh_parent(db, operation_id) + raise AppError( + code="AOI_PARTITION_NOT_RUNNING", + message="Only a running partition can complete", + status_code=409, + ) + partition.status = "skipped" if skipped else "success" + partition.result_json = result + partition.finished_at = datetime.now(timezone.utc) + db.add(partition) + db.commit() + AoiOperationService._refresh_parent(db, operation_id) return AoiOperationService.read(db, project_id, operation_id) @staticmethod - def fail(db, project_id: UUID, operation_id: UUID, partition_id: UUID, message: str, retryable: bool, details: dict): - partition = AoiOperationService._partition_row(db, project_id, operation_id, partition_id) - partition.error_message = message; partition.result_json = {"details": details} - partition.status = "queued" if retryable and partition.attempt_count < partition.max_attempts else "failed" - partition.finished_at = None if partition.status == "queued" else datetime.now(timezone.utc) - db.add(partition); db.commit(); AoiOperationService._refresh_parent(db, operation_id) + def fail( + db, + project_id: UUID, + operation_id: UUID, + partition_id: UUID, + message: str, + retryable: bool, + details: dict, + ): + partition = AoiOperationService._partition_row( + db, project_id, operation_id, partition_id + ) + partition.error_message = message + partition.result_json = {"details": details} + partition.status = ( + "queued" + if retryable and partition.attempt_count < partition.max_attempts + else "failed" + ) + partition.finished_at = ( + None if partition.status == "queued" else datetime.now(timezone.utc) + ) + db.add(partition) + db.commit() + AoiOperationService._refresh_parent(db, operation_id) return AoiOperationService.read(db, project_id, operation_id) @staticmethod def _partition_row(db, project_id, operation_id, partition_id): - operation = db.get(AoiOperation, operation_id); partition = db.get(AoiOperationPartition, partition_id) - if operation is None or operation.project_id != project_id or partition is None or partition.operation_id != operation_id: - raise AppError(code="AOI_PARTITION_NOT_FOUND", message="AOI partition not found", status_code=404) + operation = db.get(AoiOperation, operation_id) + partition = db.get(AoiOperationPartition, partition_id) + if ( + operation is None + or operation.project_id != project_id + or partition is None + or partition.operation_id != operation_id + ): + raise AppError( + code="AOI_PARTITION_NOT_FOUND", + message="AOI partition not found", + status_code=404, + ) return partition @staticmethod def _refresh_parent(db, operation_id): operation = db.get(AoiOperation, operation_id) - partitions = db.query(AoiOperationPartition).filter(AoiOperationPartition.operation_id == operation_id).order_by(AoiOperationPartition.ordinal).all() + partitions = ( + db.query(AoiOperationPartition) + .filter(AoiOperationPartition.operation_id == operation_id) + .order_by(AoiOperationPartition.ordinal) + .all() + ) statuses = [partition.status for partition in partitions] output_dataset_ids = [] for partition in partitions: - output_id = (partition.result_json or {}).get("output_dataset_id") if isinstance(partition.result_json, dict) else None + output_id = ( + (partition.result_json or {}).get("output_dataset_id") + if isinstance(partition.result_json, dict) + else None + ) if output_id and str(output_id) not in output_dataset_ids: output_dataset_ids.append(str(output_id)) operation.result_json = { "partition_count": len(partitions), - "completed_partition_count": sum(status in {"success", "skipped"} for status in statuses), + "completed_partition_count": sum( + status in {"success", "skipped"} for status in statuses + ), "failed_partition_count": statuses.count("failed"), "output_dataset_ids": output_dataset_ids, "merge_contract": "source_aware_spatial_union", "vector_deduplication": "source_feature_id_then_geometry", "raster_deduplication": "governed_mosaic_grid", - "complete_coverage": bool(statuses) and all(status in {"success", "skipped"} for status in statuses), + "complete_coverage": bool(statuses) + and all(status in {"success", "skipped"} for status in statuses), } now = datetime.now(timezone.utc) if statuses and all(status in {"success", "skipped"} for status in statuses): - operation.status = "success"; operation.finished_at = now; operation.error_message = None - elif "failed" in statuses and not any(status in {"queued", "running"} for status in statuses): - operation.status = "partial" if any(status in {"success", "skipped"} for status in statuses) else "failed"; operation.finished_at = now + operation.status = "success" + operation.finished_at = now + operation.error_message = None + elif "failed" in statuses and not any( + status in {"queued", "running"} for status in statuses + ): + operation.status = ( + "partial" + if any(status in {"success", "skipped"} for status in statuses) + else "failed" + ) + operation.finished_at = now operation.error_message = "One or more bounded source partitions failed; inspect partition evidence." - db.add(operation); db.commit() + db.add(operation) + db.commit() diff --git a/backend/app/services/area_service.py b/backend/app/services/area_service.py index 2eaa572c..62617ef6 100644 --- a/backend/app/services/area_service.py +++ b/backend/app/services/area_service.py @@ -9,7 +9,7 @@ from shapely.geometry import mapping from app.core.errors import AppError from app.models import Area, Dataset, Project, VectorFeature from app.schemas.area import AreaCreate, AreaRead, AreaUpdate -from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_to_multipolygon +from app.utils.geometry import area_m2, geometry_bbox_polygon, normalize_area_to_epsg4326 class AreaService: @@ -126,7 +126,11 @@ class AreaService: raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) try: - multipolygon = normalize_to_multipolygon(payload.geometry) + multipolygon, original_crs = normalize_area_to_epsg4326( + payload.geometry, + payload.crs or "EPSG:4326", + ) + metric_area = area_m2(multipolygon) except ValueError as exc: raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc @@ -134,8 +138,8 @@ class AreaService: project_id=project_id, name=payload.name.strip() or "Unnamed area", geometry=from_shape(multipolygon, srid=4326), - original_crs=payload.crs or "EPSG:4326", - area_m2=area_m2(multipolygon), + original_crs=original_crs, + area_m2=metric_area, bbox=from_shape(geometry_bbox_polygon(multipolygon), srid=4326), ) db.add(area) @@ -157,11 +161,28 @@ class AreaService: raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) changed = False - if payload.name: + if payload.name is not None and payload.name.strip(): area.name = payload.name.strip() or area.name changed = True - if payload.crs: - area.original_crs = payload.crs + if payload.crs is not None and payload.geometry is None: + raise AppError( + code="INVALID_AREA_CRS_UPDATE", + message="crs can only be supplied together with replacement geometry", + status_code=422, + ) + if payload.geometry is not None: + try: + multipolygon, original_crs = normalize_area_to_epsg4326( + payload.geometry, + payload.crs or "EPSG:4326", + ) + metric_area = area_m2(multipolygon) + except ValueError as exc: + raise AppError(code="INVALID_GEOMETRY", message=str(exc), status_code=400) from exc + area.geometry = from_shape(multipolygon, srid=4326) + area.original_crs = original_crs + area.area_m2 = metric_area + area.bbox = from_shape(geometry_bbox_polygon(multipolygon), srid=4326) changed = True if not changed: raise AppError(code="NO_CHANGES", message="No updatable fields provided", status_code=422) diff --git a/backend/app/services/bathymetry_profile_acquisition_service.py b/backend/app/services/bathymetry_profile_acquisition_service.py index 8bc0e0fe..19942b7a 100644 --- a/backend/app/services/bathymetry_profile_acquisition_service.py +++ b/backend/app/services/bathymetry_profile_acquisition_service.py @@ -7,7 +7,7 @@ import math from typing import Any, Callable from urllib.error import HTTPError, URLError from urllib.parse import urlencode -from urllib.request import Request, urlopen +from urllib.request import Request from uuid import UUID from geoalchemy2.shape import to_shape diff --git a/backend/app/services/bathymetry_raster_analysis_service.py b/backend/app/services/bathymetry_raster_analysis_service.py index bb7e1612..80e88c57 100644 --- a/backend/app/services/bathymetry_raster_analysis_service.py +++ b/backend/app/services/bathymetry_raster_analysis_service.py @@ -112,7 +112,6 @@ class BathymetryRasterAnalysisService: try: import numpy as np import rasterio - from rasterio.features import geometry_mask from rasterio.mask import mask except ImportError as exc: raise AppError( diff --git a/backend/app/services/dataset_service.py b/backend/app/services/dataset_service.py index ab6c490f..e10977ea 100644 --- a/backend/app/services/dataset_service.py +++ b/backend/app/services/dataset_service.py @@ -17,6 +17,7 @@ from shapely.geometry import MultiPoint, shape from sqlalchemy.orm import Session from app.core.errors import AppError +from app.core.config import get_settings from app.models import Area, Dataset, DatasetVersion, Project from app.services.data_contract_validation import ( ContractKind, @@ -476,6 +477,29 @@ class DatasetService: content_type=content_type, ) + @classmethod + def _persist_vector_source_evidence_from_path( + cls, + *, + project_id: UUID, + dataset_id: UUID, + original_filename: str, + source_path: str | Path, + content_type: str | None, + ) -> dict[str, Any]: + safe_filename = StorageService._safe_filename(original_filename) + evidence_path = ( + StorageService.dataset_root(str(project_id), str(dataset_id), "vector") + / "provenance" + / f"{dataset_id}_source_{safe_filename}" + ) + return StorageService.persist_file_from_path( + str(evidence_path), + source_path, + original_filename=safe_filename, + content_type=content_type, + ) + @classmethod def _record_vector_source_evidence( cls, @@ -853,6 +877,52 @@ class DatasetService: raise AppError(code="INVALID_UPLOAD", message="Missing file name", status_code=400) return filename + @staticmethod + async def _stage_upload( + *, + project_id: UUID, + dataset_id: uuid.UUID, + dataset_type: str, + filename: str, + file: UploadFile, + ) -> dict[str, Any]: + settings = get_settings() + max_upload_mb = int(settings.max_upload_mb) + if DatasetService._canonical_dataset_type(dataset_type) == "vector": + max_upload_mb = min(max_upload_mb, int(settings.max_in_memory_vector_mb)) + return await StorageService.persist_upload_file( + project_id=str(project_id), + dataset_id=str(dataset_id), + dataset_type=dataset_type, + original_filename=filename, + upload=file, + content_type=file.content_type, + max_bytes=max_upload_mb * 1024 * 1024, + ) + + @staticmethod + def _read_staged_vector_bytes(storage_info: dict[str, Any]) -> bytes: + settings = get_settings() + max_bytes = min( + int(settings.max_upload_mb), + int(settings.max_in_memory_vector_mb), + ) * 1024 * 1024 + path = Path(str(storage_info["storage_path"])) + with path.open("rb") as stream: + content = stream.read(max_bytes + 1) + if len(content) > max_bytes: + StorageService.remove_dataset_file(str(path)) + raise AppError( + code="UPLOAD_TOO_LARGE", + message="Vector upload exceeds the bounded in-memory parsing limit.", + details={ + "max_bytes": max_bytes, + "max_in_memory_vector_mb": max_bytes // (1024 * 1024), + }, + status_code=413, + ) + return content + @staticmethod def list_datasets(db: Session, project_id: UUID, limit: int = 50, offset: int = 0) -> tuple[list[DatasetCreateResponse], int]: total = db.query(Dataset).filter(Dataset.project_id == project_id).count() @@ -1135,15 +1205,15 @@ class DatasetService: status_code=415, ) - raw = await file.read() - storage_info = StorageService.persist_dataset_file( - project_id=str(project_id), - dataset_id=str(dataset_id := uuid.uuid4()), + dataset_id = uuid.uuid4() + storage_info = await DatasetService._stage_upload( + project_id=project_id, + dataset_id=dataset_id, dataset_type=canonical_type, - original_filename=filename, - content=raw, - content_type=file.content_type, + filename=filename, + file=file, ) + raw = DatasetService._read_staged_vector_bytes(storage_info) if canonical_type == "vector" else None metadata: dict[str, Any] = {} vector_payload: dict[str, Any] | None = None @@ -1151,6 +1221,7 @@ class DatasetService: try: status = "validating" if canonical_type == "vector": + assert raw is not None try: text = raw.decode("utf-8") except UnicodeDecodeError as exc: @@ -1319,8 +1390,15 @@ class DatasetService: temporal_granularity=temporal_granularity, source_version=source_version, ) - raw = await file.read() - checksum_sha256 = StorageService.calculate_checksum_sha256(raw) + dataset_id = uuid.uuid4() + storage_info = await DatasetService._stage_upload( + project_id=project_id, + dataset_id=dataset_id, + dataset_type=canonical_type, + filename=filename, + file=file, + ) + checksum_sha256 = storage_info["checksum_sha256"] ingest_key = DatasetService._ingest_key( project_id=project_id, source_key="manual", @@ -1333,6 +1411,7 @@ class DatasetService: ) existing = DatasetService._find_existing_ingest(db, project_id, ingest_key) if existing is not None: + StorageService.remove_dataset_file(storage_info["storage_path"]) return DatasetService._to_response(existing) raw_source_metadata = dict(source_metadata or {}) @@ -1361,8 +1440,7 @@ class DatasetService: } ) - dataset_id = uuid.uuid4() - storage_info: dict[str, Any] | None = None + raw = DatasetService._read_staged_vector_bytes(storage_info) if canonical_type == "vector" else None storage_content = raw source_evidence: dict[str, Any] | None = None imported_at = datetime.now(timezone.utc) @@ -1372,6 +1450,7 @@ class DatasetService: parser_error: tuple[str, str] | None = None try: if canonical_type == "vector": + assert raw is not None try: payload = json.loads(raw.decode("utf-8")) except UnicodeDecodeError as exc: @@ -1394,22 +1473,22 @@ class DatasetService: ) if DatasetService._vector_storage_requires_canonicalization(source_crs): storage_content = DatasetService._canonical_vector_storage_bytes(canonical_vector_payload) - source_evidence = DatasetService._persist_vector_source_evidence( + source_evidence = DatasetService._persist_vector_source_evidence_from_path( project_id=project_id, dataset_id=dataset_id, original_filename=filename, - content=raw, + source_path=storage_info["storage_path"], + content_type=file.content_type, + ) + storage_info = StorageService.persist_dataset_file( + project_id=str(project_id), + dataset_id=str(dataset_id), + dataset_type=canonical_type, + original_filename=filename, + content=storage_content, content_type=file.content_type, ) else: - storage_info = StorageService.persist_dataset_file( - project_id=str(project_id), - dataset_id=str(dataset_id), - dataset_type=canonical_type, - original_filename=filename, - content=raw, - content_type=file.content_type, - ) metadata = extract_raster_metadata(storage_info["storage_path"]) metadata["dataset_type"] = "raster" source_crs = metadata.get("crs") @@ -1422,16 +1501,7 @@ class DatasetService: "processing_code": code, } - if storage_info is None: - storage_info = StorageService.persist_dataset_file( - project_id=str(project_id), - dataset_id=str(dataset_id), - dataset_type=canonical_type, - original_filename=filename, - content=storage_content, - content_type=file.content_type, - ) - computed_storage_checksum_sha256 = StorageService.calculate_checksum_sha256(storage_content) + computed_storage_checksum_sha256 = storage_info["checksum_sha256"] if source_evidence is not None: resolved_source_crs = source_crs or DatasetService.CANONICAL_VECTOR_CRS DatasetService._record_vector_source_evidence( @@ -1502,7 +1572,7 @@ class DatasetService: feature_collection=canonical_vector_payload or {"type": "FeatureCollection", "features": []}, checksum_sha256=storage_info["checksum_sha256"], computed_checksum_sha256=computed_storage_checksum_sha256, - content=storage_content, + content=None, source_registry_id=str(source_registry.id), source_snapshot_id=str(source_snapshot.id), imported_at=imported_at, @@ -1541,8 +1611,8 @@ class DatasetService: bounds=DatasetService._extract_raster_bounds_json(metadata), resolution=resolution, checksum_sha256=storage_info["checksum_sha256"], - computed_checksum_sha256=checksum_sha256, - content=raw, + computed_checksum_sha256=storage_info["checksum_sha256"], + content=None, source_registry_id=str(source_registry.id), source_snapshot_id=str(source_snapshot.id), imported_at=imported_at, diff --git a/backend/app/services/detection_review_service.py b/backend/app/services/detection_review_service.py index a3b9bcc5..ce296aca 100644 --- a/backend/app/services/detection_review_service.py +++ b/backend/app/services/detection_review_service.py @@ -1,7 +1,6 @@ from __future__ import annotations from collections import Counter -from typing import Any from uuid import UUID from sqlalchemy.orm import Session diff --git a/backend/app/services/detection_service.py b/backend/app/services/detection_service.py index 7b6000f5..8556ccaa 100644 --- a/backend/app/services/detection_service.py +++ b/backend/app/services/detection_service.py @@ -33,6 +33,7 @@ from app.services.storage_service import StorageService from app.services.quality_service import QualityService from app.services.runtime_model_provenance_service import RuntimeModelProvenance, RuntimeModelProvenanceService from app.services.temporal_compatibility_service import TemporalCompatibilityService +from app.services.tile_manifest_service import TileManifestService from app.services.yolo_adapter import YoloDetectionAdapter @@ -968,6 +969,18 @@ class DetectionService: yolo_adapter_class: Type[YoloDetectionAdapter], ) -> tuple[list[Detection], dict[str, Any]]: manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles, settings) + dataset = db.get(Dataset, dataset_id) + if dataset is None: + raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404) + manifest_binding = TileManifestService.validate_for_inference( + db, + dataset, + manifest, + manifest_path=tile_manifest_path or "", + settings=settings, + error_prefix="DETECTION", + ) + DetectionService._attach_tile_manifest_binding(analysis_run, job, manifest_binding) model_path = Path(settings.yolo_model_path or "").expanduser() runtime_model_provenance = RuntimeModelProvenanceService.validate_for_production_runtime( db=db, @@ -1075,9 +1088,23 @@ class DetectionService: "tile_edge_truncated_count": len(candidates) - len(edge_filtered_candidates), "duplicate_iou_threshold": float(settings.yolo_duplicate_iou_threshold), "containment_suppression_threshold": float(settings.yolo_containment_nms_threshold), + "tile_manifest_binding": manifest_binding, "runtime_model_provenance": runtime_model_provenance.as_dict(), } + @staticmethod + def _attach_tile_manifest_binding( + analysis_run: AnalysisRun, + job: Job, + binding: dict[str, Any], + ) -> None: + analysis_parameters = dict(analysis_run.parameters_json or {}) + analysis_parameters["tile_manifest_binding"] = dict(binding) + analysis_run.parameters_json = analysis_parameters + job_parameters = dict(job.parameters_json or {}) + job_parameters["tile_manifest_binding"] = dict(binding) + job.parameters_json = job_parameters + @staticmethod def _attach_runtime_model_provenance( analysis_run: AnalysisRun, diff --git a/backend/app/services/dhmv_acquisition_service.py b/backend/app/services/dhmv_acquisition_service.py index d19a00fc..a732ebfe 100644 --- a/backend/app/services/dhmv_acquisition_service.py +++ b/backend/app/services/dhmv_acquisition_service.py @@ -12,7 +12,7 @@ from pathlib import Path from typing import Any, Callable from urllib.error import HTTPError, URLError from urllib.parse import urlencode -from urllib.request import Request, urlopen +from urllib.request import Request from uuid import UUID from geoalchemy2.shape import to_shape diff --git a/backend/app/services/flood_hazard_acquisition_service.py b/backend/app/services/flood_hazard_acquisition_service.py index 29834d00..363773c2 100644 --- a/backend/app/services/flood_hazard_acquisition_service.py +++ b/backend/app/services/flood_hazard_acquisition_service.py @@ -12,7 +12,7 @@ from pathlib import Path from typing import Any, Callable from urllib.error import HTTPError, URLError from urllib.parse import urlencode -from urllib.request import Request, urlopen +from urllib.request import Request from uuid import UUID from xml.etree import ElementTree diff --git a/backend/app/services/flood_hazard_analysis_service.py b/backend/app/services/flood_hazard_analysis_service.py index 570bca88..cf0449f4 100644 --- a/backend/app/services/flood_hazard_analysis_service.py +++ b/backend/app/services/flood_hazard_analysis_service.py @@ -232,7 +232,6 @@ class FloodHazardAnalysisService: try: import numpy as np import rasterio - from rasterio.features import geometry_mask from rasterio.mask import mask except ImportError as exc: raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for flood-hazard analysis", status_code=503) from exc diff --git a/backend/app/services/mdk_bathymetry_acquisition_service.py b/backend/app/services/mdk_bathymetry_acquisition_service.py index 945e5113..7608087f 100644 --- a/backend/app/services/mdk_bathymetry_acquisition_service.py +++ b/backend/app/services/mdk_bathymetry_acquisition_service.py @@ -5,7 +5,7 @@ from datetime import UTC, datetime from typing import Any, Callable from urllib.error import HTTPError, URLError from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit -from urllib.request import Request, urlopen +from urllib.request import Request from uuid import UUID from app.core.config import Settings, get_settings diff --git a/backend/app/services/mdk_bathymetry_probe_service.py b/backend/app/services/mdk_bathymetry_probe_service.py index 6de5c4cb..3f29ceed 100644 --- a/backend/app/services/mdk_bathymetry_probe_service.py +++ b/backend/app/services/mdk_bathymetry_probe_service.py @@ -6,11 +6,12 @@ import ssl from typing import Any, Callable from urllib.error import HTTPError, URLError from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit -from urllib.request import Request, urlopen +from urllib.request import Request from xml.etree import ElementTree from app.core.config import Settings, get_settings from app.schemas.bathymetry import BathymetrySourceProbeRead +from app.services.outbound_request_guard import guarded_opener class MdkBathymetryProbeService: diff --git a/backend/app/services/model_asset_catalog_service.py b/backend/app/services/model_asset_catalog_service.py index bb73011e..ab8391ad 100644 --- a/backend/app/services/model_asset_catalog_service.py +++ b/backend/app/services/model_asset_catalog_service.py @@ -79,11 +79,15 @@ class ModelAssetCatalogService: size_bytes=path.stat().st_size, sha256=ModelAssetCatalogService._sha256(path), active=active_model_path == resolved_path, - status="approved" if active_model_path == resolved_path else "available", + runtime_available=True, + runtime_status="active" if active_model_path == resolved_path else "available", + governed_validation_status="not_verified_by_catalog", + promotion_status="not_verified_by_catalog", + status="runtime_active" if active_model_path == resolved_path else "runtime_available", limitation_message=( - "Approved local runtime model asset. GeoIntel will not download or mutate model weights." + "Active local runtime model asset. Runtime selection is not evidence of governed validation or promotion." if active_model_path == resolved_path - else "Local development model asset. Configure it explicitly before production use." + else "Local runtime model asset. Governed validation and promotion are not established by this catalog." ), will_download_models=False, ) diff --git a/backend/app/services/orthophoto_acquisition_service.py b/backend/app/services/orthophoto_acquisition_service.py index c8ed9450..9c272bc8 100644 --- a/backend/app/services/orthophoto_acquisition_service.py +++ b/backend/app/services/orthophoto_acquisition_service.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any, Callable from urllib.error import HTTPError, URLError from urllib.parse import urlencode -from urllib.request import Request, urlopen +from urllib.request import Request from uuid import UUID from geoalchemy2.shape import to_shape diff --git a/backend/app/services/raster_operations_service.py b/backend/app/services/raster_operations_service.py index bf8f49ba..22376e94 100644 --- a/backend/app/services/raster_operations_service.py +++ b/backend/app/services/raster_operations_service.py @@ -1,8 +1,8 @@ from __future__ import annotations -import json import uuid from datetime import datetime, timezone +from hashlib import sha256 from pathlib import Path from typing import Any @@ -16,6 +16,7 @@ from app.models import Area, Dataset, DatasetVersion from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService from app.services.raster_service import extract_raster_metadata from app.services.storage_service import StorageService +from app.services.tile_manifest_service import TileManifestService, canonical_manifest_json def _import_rasterio(): @@ -70,6 +71,39 @@ class RasterOperationsService: raise AppError(code="DATASET_FILE_MISSING", message="Stored raster file missing", status_code=404) return dataset + @staticmethod + def _validate_storage_checksum(dataset: Dataset) -> None: + """Refuse tiling pixels that no longer match the governed Dataset row.""" + + expected_checksum = str(dataset.checksum_sha256 or "").strip().lower() + governed_artifact = bool(getattr(dataset, "data_contract_key", None)) + valid_checksum = len(expected_checksum) == 64 and all(character in "0123456789abcdef" for character in expected_checksum) + if not valid_checksum: + if expected_checksum or governed_artifact: + raise AppError( + code="DATASET_STORAGE_CHECKSUM_UNVERIFIABLE", + message="Governed raster storage requires a valid SHA-256 checksum before tiling.", + details={"checksum_sha256": dataset.checksum_sha256}, + status_code=409, + ) + return + + digest = sha256() + with Path(str(dataset.storage_path)).open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + actual_checksum = digest.hexdigest() + if actual_checksum != expected_checksum: + raise AppError( + code="DATASET_STORAGE_CHECKSUM_MISMATCH", + message="Raster dataset storage no longer matches its validated checksum.", + details={ + "expected_checksum_sha256": expected_checksum, + "actual_checksum_sha256": actual_checksum, + }, + status_code=409, + ) + @staticmethod def _raster_dependencies() -> tuple[Any, Any]: try: @@ -970,14 +1004,17 @@ class RasterOperationsService: tile_size: int = 512, overlap: int = 64, output_name: str | None = None, + max_tiles: int | None = None, ) -> dict[str, Any]: RasterOperationsService._validate_tile_request(tile_size=tile_size, overlap=overlap) + if max_tiles is not None and max_tiles <= 0: + raise AppError(code="INVALID_PARAMETERS", message="max_tiles must be positive", status_code=400) dataset = RasterOperationsService._load_dataset(db, dataset_id) + RasterOperationsService._validate_storage_checksum(dataset) rasterio, _ = RasterOperationsService._raster_dependencies() tile_set_id = str(uuid.uuid4()) tile_root = StorageService.raster_tiles_root(str(dataset.project_id), str(dataset.id), tile_set_id) - tile_root.mkdir(parents=True, exist_ok=True) manifest_tiles: list[dict[str, Any]] = [] tile_paths: list[str] = [] @@ -997,9 +1034,23 @@ class RasterOperationsService: source_width = int(source.width) source_height = int(source.height) step = max(1, tile_size - overlap) + x_offsets = RasterOperationsService._tile_offsets(source_width, tile_size, step) + y_offsets = RasterOperationsService._tile_offsets(source_height, tile_size, step) + expected_tile_count = len(x_offsets) * len(y_offsets) + if max_tiles is not None and expected_tile_count > max_tiles: + raise AppError( + code="RASTER_TILE_LIMIT_EXCEEDED", + message="Raster tile generation exceeds the guest analysis limit.", + details={ + "expected_tile_count": expected_tile_count, + "max_tiles": max_tiles, + }, + status_code=422, + ) + tile_root.mkdir(parents=True, exist_ok=True) tile_index = 0 - for yoff in range(0, source_height, step): - for xoff in range(0, source_width, step): + for yoff in y_offsets: + for xoff in x_offsets: tile_width = min(tile_size, source_width - xoff) tile_height = min(tile_size, source_height - yoff) if tile_width <= 0 or tile_height <= 0: @@ -1022,6 +1073,7 @@ class RasterOperationsService: tile_dest.write(tile_data) tile_paths.append(str(tile_path)) + tile_integrity = TileManifestService.tile_integrity(tile_path) manifest_tiles.append( { "path": str(tile_path), @@ -1030,6 +1082,7 @@ class RasterOperationsService: "transform": [float(item) for item in transform.to_gdal()], "crs": source_crs, "index": tile_index, + **tile_integrity, }, ) tile_index += 1 @@ -1044,9 +1097,8 @@ class RasterOperationsService: bounds = source_metadata.get("bounds", [0.0, 0.0, 0.0, 0.0]) manifest_crs = source_crs or source_metadata.get("crs") or dataset.crs manifest_payload = { + **TileManifestService.dataset_binding(db, dataset), "tile_set_id": tile_set_id, - "source_dataset_id": str(dataset.id), - "source_raster_id": str(dataset.id), "crs": manifest_crs, "source_crs": manifest_crs, "dataset_crs": dataset.crs, @@ -1066,7 +1118,7 @@ class RasterOperationsService: "tile_server": None, } manifest_path = tile_root / "manifest.json" - manifest_path.write_text(json.dumps(manifest_payload), encoding="utf-8") + manifest_path.write_text(canonical_manifest_json(manifest_payload), encoding="utf-8") return { "dataset_id": str(dataset.id), @@ -1079,3 +1131,17 @@ class RasterOperationsService: "count": len(manifest_tiles), "manifest": manifest_payload, } + + @staticmethod + def _tile_offsets(dimension: int, tile_size: int, step: int) -> list[int]: + """Return full-tile starts plus one unique edge-aligned final start.""" + + if dimension <= 0 or tile_size <= 0 or step <= 0: + raise AppError(code="INVALID_PARAMETERS", message="Raster tile dimensions must be positive", status_code=400) + if dimension <= tile_size: + return [0] + final_start = dimension - tile_size + offsets = list(range(0, final_start + 1, step)) + if offsets[-1] != final_start: + offsets.append(final_start) + return offsets diff --git a/backend/app/services/raster_partition_analysis_service.py b/backend/app/services/raster_partition_analysis_service.py index a81df15b..76e61037 100644 --- a/backend/app/services/raster_partition_analysis_service.py +++ b/backend/app/services/raster_partition_analysis_service.py @@ -8,7 +8,6 @@ from typing import Any from uuid import UUID from pyproj import Transformer -from shapely.geometry import mapping from shapely.ops import transform as shapely_transform from app.core.errors import AppError @@ -116,7 +115,6 @@ class RasterPartitionAnalysisService: try: import numpy as np import rasterio - from rasterio.features import geometry_mask from rasterio.merge import merge except ImportError as exc: raise AppError( diff --git a/backend/app/services/segmentation_service.py b/backend/app/services/segmentation_service.py index 62031149..51196ee0 100644 --- a/backend/app/services/segmentation_service.py +++ b/backend/app/services/segmentation_service.py @@ -34,6 +34,7 @@ from app.services.segmentation_adapter import ( SamSegmentationAdapter, YoloSegmentationAdapter, ) +from app.services.tile_manifest_service import TileManifestService class SegmentationService: @@ -749,6 +750,18 @@ class SegmentationService: sam_adapter_class: type[SamSegmentationAdapter], ) -> tuple[list[Segmentation], dict[str, Any]]: manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles, settings) + dataset = db.get(Dataset, dataset_id) + if dataset is None: + raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404) + manifest_binding = TileManifestService.validate_for_inference( + db, + dataset, + manifest, + manifest_path=tile_manifest_path or "", + settings=settings, + error_prefix="SEGMENTATION", + ) + DetectionService._attach_tile_manifest_binding(analysis_run, job, manifest_binding) if model_name == settings.sam_model_id: model_path = Path(settings.sam_model_path or "").expanduser() allowed_frameworks = ("ultralytics/sam", "sam", "ultralytics", "pytorch") @@ -861,6 +874,7 @@ class SegmentationService: "duplicate_iou_threshold": float(settings.segmentation_duplicate_iou_threshold), "containment_suppression_threshold": float(settings.segmentation_containment_nms_threshold), "tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()), + "tile_manifest_binding": manifest_binding, "runtime_model_provenance": runtime_model_provenance.as_dict(), } diff --git a/backend/app/services/source_catalog_probe_service.py b/backend/app/services/source_catalog_probe_service.py index cfe13f4a..a2bfe448 100644 --- a/backend/app/services/source_catalog_probe_service.py +++ b/backend/app/services/source_catalog_probe_service.py @@ -10,7 +10,7 @@ from threading import Lock from typing import Any, Callable from urllib.error import HTTPError, URLError from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit -from urllib.request import Request, urlopen +from urllib.request import Request from uuid import UUID from xml.etree import ElementTree diff --git a/backend/app/services/storage_service.py b/backend/app/services/storage_service.py index 66609ddb..f8559b6f 100644 --- a/backend/app/services/storage_service.py +++ b/backend/app/services/storage_service.py @@ -10,6 +10,8 @@ from app.core.errors import AppError class StorageService: + UPLOAD_CHUNK_SIZE = 8 * 1024 * 1024 + @staticmethod def _base_dir() -> Path: return Path(get_settings().storage_root).resolve() @@ -143,6 +145,75 @@ class StorageService: } return metadata + @staticmethod + async def persist_upload_file( + *, + project_id: str, + dataset_id: str, + dataset_type: str, + original_filename: str, + upload: Any, + content_type: str | None, + max_bytes: int, + chunk_size: int | None = None, + ) -> dict[str, Any]: + """Stream an UploadFile to governed storage with a hard byte limit. + + The reverse proxy limit is defense in depth. This backend boundary is + authoritative as direct/loopback requests can bypass that proxy. + """ + + if max_bytes <= 0: + raise ValueError("max_bytes must be positive") + resolved_chunk_size = chunk_size or StorageService.UPLOAD_CHUNK_SIZE + if resolved_chunk_size <= 0: + raise ValueError("chunk_size must be positive") + normalized_type = StorageService.normalize_dataset_type(dataset_type) + file_path = Path( + StorageService.dataset_file_path( + project_id, + dataset_id, + normalized_type, + original_filename, + ) + ) + file_path.parent.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256() + size_bytes = 0 + try: + with file_path.open("wb") as stream: + while True: + chunk = await upload.read(resolved_chunk_size) + if not chunk: + break + size_bytes += len(chunk) + if size_bytes > max_bytes: + raise AppError( + code="UPLOAD_TOO_LARGE", + message="Upload exceeds the configured backend size limit.", + details={ + "max_bytes": max_bytes, + "max_upload_mb": max_bytes // (1024 * 1024), + }, + status_code=413, + ) + stream.write(chunk) + digest.update(chunk) + except Exception: + file_path.unlink(missing_ok=True) + parent = file_path.parent + if parent.exists() and parent.is_dir() and not any(parent.iterdir()): + parent.rmdir() + raise + return { + "original_filename": StorageService._safe_filename(original_filename), + "stored_filename": file_path.name, + "content_type": content_type or "application/octet-stream", + "size_bytes": size_bytes, + "checksum_sha256": digest.hexdigest(), + "storage_path": str(file_path), + } + @staticmethod def persist_dataset_file_from_path( project_id: str, @@ -198,6 +269,34 @@ class StorageService: } return metadata + @staticmethod + def persist_file_from_path( + storage_path: str, + source_path: str | Path, + original_filename: str, + content_type: str | None, + ) -> dict[str, Any]: + source = Path(source_path).resolve() + if not source.is_file(): + raise FileNotFoundError(f"Source artifact does not exist: {source}") + target = Path(storage_path) + target.parent.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256() + size_bytes = 0 + with source.open("rb") as input_stream, target.open("wb") as output_stream: + for chunk in iter(lambda: input_stream.read(StorageService.UPLOAD_CHUNK_SIZE), b""): + output_stream.write(chunk) + digest.update(chunk) + size_bytes += len(chunk) + return { + "original_filename": StorageService._safe_filename(original_filename), + "stored_filename": target.name, + "content_type": content_type or "application/octet-stream", + "size_bytes": size_bytes, + "checksum_sha256": digest.hexdigest(), + "storage_path": str(target), + } + @staticmethod def remove_dataset_file(path: str) -> None: target = Path(path) diff --git a/backend/app/services/terrain_analysis_service.py b/backend/app/services/terrain_analysis_service.py index 3e096368..bfac41f8 100644 --- a/backend/app/services/terrain_analysis_service.py +++ b/backend/app/services/terrain_analysis_service.py @@ -111,7 +111,6 @@ class TerrainAnalysisService: try: import numpy as np import rasterio - from rasterio.features import geometry_mask from rasterio.mask import mask except ImportError as exc: raise AppError( diff --git a/backend/app/services/thematic_raster_acquisition_service.py b/backend/app/services/thematic_raster_acquisition_service.py index 366c5bfc..91472ac3 100644 --- a/backend/app/services/thematic_raster_acquisition_service.py +++ b/backend/app/services/thematic_raster_acquisition_service.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any, Callable from urllib.error import HTTPError, URLError from urllib.parse import urlencode -from urllib.request import Request, urlopen +from urllib.request import Request from uuid import UUID from geoalchemy2.shape import to_shape @@ -407,7 +407,6 @@ class ThematicRasterAcquisitionService: if len(coverages) == 1: return coverages[0] try: - import rasterio from rasterio.io import MemoryFile from rasterio.merge import merge except ImportError as exc: diff --git a/backend/app/services/thematic_raster_analysis_service.py b/backend/app/services/thematic_raster_analysis_service.py index d916486f..40126fc5 100644 --- a/backend/app/services/thematic_raster_analysis_service.py +++ b/backend/app/services/thematic_raster_analysis_service.py @@ -96,7 +96,6 @@ class ThematicRasterAnalysisService: try: import numpy as np import rasterio - from rasterio.features import geometry_mask from rasterio.mask import mask except ImportError as exc: raise AppError(code="RASTER_PROCESSING_UNAVAILABLE", message="Rasterio and numpy are required for thematic raster analysis", status_code=503) from exc diff --git a/backend/app/services/tile_manifest_service.py b/backend/app/services/tile_manifest_service.py new file mode 100644 index 00000000..683b33ef --- /dev/null +++ b/backend/app/services/tile_manifest_service.py @@ -0,0 +1,485 @@ +from __future__ import annotations + +from hashlib import sha256 +import json +from math import isfinite +from pathlib import Path +from typing import Any + +from geoalchemy2.shape import to_shape +from pyproj import CRS, Transformer +from shapely.geometry import box +from shapely.ops import transform as shapely_transform +from shapely.ops import unary_union + +from app.core.errors import AppError +from app.models import Area, Dataset, DatasetVersion +from app.services.storage_service import StorageService + + +class TileManifestService: + """Versioned provenance and integrity contract for inference tile sets.""" + + CONTRACT_KEY = "geointel.raster.tile-manifest" + CONTRACT_VERSION = "2.0.0" + _BINDING_FIELDS = ( + "source_dataset_id", + "source_dataset_checksum_sha256", + "source_dataset_size_bytes", + "source_registry_id", + "source_snapshot_id", + "source_snapshot_checksum_sha256", + "data_contract_key", + "data_contract_version", + "source_version", + "dataset_version_id", + "dataset_version", + "dataset_version_checksum_sha256", + "source_area_id", + "source_area_geometry_sha256", + ) + _REQUIRED_INFERENCE_BINDING_FIELDS = ( + "source_dataset_checksum_sha256", + "source_registry_id", + "source_snapshot_id", + "source_snapshot_checksum_sha256", + "data_contract_key", + "data_contract_version", + "dataset_version_id", + "dataset_version", + "dataset_version_checksum_sha256", + ) + _CHECKSUM_FIELDS = ( + "source_dataset_checksum_sha256", + "source_snapshot_checksum_sha256", + "dataset_version_checksum_sha256", + ) + + @staticmethod + def file_sha256(path: str | Path) -> str: + digest = sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + @staticmethod + def _latest_dataset_version(dataset: Dataset) -> DatasetVersion | None: + versions = list(dataset.versions or []) + if not versions: + return None + return max(versions, key=lambda item: (int(item.version or 0), str(item.id or ""))) + + @staticmethod + def _source_snapshot_checksum(dataset: Dataset) -> str | None: + snapshot = dataset.source_snapshot + checksum = getattr(snapshot, "checksum_sha256", None) if snapshot is not None else None + return str(checksum).lower() if checksum else None + + @staticmethod + def _area_geometry_binding(db, dataset: Dataset) -> tuple[str | None, str | None]: + if dataset.area_id is None: + return None, None + area = db.get(Area, dataset.area_id) + if area is None or area.geometry is None: + return str(dataset.area_id), None + geometry = to_shape(area.geometry) + return str(dataset.area_id), sha256(geometry.wkb).hexdigest() + + @classmethod + def dataset_binding(cls, db, dataset: Dataset) -> dict[str, Any]: + version = cls._latest_dataset_version(dataset) + area_id, area_geometry_sha256 = cls._area_geometry_binding(db, dataset) + return { + "manifest_contract_key": cls.CONTRACT_KEY, + "manifest_contract_version": cls.CONTRACT_VERSION, + "source_dataset_id": str(dataset.id), + "source_raster_id": str(dataset.id), + "source_dataset_checksum_sha256": ( + str(dataset.checksum_sha256).lower() if dataset.checksum_sha256 else None + ), + "source_dataset_size_bytes": dataset.size_bytes, + "source_registry_id": str(dataset.source_registry_id) if dataset.source_registry_id else None, + "source_snapshot_id": str(dataset.source_snapshot_id) if dataset.source_snapshot_id else None, + "source_snapshot_checksum_sha256": cls._source_snapshot_checksum(dataset), + "data_contract_key": dataset.data_contract_key, + "data_contract_version": dataset.data_contract_version, + "source_version": dataset.source_version, + "dataset_version_id": str(version.id) if version is not None and version.id else None, + "dataset_version": int(version.version) if version is not None and version.version is not None else None, + "dataset_version_checksum_sha256": ( + str(version.checksum_sha256).lower() + if version is not None and version.checksum_sha256 + else None + ), + "source_area_id": area_id, + "source_area_geometry_sha256": area_geometry_sha256, + } + + @staticmethod + def tile_integrity(path: str | Path) -> dict[str, Any]: + resolved = Path(path) + return { + "size_bytes": resolved.stat().st_size, + "sha256": TileManifestService.file_sha256(resolved), + } + + @staticmethod + def _error( + error_prefix: str, + suffix: str, + message: str, + *, + details: dict[str, Any] | None = None, + ) -> AppError: + return AppError( + code=f"{error_prefix}_TILE_MANIFEST_{suffix}", + message=message, + details=details, + status_code=422, + ) + + @staticmethod + def _bounds_values(value: Any) -> tuple[float, float, float, float] | None: + if isinstance(value, dict): + aliases = ( + ("min_x", "min_y", "max_x", "max_y"), + ("minx", "miny", "maxx", "maxy"), + ("left", "bottom", "right", "top"), + ) + selected = next( + ([value.get(key) for key in keys] for keys in aliases if all(key in value for key in keys)), + None, + ) + elif isinstance(value, (list, tuple)) and len(value) == 4: + selected = list(value) + else: + return None + try: + bounds = tuple(float(item) for item in selected) if selected is not None else None + except (TypeError, ValueError): + return None + if bounds is None or not all(isfinite(item) for item in bounds): + return None + if bounds[0] >= bounds[2] or bounds[1] >= bounds[3]: + return None + return bounds + + @staticmethod + def _to_epsg4326(bounds: tuple[float, float, float, float], raw_crs: Any): + source_crs = CRS.from_user_input(raw_crs) + geometry = box(*bounds) + if not source_crs.equals(CRS.from_epsg(4326)): + transformer = Transformer.from_crs(source_crs, "EPSG:4326", always_xy=True) + geometry = shapely_transform(transformer.transform, geometry) + if geometry.is_empty or not geometry.is_valid: + raise ValueError("Bounds do not form a valid transformed geometry") + if not all(isfinite(float(value)) for value in geometry.bounds): + raise ValueError("Bounds transform to non-finite coordinates") + return geometry + + @classmethod + def _manifest_coverage(cls, manifest: dict[str, Any], *, error_prefix: str): + tiles = manifest.get("tiles") + default_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs") + parts = [] + for index, tile in enumerate(tiles if isinstance(tiles, list) else []): + if not isinstance(tile, dict): + raise cls._error( + error_prefix, + "SCOPE_MISMATCH", + "Tile manifest entries must be objects with explicit spatial metadata.", + details={"tile_index": index}, + ) + bounds = cls._bounds_values(tile.get("bounds")) + raw_crs = tile.get("crs") or default_crs + if bounds is None or not raw_crs: + raise cls._error( + error_prefix, + "SCOPE_MISMATCH", + "Every inference tile requires finite bounds and an explicit CRS.", + details={"tile_index": index}, + ) + try: + parts.append(cls._to_epsg4326(bounds, raw_crs)) + except Exception as exc: + raise cls._error( + error_prefix, + "SCOPE_MISMATCH", + "Inference tile bounds or CRS could not be normalized to EPSG:4326.", + details={"tile_index": index, "reason": str(exc)}, + ) from exc + coverage = unary_union(parts) + if coverage.is_empty or not coverage.is_valid: + raise cls._error( + error_prefix, + "SCOPE_MISMATCH", + "Inference tile union is empty or invalid.", + ) + min_x, min_y, max_x, max_y = coverage.bounds + if min_x < -180 or min_y < -90 or max_x > 180 or max_y > 90: + raise cls._error( + error_prefix, + "SCOPE_MISMATCH", + "Inference tile union falls outside EPSG:4326 bounds.", + details={"bounds": list(coverage.bounds)}, + ) + return coverage + + @classmethod + def _validate_binding(cls, db, dataset: Dataset, manifest: dict[str, Any], *, error_prefix: str) -> dict[str, Any]: + if ( + manifest.get("manifest_contract_key") != cls.CONTRACT_KEY + or manifest.get("manifest_contract_version") != cls.CONTRACT_VERSION + ): + raise cls._error( + error_prefix, + "PROVENANCE_MISMATCH", + "Inference requires a versioned GeoIntel tile-manifest contract.", + details={ + "required_contract": f"{cls.CONTRACT_KEY}@{cls.CONTRACT_VERSION}", + "manifest_contract": ( + f"{manifest.get('manifest_contract_key')}@{manifest.get('manifest_contract_version')}" + ), + }, + ) + expected = cls.dataset_binding(db, dataset) + missing = [ + field + for field in cls._REQUIRED_INFERENCE_BINDING_FIELDS + if expected.get(field) in {None, ""} + ] + invalid_checksums = [ + field + for field in cls._CHECKSUM_FIELDS + if len(str(expected.get(field) or "")) != 64 + or any(character not in "0123456789abcdef" for character in str(expected.get(field) or "").lower()) + ] + if missing or invalid_checksums: + raise cls._error( + error_prefix, + "PROVENANCE_MISMATCH", + "The requested Dataset lacks complete immutable provenance for inference tiling.", + details={ + "missing_fields": missing, + "invalid_checksum_fields": invalid_checksums, + }, + ) + manifest_dataset_id = manifest.get("source_dataset_id") or manifest.get("source_raster_id") + if str(manifest_dataset_id or "") != expected["source_dataset_id"]: + raise cls._error( + error_prefix, + "DATASET_MISMATCH", + "Tile manifest belongs to a different raster Dataset.", + details={ + "requested_dataset_id": expected["source_dataset_id"], + "manifest_dataset_id": manifest_dataset_id, + }, + ) + mismatches = {} + for field in cls._BINDING_FIELDS: + expected_value = expected.get(field) + if expected_value is None or field == "source_dataset_id": + continue + observed_value = manifest.get(field) + if str(observed_value) != str(expected_value): + mismatches[field] = {"expected": expected_value, "observed": observed_value} + if mismatches: + raise cls._error( + error_prefix, + "PROVENANCE_MISMATCH", + "Tile manifest provenance no longer matches the requested Dataset snapshot.", + details={"mismatches": mismatches}, + ) + return expected + + @classmethod + def _validate_tile_files( + cls, + manifest: dict[str, Any], + manifest_path: Path, + *, + settings, + error_prefix: str, + ) -> list[str]: + resolved_paths: list[str] = [] + seen_paths: set[Path] = set() + for index, tile in enumerate(manifest["tiles"]): + raw_path = tile.get("path") if isinstance(tile, dict) else None + if not isinstance(raw_path, str) or not raw_path.strip(): + raise cls._error( + error_prefix, + "TILE_INTEGRITY_MISMATCH", + "Every inference tile requires a path and immutable integrity evidence.", + details={"tile_index": index}, + ) + candidate = Path(raw_path).expanduser() + if not candidate.is_absolute(): + candidate = manifest_path.parent / candidate + candidate = StorageService.assert_within_storage_root( + candidate, + label="raster tile", + settings=settings, + ) + if not candidate.is_file(): + raise cls._error( + error_prefix, + "TILE_INTEGRITY_MISMATCH", + "An inference tile referenced by the manifest does not exist.", + details={"tile_index": index, "tile_path": str(candidate)}, + ) + if candidate in seen_paths: + raise cls._error( + error_prefix, + "TILE_INTEGRITY_MISMATCH", + "A tile path occurs more than once in the inference manifest.", + details={"tile_index": index, "tile_path": str(candidate)}, + ) + seen_paths.add(candidate) + observed_size = candidate.stat().st_size + expected_size = tile.get("size_bytes") + expected_checksum = str(tile.get("sha256") or "").strip().lower() + if expected_size != observed_size or len(expected_checksum) != 64: + raise cls._error( + error_prefix, + "TILE_INTEGRITY_MISMATCH", + "Tile size/checksum evidence is missing or no longer matches the staged file.", + details={ + "tile_index": index, + "expected_size_bytes": expected_size, + "observed_size_bytes": observed_size, + }, + ) + observed_checksum = cls.file_sha256(candidate) + if observed_checksum != expected_checksum: + raise cls._error( + error_prefix, + "TILE_INTEGRITY_MISMATCH", + "Tile checksum no longer matches the immutable manifest evidence.", + details={ + "tile_index": index, + "expected_sha256": expected_checksum, + "observed_sha256": observed_checksum, + }, + ) + resolved_paths.append(str(candidate)) + declared_count = manifest.get("count") + if declared_count != len(resolved_paths): + raise cls._error( + error_prefix, + "TILE_INTEGRITY_MISMATCH", + "Tile manifest count does not match its tile records.", + details={"declared_count": declared_count, "tile_count": len(resolved_paths)}, + ) + return resolved_paths + + @classmethod + def _validate_scope( + cls, + db, + dataset: Dataset, + manifest: dict[str, Any], + coverage, + *, + error_prefix: str, + ) -> None: + manifest_bounds = cls._bounds_values(manifest.get("bounds")) + manifest_crs = manifest.get("crs") or manifest.get("source_crs") or manifest.get("dataset_crs") + dataset_bounds = cls._bounds_values(dataset.bounds_json) + if dataset_bounds is None and isinstance(dataset.metadata_json, dict): + dataset_bounds = cls._bounds_values( + dataset.metadata_json.get("bounds_json") or dataset.metadata_json.get("bounds") + ) + if manifest_bounds is None or not manifest_crs or dataset_bounds is None or not dataset.crs: + raise cls._error( + error_prefix, + "SCOPE_MISMATCH", + "Dataset and tile manifest require explicit CRS and finite bounds for inference.", + ) + try: + manifest_extent = cls._to_epsg4326(manifest_bounds, manifest_crs) + dataset_extent = cls._to_epsg4326(dataset_bounds, dataset.crs) + except Exception as exc: + raise cls._error( + error_prefix, + "SCOPE_MISMATCH", + "Dataset or manifest bounds could not be normalized to EPSG:4326.", + details={"reason": str(exc)}, + ) from exc + tolerance = max(dataset_extent.bounds[2] - dataset_extent.bounds[0], dataset_extent.bounds[3] - dataset_extent.bounds[1]) * 1e-7 + 1e-10 + if not manifest_extent.buffer(tolerance).covers(coverage): + raise cls._error( + error_prefix, + "SCOPE_MISMATCH", + "Tile union exceeds the extent declared by its manifest.", + details={"tile_union_bounds": list(coverage.bounds), "manifest_bounds": list(manifest_extent.bounds)}, + ) + if not dataset_extent.buffer(tolerance).covers(coverage): + raise cls._error( + error_prefix, + "SCOPE_MISMATCH", + "Tile union exceeds the persisted Dataset extent.", + details={"tile_union_bounds": list(coverage.bounds), "dataset_bounds": list(dataset_extent.bounds)}, + ) + if dataset.area_id is not None: + area = db.get(Area, dataset.area_id) + if area is None or area.geometry is None: + raise cls._error( + error_prefix, + "SCOPE_MISMATCH", + "Dataset references an Area that is unavailable for inference-scope validation.", + details={"area_id": str(dataset.area_id)}, + ) + area_geometry = to_shape(area.geometry) + if area_geometry.is_empty or not area_geometry.is_valid or not coverage.intersects(area_geometry): + raise cls._error( + error_prefix, + "SCOPE_MISMATCH", + "Tile union does not overlap the persisted Dataset Area.", + details={"area_id": str(dataset.area_id), "tile_union_bounds": list(coverage.bounds)}, + ) + + @classmethod + def validate_for_inference( + cls, + db, + dataset: Dataset, + manifest: dict[str, Any], + *, + manifest_path: str | Path, + settings, + error_prefix: str, + ) -> dict[str, Any]: + resolved_manifest_path = StorageService.assert_within_storage_root( + manifest_path, + label="tile manifest", + settings=settings, + ) + expected = cls._validate_binding(db, dataset, manifest, error_prefix=error_prefix) + resolved_paths = cls._validate_tile_files( + manifest, + resolved_manifest_path, + settings=settings, + error_prefix=error_prefix, + ) + coverage = cls._manifest_coverage(manifest, error_prefix=error_prefix) + cls._validate_scope(db, dataset, manifest, coverage, error_prefix=error_prefix) + return { + "manifest_contract_key": cls.CONTRACT_KEY, + "manifest_contract_version": cls.CONTRACT_VERSION, + "manifest_path": str(resolved_manifest_path), + "manifest_sha256": cls.file_sha256(resolved_manifest_path), + "source_dataset_id": expected["source_dataset_id"], + "source_dataset_checksum_sha256": expected.get("source_dataset_checksum_sha256"), + "source_snapshot_id": expected.get("source_snapshot_id"), + "dataset_version_id": expected.get("dataset_version_id"), + "source_area_id": expected.get("source_area_id"), + "tile_count": len(resolved_paths), + "tile_union_bounds_epsg4326": [float(value) for value in coverage.bounds], + } + + +def canonical_manifest_json(payload: dict[str, Any]) -> str: + """Stable serializer shared by the writer and manifest-hash tests.""" + + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True) diff --git a/backend/app/utils/geometry.py b/backend/app/utils/geometry.py index d12fd6f6..7445307b 100644 --- a/backend/app/utils/geometry.py +++ b/backend/app/utils/geometry.py @@ -1,43 +1,101 @@ from __future__ import annotations +from math import isfinite +from numbers import Real from typing import Any -from pyproj import Transformer -from shapely import force_2d -from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, box, shape +from pyproj import CRS, Transformer +from shapely import force_2d, get_coordinates +from shapely.geometry import MultiPolygon, box, shape from shapely.ops import transform -from shapely.validation import make_valid + + +# This is a deliberately broad guard envelope around Belgium and the Belgian +# North Sea. Exact legal/regional clipping remains the responsibility of the +# persisted coverage Areas; this boundary prevents an AOI with valid-looking +# but globally misplaced coordinates from entering the workbench. +BELGIUM_AND_NORTH_SEA_GUARD_BOUNDS = (1.5, 48.5, 7.5, 52.5) + + +def _raw_coordinates_are_finite(value: Any) -> bool: + if isinstance(value, (list, tuple)): + return bool(value) and all(_raw_coordinates_are_finite(item) for item in value) + return isinstance(value, Real) and not isinstance(value, bool) and isfinite(float(value)) def normalize_to_multipolygon(raw_geometry: dict[str, Any]) -> MultiPolygon: - geom = force_2d(shape(raw_geometry)) + if isinstance(raw_geometry, dict) and "coordinates" in raw_geometry: + if not _raw_coordinates_are_finite(raw_geometry["coordinates"]): + raise ValueError("Geometry coordinates must be finite numbers") + try: + geom = force_2d(shape(raw_geometry)) + except Exception as exc: + raise ValueError("Geometry is not valid GeoJSON") from exc + coordinates = get_coordinates(geom, include_z=False) + if coordinates.size == 0 or not all(isfinite(float(value)) for row in coordinates for value in row): + raise ValueError("Geometry coordinates must be finite numbers") if geom.is_empty: raise ValueError("Geometry is empty") - if not geom.is_valid: - geom = make_valid(geom) - - if not geom.is_valid: - raise ValueError("Geometry is invalid and could not be repaired") + raise ValueError("Geometry is invalid") if geom.geom_type == "Polygon": return MultiPolygon([geom]) if geom.geom_type == "MultiPolygon": return MultiPolygon(geom.geoms) - if isinstance(geom, GeometryCollection): - polygons = [g for g in geom.geoms if isinstance(g, Polygon)] - multipolygons = [g for g in geom.geoms if g.geom_type == "MultiPolygon"] - if not polygons and not multipolygons: - raise ValueError("Only polygon geometries are supported for AOI") - normalized = [] - normalized.extend(polygons) - for mp in multipolygons: - normalized.extend(mp.geoms) - return MultiPolygon(normalized) raise ValueError("Only Polygon or MultiPolygon geometries are accepted") +def normalize_area_to_epsg4326( + raw_geometry: dict[str, Any], + source_crs: str, +) -> tuple[MultiPolygon, str]: + """Validate an AOI and normalize its declared CRS to canonical WGS84. + + The returned CRS string preserves the caller's declaration for provenance; + the returned geometry is always finite, polygonal and stored as EPSG:4326. + """ + + declared_crs = str(source_crs or "").strip() + if not declared_crs: + raise ValueError("Area CRS is required") + try: + parsed_crs = CRS.from_user_input(declared_crs) + except Exception as exc: + raise ValueError("Area CRS is unknown or invalid") from exc + if not (parsed_crs.is_geographic or parsed_crs.is_projected): + raise ValueError("Area CRS must be a geographic or projected two-dimensional CRS") + if len(parsed_crs.axis_info) != 2: + raise ValueError("Area CRS must have exactly two spatial axes") + + geometry = normalize_to_multipolygon(raw_geometry) + target_crs = CRS.from_epsg(4326) + if not parsed_crs.equals(target_crs): + try: + transformer = Transformer.from_crs(parsed_crs, target_crs, always_xy=True) + geometry = normalize_to_multipolygon( + transform(transformer.transform, geometry).__geo_interface__ + ) + except ValueError: + raise + except Exception as exc: + raise ValueError("Area geometry could not be transformed to EPSG:4326") from exc + + min_x, min_y, max_x, max_y = geometry.bounds + if not all(isfinite(value) for value in (min_x, min_y, max_x, max_y)): + raise ValueError("Transformed area geometry contains non-finite coordinates") + world_bounds = (-180.0, -90.0, 180.0, 90.0) + if min_x < world_bounds[0] or min_y < world_bounds[1] or max_x > world_bounds[2] or max_y > world_bounds[3]: + raise ValueError("Transformed area geometry falls outside the EPSG:4326 coordinate domain") + guard = box(*BELGIUM_AND_NORTH_SEA_GUARD_BOUNDS) + if not guard.intersects(geometry): + raise ValueError("Area geometry falls outside Belgium and the Belgian North Sea workbench domain") + if not guard.covers(geometry): + raise ValueError("Area geometry must remain within the Belgium and Belgian North Sea workbench domain") + return geometry, declared_crs + + def area_bounds_multipolygon(geom: MultiPolygon): return { "min_x": float(geom.bounds[0]), @@ -52,7 +110,10 @@ def area_m2(geom: MultiPolygon) -> float: Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True).transform, geom, ) - return float(projected.area) + result = float(projected.area) + if not isfinite(result) or result <= 0: + raise ValueError("Area geometry must have a finite positive surface") + return result def geometry_bbox_polygon(geom: MultiPolygon): diff --git a/backend/scripts/cleanup_demo_artifacts.py b/backend/scripts/cleanup_demo_artifacts.py index 1b377db2..d756e363 100644 --- a/backend/scripts/cleanup_demo_artifacts.py +++ b/backend/scripts/cleanup_demo_artifacts.py @@ -14,10 +14,13 @@ SCRIPTS_ROOT = REPOSITORY_ROOT / "scripts" if SCRIPTS_ROOT.is_dir(): sys.path.insert(0, str(SCRIPTS_ROOT)) -from app.core.config import get_settings -from app.db.session import SessionLocal -from app.models import Export, Project -from release_backup_guard import require_confirmation, verify_current_backup +from app.core.config import get_settings # noqa: E402 - imported after backend path bootstrap +from app.db.session import SessionLocal # noqa: E402 - imported after backend path bootstrap +from app.models import Export, Project # noqa: E402 - imported after backend path bootstrap +from release_backup_guard import ( # noqa: E402 - imported after scripts path bootstrap + require_confirmation, + verify_current_backup, +) DEMO_PROJECT_NAME = "GeoIntel Demo - Building QA" diff --git a/backend/tests/test_analysis_job_queue.py b/backend/tests/test_analysis_job_queue.py index 543f279e..5ae0d302 100644 --- a/backend/tests/test_analysis_job_queue.py +++ b/backend/tests/test_analysis_job_queue.py @@ -14,7 +14,7 @@ from uuid import uuid4 import pytest from app.core.errors import AppError -from app.models import AnalysisRun, Detection, Job +from app.models import Job from app.services.analysis_job_worker import AnalysisJobWorker from app.services.detection_service import DetectionService diff --git a/backend/tests/test_area_crs_semantics.py b/backend/tests/test_area_crs_semantics.py new file mode 100644 index 00000000..c8f494c3 --- /dev/null +++ b/backend/tests/test_area_crs_semantics.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +from uuid import uuid4 + +from geoalchemy2.shape import from_shape, to_shape +from pyproj import Transformer +import pytest +from shapely.geometry import Polygon, mapping +from shapely.ops import transform + +from app.core.errors import AppError +from app.models import Area, Project +from app.schemas.area import AreaCreate, AreaUpdate +from app.services.area_service import AreaService +from app.utils.geometry import area_m2, normalize_area_to_epsg4326 + + +class FakeSession: + def __init__(self, objects=None) -> None: + self.objects = objects or {} + self.added = [] + self.commits = 0 + self.refreshes = [] + + def get(self, model, item_id): + return self.objects.get((model, item_id)) + + def add(self, item) -> None: + self.added.append(item) + + def commit(self) -> None: + self.commits += 1 + + def refresh(self, item) -> None: + self.refreshes.append(item) + + +def _wgs84_polygon(offset: float = 0.0) -> Polygon: + return Polygon( + [ + (5.00 + offset, 51.00), + (5.01 + offset, 51.00), + (5.01 + offset, 51.01), + (5.00 + offset, 51.01), + (5.00 + offset, 51.00), + ] + ) + + +def _to_lambert(geometry: Polygon) -> Polygon: + transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True) + return transform(transformer.transform, geometry) + + +def test_create_area_transforms_declared_lambert_geometry_before_storage() -> None: + project_id = uuid4() + db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")}) + source = _wgs84_polygon() + + area = AreaService.create_area( + db, + project_id, + AreaCreate(name="Lambert AOI", geometry=mapping(_to_lambert(source)), crs="EPSG:31370"), + ) + + stored = to_shape(area.geometry) + assert stored.bounds == pytest.approx(source.bounds, abs=1e-7) + assert area.original_crs == "EPSG:31370" + assert area.area_m2 == pytest.approx(area_m2(normalize_area_to_epsg4326(mapping(source), "EPSG:4326")[0])) + assert area.area_m2 and area.area_m2 > 0 + assert to_shape(area.bbox).bounds == pytest.approx(source.bounds, abs=1e-7) + + +def test_patch_area_replaces_geometry_and_recomputes_all_spatial_fields() -> None: + area_id = uuid4() + project_id = uuid4() + original = _wgs84_polygon() + normalized, _ = normalize_area_to_epsg4326(mapping(original), "EPSG:4326") + area = Area( + id=area_id, + project_id=project_id, + name="Original", + geometry=from_shape(normalized, srid=4326), + bbox=from_shape(normalized.envelope, srid=4326), + original_crs="EPSG:4326", + area_m2=area_m2(normalized), + ) + db = FakeSession({(Area, area_id): area}) + replacement = _wgs84_polygon(offset=0.05) + + updated = AreaService.update_area( + db, + area_id, + AreaUpdate( + name="Replacement", + geometry=mapping(_to_lambert(replacement)), + crs="EPSG:31370", + ), + ) + + assert updated.name == "Replacement" + assert updated.original_crs == "EPSG:31370" + assert to_shape(updated.geometry).bounds == pytest.approx(replacement.bounds, abs=1e-7) + assert to_shape(updated.bbox).bounds == pytest.approx(replacement.bounds, abs=1e-7) + assert updated.area_m2 and updated.area_m2 > 0 + assert db.commits == 1 + + +@pytest.mark.parametrize( + ("geometry", "crs", "message_fragment"), + [ + (mapping(_wgs84_polygon()), "EPSG:not-real", "unknown or invalid"), + (mapping(_wgs84_polygon()), "EPSG:4979", "exactly two spatial axes"), + ( + { + "type": "Polygon", + "coordinates": [[[5.0, 51.0], [float("nan"), 51.0], [5.1, 51.1], [5.0, 51.0]]], + }, + "EPSG:4326", + "finite", + ), + (mapping(Polygon([(10.0, 51.0), (10.1, 51.0), (10.1, 51.1), (10.0, 51.0)])), "EPSG:4326", "workbench domain"), + ( + { + "type": "Polygon", + "coordinates": [[[5.0, 51.0], [5.1, 51.1], [5.1, 51.0], [5.0, 51.1], [5.0, 51.0]]], + }, + "EPSG:4326", + "invalid", + ), + ({"type": "Point", "coordinates": [5.0, 51.0]}, "EPSG:4326", "Polygon or MultiPolygon"), + ], +) +def test_create_area_rejects_invalid_crs_nonfinite_and_out_of_domain_geometry( + geometry: dict, + crs: str, + message_fragment: str, +) -> None: + project_id = uuid4() + db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")}) + + with pytest.raises(AppError) as exc_info: + AreaService.create_area(db, project_id, AreaCreate(name="Invalid", geometry=geometry, crs=crs)) + + assert exc_info.value.code == "INVALID_GEOMETRY" + assert message_fragment in exc_info.value.message + assert db.commits == 0 + + +def test_patch_area_rejects_crs_without_replacement_geometry() -> None: + area_id = uuid4() + area = Area(id=area_id, project_id=uuid4(), name="AOI", original_crs="EPSG:4326") + db = FakeSession({(Area, area_id): area}) + + with pytest.raises(AppError) as exc_info: + AreaService.update_area(db, area_id, AreaUpdate(crs="EPSG:31370")) + + assert exc_info.value.code == "INVALID_AREA_CRS_UPDATE" + assert db.commits == 0 diff --git a/backend/tests/test_dataset_consumption_gate.py b/backend/tests/test_dataset_consumption_gate.py index 6862ffa7..ee269dd8 100644 --- a/backend/tests/test_dataset_consumption_gate.py +++ b/backend/tests/test_dataset_consumption_gate.py @@ -190,6 +190,17 @@ def test_passed_manual_or_experimental_dataset_cannot_cross_production_boundary( assert "experimental_source_not_allowed_for_purpose" in exc_info.value.details["reasons"] +def test_fully_governed_demo_fixture_still_cannot_enter_production_inference() -> None: + fixture = _governed_dataset(source_key="fixture", classification="experimental") + fixture.source_metadata = {"fixture": True, "usage": "offline demo raster workflow only"} + + with pytest.raises(AppError) as exc_info: + DatasetConsumptionGate.assert_eligible(fixture, purpose="production_inference") + + assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE" + assert "experimental_source_not_allowed_for_purpose" in exc_info.value.details["reasons"] + + def test_reference_validation_requires_authoritative_ground_truth_reference() -> None: reference = _governed_dataset() reference.dataset_type = "vector" diff --git a/backend/tests/test_flood_hazard_selection_data_coverage.py b/backend/tests/test_flood_hazard_selection_data_coverage.py index 8f8c0eea..952c2ed2 100644 --- a/backend/tests/test_flood_hazard_selection_data_coverage.py +++ b/backend/tests/test_flood_hazard_selection_data_coverage.py @@ -17,7 +17,9 @@ import pytest np = pytest.importorskip("numpy") -from app.services.flood_hazard_analysis_service import FloodHazardCellStatistics +from app.services.flood_hazard_analysis_service import ( # noqa: E402 - optional NumPy gate precedes service import + FloodHazardCellStatistics, +) NODATA = -9999.0 diff --git a/backend/tests/test_model_asset_catalog.py b/backend/tests/test_model_asset_catalog.py index 6720c805..6787bc64 100644 --- a/backend/tests/test_model_asset_catalog.py +++ b/backend/tests/test_model_asset_catalog.py @@ -11,10 +11,20 @@ from fastapi.testclient import TestClient from app.core.config import Settings from app.core.errors import AppError from app.main import app -from app.models import AnalysisRun, Dataset, Detection, Job, Project, SourceRegistry, SourceSnapshot +from app.models import ( + AnalysisRun, + Dataset, + DatasetVersion, + Detection, + Job, + Project, + SourceRegistry, + SourceSnapshot, +) from app.services.detection_service import DetectionService from app.services.model_asset_catalog_service import ModelAssetCatalogService from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService +from app.services.tile_manifest_service import TileManifestService class FakeSession: @@ -98,6 +108,8 @@ def _project_and_raster_dataset(): source_name="test-derived-raster", storage_path="storage/uploads/source.tif", checksum_sha256=checksum, + crs="EPSG:4326", + bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0}, source_registry_id=source_registry_id, source_snapshot_id=source_snapshot_id, data_contract_key="geointel.raster.geotiff", @@ -110,17 +122,22 @@ def _project_and_raster_dataset(): ) dataset.source_registry = source_registry dataset.source_snapshot = source_snapshot + dataset.versions.append( + DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1, checksum_sha256=checksum) + ) db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset}) return db, project_id, dataset_id -def _manifest(tmp_path: Path) -> Path: +def _manifest(tmp_path: Path, db: FakeSession, dataset: Dataset) -> Path: tile_path = tmp_path / "tile_0000.tif" tile_path.write_bytes(b"tile") + binding = TileManifestService.dataset_binding(db, dataset) manifest_path = tmp_path / "manifest.json" manifest_path.write_text( json.dumps( { + **binding, "tile_set_id": "tiles-fixture", "count": 1, "crs": "EPSG:4326", @@ -133,6 +150,7 @@ def _manifest(tmp_path: Path) -> Path: "transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01], "crs": "EPSG:4326", "index": 0, + **TileManifestService.tile_integrity(tile_path), } ], } @@ -226,7 +244,11 @@ def test_model_asset_catalog_lists_supported_local_model_files(tmp_path: Path) - assert asset.size_bytes == len(b"local model") assert len(asset.sha256) == 64 assert asset.active is True - assert asset.status == "approved" + assert asset.runtime_available is True + assert asset.runtime_status == "active" + assert asset.governed_validation_status == "not_verified_by_catalog" + assert asset.promotion_status == "not_verified_by_catalog" + assert asset.status == "runtime_active" assert asset.will_download_models is False @@ -257,7 +279,10 @@ def test_model_asset_catalog_only_exposes_explicit_active_asset_in_runtime(tmp_p assert response.total == 1 assert response.items[0].filename == active_file.name assert response.items[0].active is True - assert response.items[0].status == "approved" + assert response.items[0].runtime_status == "active" + assert response.items[0].governed_validation_status == "not_verified_by_catalog" + assert response.items[0].promotion_status == "not_verified_by_catalog" + assert response.items[0].status == "runtime_active" def test_model_asset_catalog_rejects_unknown_asset(tmp_path: Path) -> None: @@ -284,6 +309,9 @@ def test_model_assets_api_returns_canonical_envelope(monkeypatch, tmp_path: Path assert payload["data"]["total"] == 1 assert payload["data"]["items"][0]["model_asset_id"] == "building-detector-pt" assert payload["data"]["items"][0]["active"] is True + assert payload["data"]["items"][0]["runtime_status"] == "active" + assert payload["data"]["items"][0]["governed_validation_status"] == "not_verified_by_catalog" + assert payload["data"]["items"][0]["promotion_status"] == "not_verified_by_catalog" assert payload["data"]["items"][0]["will_download_models"] is False @@ -309,7 +337,7 @@ def test_detection_run_persists_selected_model_asset_parameters(tmp_path, monkey model_id="yolo-configured", model_asset_id="building-detector-pt", confidence_threshold=0.5, - tile_manifest_path=str(_manifest(tmp_path)), + tile_manifest_path=str(_manifest(tmp_path, db, db.get(Dataset, dataset_id))), settings=settings, yolo_adapter_class=MockYoloAdapter, ) diff --git a/backend/tests/test_raster_cell_selection.py b/backend/tests/test_raster_cell_selection.py index 3c81c23c..40474376 100644 --- a/backend/tests/test_raster_cell_selection.py +++ b/backend/tests/test_raster_cell_selection.py @@ -18,10 +18,10 @@ import pytest np = pytest.importorskip("numpy") rasterio = pytest.importorskip("rasterio") -from rasterio.transform import from_origin -from shapely.geometry import box +from rasterio.transform import from_origin # noqa: E402 - optional rasterio gate precedes imports +from shapely.geometry import box # noqa: E402 - optional rasterio gate precedes imports -from app.services.raster_cell_selection import select_cells +from app.services.raster_cell_selection import select_cells # noqa: E402 - optional rasterio gate precedes service import # 100 m cells, origin at the top-left corner of a 3x3 grid. diff --git a/backend/tests/test_raster_operations_service.py b/backend/tests/test_raster_operations_service.py index 5740a316..ddec3448 100644 --- a/backend/tests/test_raster_operations_service.py +++ b/backend/tests/test_raster_operations_service.py @@ -1,14 +1,16 @@ from __future__ import annotations -from types import ModuleType, SimpleNamespace +from types import SimpleNamespace from uuid import uuid4 from pathlib import Path import importlib +from hashlib import sha256 from geoalchemy2.shape import from_shape from app.core.errors import AppError from app.models import Area, Dataset, DatasetVersion from app.services.raster_operations_service import RasterOperationsService +from app.services.storage_service import StorageService from app.api.routes.datasets import _run_job_sync from shapely.geometry import box import pytest @@ -688,6 +690,104 @@ def test_raster_tile_returns_manifest_payload(monkeypatch, tmp_path) -> None: assert payload["manifest"]["tile_server"] is None +@pytest.mark.parametrize( + ("dimension", "expected"), + [ + (512, [0]), + (513, [0, 1]), + (960, [0, 448]), + (961, [0, 448, 449]), + ], +) +def test_raster_tile_offsets_use_full_tiles_and_one_unique_edge_start(dimension, expected) -> None: + assert RasterOperationsService._tile_offsets(dimension, tile_size=512, step=448) == expected + + +def test_raster_tile_rejects_limit_before_creating_output(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + source = tmp_path / "large-raster.tif" + source.write_bytes(b"source") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="large-raster.tif", + dataset_type="raster", + source="user_upload", + storage_path=str(source), + original_filename="large-raster.tif", + stored_filename="large-raster.tif", + content_type="image/tiff", + size_bytes=6, + ) + db = FakeSession([dataset]) + + class FakeSource: + width = 2048 + height = 2048 + count = 1 + crs = None + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return None + + fake_rasterio = SimpleNamespace(open=lambda _path: FakeSource()) + monkeypatch.setattr( + "app.services.raster_operations_service._import_rasterio", + lambda: (fake_rasterio, SimpleNamespace()), + ) + tile_root = tmp_path / "tiles-that-must-not-exist" + monkeypatch.setattr( + StorageService, + "raster_tiles_root", + staticmethod(lambda *_args: tile_root), + ) + + with pytest.raises(AppError) as error: + RasterOperationsService.tile(db, dataset_id, tile_size=512, overlap=64, max_tiles=1) + + assert error.value.code == "RASTER_TILE_LIMIT_EXCEEDED" + assert error.value.details == {"expected_tile_count": 25, "max_tiles": 1} + assert not tile_root.exists() + + +def test_raster_tile_rejects_changed_source_bytes_before_creating_output(monkeypatch, tmp_path) -> None: + project_id = uuid4() + dataset_id = uuid4() + source = tmp_path / "changed-raster.tif" + source.write_bytes(b"changed") + dataset = Dataset( + id=dataset_id, + project_id=project_id, + name="changed-raster.tif", + dataset_type="raster", + source="user_upload", + storage_path=str(source), + original_filename="changed-raster.tif", + stored_filename="changed-raster.tif", + content_type="image/tiff", + size_bytes=7, + checksum_sha256=sha256(b"original").hexdigest(), + data_contract_key="raster.generic", + ) + db = FakeSession([dataset]) + tile_root = tmp_path / "tiles-that-must-not-exist" + monkeypatch.setattr( + StorageService, + "raster_tiles_root", + staticmethod(lambda *_args: tile_root), + ) + + with pytest.raises(AppError) as error: + RasterOperationsService.tile(db, dataset_id) + + assert error.value.code == "DATASET_STORAGE_CHECKSUM_MISMATCH" + assert not tile_root.exists() + + def test_raster_clip_persists_derived_dataset(monkeypatch, tmp_path) -> None: project_id = uuid4() dataset_id = uuid4() @@ -1393,4 +1493,3 @@ def test_run_job_sync_serializes_index_job_output_dataset_id(monkeypatch) -> Non assert result["job_type"] == "raster.ndvi" assert result["output_dataset_id"] == str(output_dataset_id) assert result["result_json"]["output_dataset_id"] == str(output_dataset_id) - diff --git a/backend/tests/test_segmentation_configured_models.py b/backend/tests/test_segmentation_configured_models.py index 60c23dfd..df79056f 100644 --- a/backend/tests/test_segmentation_configured_models.py +++ b/backend/tests/test_segmentation_configured_models.py @@ -9,11 +9,12 @@ import pytest from geoalchemy2.shape import to_shape from app.core.config import Settings -from app.models import Dataset, Project, Segmentation, SourceRegistry, SourceSnapshot +from app.models import Dataset, DatasetVersion, Project, Segmentation, SourceRegistry, SourceSnapshot from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon from app.services.model_registry_service import ModelRegistryService from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService from app.services.segmentation_service import SegmentationService +from app.services.tile_manifest_service import TileManifestService ROOT = Path(__file__).resolve().parents[2] @@ -122,6 +123,8 @@ def _project_and_dataset(dataset_type: str = "raster"): source_name="digitaal_vlaanderen_orthophoto", storage_path="storage/uploads/ortho.tif", checksum_sha256=checksum, + crs="EPSG:4326", + bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0}, source_registry_id=source_id, source_snapshot_id=snapshot_id, data_contract_key="geointel.raster.geotiff", @@ -134,6 +137,9 @@ def _project_and_dataset(dataset_type: str = "raster"): ) dataset.source_registry = source dataset.source_snapshot = snapshot + dataset.versions.append( + DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1, checksum_sha256=checksum) + ) db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset}) return db, project_id, dataset_id @@ -249,28 +255,36 @@ def _write_configured_model_sidecars( ) -def _manifest(tmp_path: Path, tile_count: int = 1) -> Path: +def _manifest( + tmp_path: Path, + tile_count: int = 1, + *, + db: FakeSession | None = None, + dataset: Dataset | None = None, +) -> Path: tiles = [] for index in range(tile_count): tile_path = tmp_path / f"tile_{index:04d}.tif" tile_path.write_bytes(b"fixture") - tiles.append( - { + tile = { "path": str(tile_path), "pixel_window": [0, 0, 100, 100], "bounds": [4.0, 51.0, 5.0, 52.0], "transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01], "crs": "EPSG:4326", "index": index, + **TileManifestService.tile_integrity(tile_path), } - ) + tiles.append(tile) + binding = TileManifestService.dataset_binding(db or FakeSession(), dataset) if dataset is not None else {} manifest_path = tmp_path / "manifest.json" manifest_path.write_text( json.dumps( { + **binding, "tile_set_id": "tiles-fixture", - "source_dataset_id": str(uuid4()), - "source_raster_id": str(uuid4()), + "source_dataset_id": binding.get("source_dataset_id", str(uuid4())), + "source_raster_id": binding.get("source_raster_id", str(uuid4())), "crs": "EPSG:4326", "bounds": [4.0, 51.0, 5.0, 52.0], "tile_size": 100, @@ -424,7 +438,9 @@ def test_configured_segmentation_rejects_unbound_model_snapshot_before_adapter_l dataset_id=dataset_id, model_id="yolo-seg-configured", confidence_threshold=0.5, - tile_manifest_path=str(_manifest(tmp_path)), + tile_manifest_path=str( + _manifest(tmp_path, db=db, dataset=db.get(Dataset, dataset_id)) + ), settings=settings, yolo_seg_adapter_class=NeverLoadSegAdapter, sam_adapter_class=ClassAgnosticSamAdapter, @@ -440,7 +456,7 @@ def test_configured_yolo_seg_run_persists_georeferenced_masks(tmp_path: Path) -> db, project_id, dataset_id = _project_and_dataset() settings = _settings(tmp_path) _write_configured_model_sidecars(tmp_path, settings, include_sam=False, db=db) - manifest_path = _manifest(tmp_path) + manifest_path = _manifest(tmp_path, db=db, dataset=db.get(Dataset, dataset_id)) response = SegmentationService.run_segmentation( db=db, @@ -479,7 +495,7 @@ def test_configured_sam_run_is_class_agnostic(tmp_path: Path) -> None: db, project_id, dataset_id = _project_and_dataset() settings = _settings(tmp_path) _write_configured_model_sidecars(tmp_path, settings, include_yolo=False, db=db) - manifest_path = _manifest(tmp_path) + manifest_path = _manifest(tmp_path, db=db, dataset=db.get(Dataset, dataset_id)) response = SegmentationService.run_segmentation( db=db, diff --git a/backend/tests/test_small_selection_raster_analysis.py b/backend/tests/test_small_selection_raster_analysis.py index ee5bc1a1..213b1323 100644 --- a/backend/tests/test_small_selection_raster_analysis.py +++ b/backend/tests/test_small_selection_raster_analysis.py @@ -15,14 +15,18 @@ import pytest np = pytest.importorskip("numpy") rasterio = pytest.importorskip("rasterio") -from pyproj import Transformer -from rasterio.transform import from_origin +from pyproj import Transformer # noqa: E402 - optional rasterio gate precedes geospatial imports +from rasterio.transform import from_origin # noqa: E402 - optional rasterio gate precedes geospatial imports -from app.core.config import Settings -from app.models import Dataset -from app.schemas.flood_hazard import FloodHazardSelectionRequest -from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService -from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService +from app.core.config import Settings # noqa: E402 - optional rasterio gate precedes app imports +from app.models import Dataset # noqa: E402 - optional rasterio gate precedes app imports +from app.schemas.flood_hazard import FloodHazardSelectionRequest # noqa: E402 - optional rasterio gate precedes app imports +from app.services.flood_hazard_acquisition_service import ( # noqa: E402 - optional rasterio gate precedes app imports + FloodHazardAcquisitionService, +) +from app.services.flood_hazard_analysis_service import ( # noqa: E402 - optional rasterio gate precedes app imports + FloodHazardAnalysisService, +) TO_4326 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True) diff --git a/backend/tests/test_sprint100_segmentation_manifest_handoff.py b/backend/tests/test_sprint100_segmentation_manifest_handoff.py index a3d03314..948411fd 100644 --- a/backend/tests/test_sprint100_segmentation_manifest_handoff.py +++ b/backend/tests/test_sprint100_segmentation_manifest_handoff.py @@ -14,7 +14,10 @@ def test_raster_tile_manifest_can_handoff_to_segmentation_lab() -> None: assert "segmentationTileManifestPath" in hook assert "setSegmentationTileManifestPath" in hook - assert "tile_manifest_path: segmentationTileManifestPath.trim() || null" in hook + assert "let manifestPath = segmentationTileManifestPath.trim()" in hook + assert "datasetsApi.rasterInspect(projectId, datasetId)" in hook + assert "datasetsApi.rasterTile(projectId, datasetId" in hook + assert "tile_manifest_path: manifestPath" in hook assert "segmentationTileManifestPath={segmentationTileManifestPath}" in app assert "onSetTileManifestPath={setSegmentationTileManifestPath}" in app assert "onUseTileManifestForSegmentation: useRasterTileManifestForSegmentation" in app @@ -24,4 +27,3 @@ def test_raster_tile_manifest_can_handoff_to_segmentation_lab() -> None: assert "Gebruik voor segmentatie" in raster_controls assert "disabled={!latestRasterTileManifestPath}" in raster_controls assert "Beeldtegelmanifest" in segmentation_lab - assert "Beeldtegelmanifest" in segmentation_lab diff --git a/backend/tests/test_sprint106_map_bbox_extract.py b/backend/tests/test_sprint106_map_bbox_extract.py index 5abd4e33..19e6d959 100644 --- a/backend/tests/test_sprint106_map_bbox_extract.py +++ b/backend/tests/test_sprint106_map_bbox_extract.py @@ -1,6 +1,5 @@ from __future__ import annotations -import re import uuid from pathlib import Path from types import SimpleNamespace @@ -11,7 +10,7 @@ from shapely.geometry import Polygon, box from app.core.errors import AppError from app.models import Dataset, VectorFeature from app.services.vector_feature_service import VectorFeatureService -from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace, read_feature +from tests.frontend_contract import assert_calls, assert_wired, read_map_workspace, read_feature ROOT = Path(__file__).resolve().parents[2] diff --git a/backend/tests/test_sprint122_raster_upload_metadata_mapping.py b/backend/tests/test_sprint122_raster_upload_metadata_mapping.py index cd7b5928..ae766d80 100644 --- a/backend/tests/test_sprint122_raster_upload_metadata_mapping.py +++ b/backend/tests/test_sprint122_raster_upload_metadata_mapping.py @@ -12,8 +12,12 @@ class FakeUploadFile: filename = "real-orthophoto.tif" content_type = "image/tiff" - async def read(self) -> bytes: - return b"fake-raster" + def __init__(self) -> None: + self._content = b"fake-raster" + + async def read(self, size: int) -> bytes: + chunk, self._content = self._content[:size], self._content[size:] + return chunk class FakeSession: @@ -40,16 +44,19 @@ def test_raster_upload_maps_metadata_bounds_resolution_and_bands(monkeypatch) -> project_id = uuid4() db = FakeSession(project_id) - monkeypatch.setattr( - "app.services.dataset_service.StorageService.persist_dataset_file", - lambda **_: { + async def persist_upload_file(**_kwargs): + return { "storage_path": "/tmp/real-orthophoto.tif", "original_filename": "real-orthophoto.tif", "stored_filename": "real-orthophoto.tif", "content_type": "image/tiff", "size_bytes": 11, "checksum_sha256": "checksum", - }, + } + + monkeypatch.setattr( + "app.services.dataset_service.StorageService.persist_upload_file", + persist_upload_file, ) monkeypatch.setattr( "app.services.dataset_service.extract_raster_metadata", diff --git a/backend/tests/test_sprint186_map_first_geographic_explorer.py b/backend/tests/test_sprint186_map_first_geographic_explorer.py index c9e67718..632478f8 100644 --- a/backend/tests/test_sprint186_map_first_geographic_explorer.py +++ b/backend/tests/test_sprint186_map_first_geographic_explorer.py @@ -1,9 +1,8 @@ from __future__ import annotations -import re from pathlib import Path -from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace, read_feature +from tests.frontend_contract import assert_calls, assert_wired, read_map_workspace, read_feature ROOT = Path(__file__).resolve().parents[2] diff --git a/backend/tests/test_sprint192_regional_map_state.py b/backend/tests/test_sprint192_regional_map_state.py index 33dd6759..fec239e9 100644 --- a/backend/tests/test_sprint192_regional_map_state.py +++ b/backend/tests/test_sprint192_regional_map_state.py @@ -1,5 +1,5 @@ from pathlib import Path -from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace, read_feature +from tests.frontend_contract import assert_calls, read_map_workspace, read_feature ROOT = Path(__file__).resolve().parents[2] diff --git a/backend/tests/test_sprint194_regional_timeseries.py b/backend/tests/test_sprint194_regional_timeseries.py index 3083a3ee..0286ce63 100644 --- a/backend/tests/test_sprint194_regional_timeseries.py +++ b/backend/tests/test_sprint194_regional_timeseries.py @@ -251,7 +251,8 @@ def test_end_user_dataset_sources_are_human_readable() -> None: assert "statbel: 'Statbel'" in display assert "getDatasetSourceDisplayName(activeThemeDataset)" in workspace assert "getDatasetSourceDisplayName(resultDataset)" in workspace - assert "Zoek optioneel een gemeente" in workspace + assert 'aria-label="Optioneel een gemeente zoeken"' in workspace + assert 'placeholder="Gemeentenaam of NIS-code"' in workspace assert "latestDatasetBySeries" in catalog assert "Historische meetmomenten" in catalog assert "getDatasetSourceDisplayName(dataset)" in catalog diff --git a/backend/tests/test_sprint196_map_orthophoto_analysis.py b/backend/tests/test_sprint196_map_orthophoto_analysis.py index f96b4ea6..f9a65905 100644 --- a/backend/tests/test_sprint196_map_orthophoto_analysis.py +++ b/backend/tests/test_sprint196_map_orthophoto_analysis.py @@ -20,6 +20,7 @@ from app.db.session import get_db from app.main import app from app.models import Area, Dataset, DatasetVersion, Job, Project, SourceRegistry, SourceSnapshot from app.schemas.orthophoto import OrthophotoAcquireRequest +from app.services.dataset_consumption_gate_service import DatasetConsumptionGate from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService from tests.frontend_contract import read_feature @@ -262,6 +263,9 @@ def test_regional_orthophoto_products_bind_provider_and_governed_scope( assert snapshot.checksum_sha256 == dataset.checksum_sha256 assert snapshot.ingest_status == "ingested" assert snapshot.freshness_status == "current" + dataset.source_registry = source + dataset.source_snapshot = snapshot + assert DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference").eligible is True prepared = OrthophotoAcquisitionService._prepared_request(_selection_payload(product_key="1971"), settings) assert prepared["params"]["LAYERS"] == "OKZPAN71VL" diff --git a/backend/tests/test_sprint200_temporal_explorer_handoff.py b/backend/tests/test_sprint200_temporal_explorer_handoff.py index ae7b368c..5649480b 100644 --- a/backend/tests/test_sprint200_temporal_explorer_handoff.py +++ b/backend/tests/test_sprint200_temporal_explorer_handoff.py @@ -1,5 +1,5 @@ from pathlib import Path -from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace +from tests.frontend_contract import assert_wired, read_map_workspace ROOT = Path(__file__).resolve().parents[2] diff --git a/backend/tests/test_sprint205_agricultural_parcel_history.py b/backend/tests/test_sprint205_agricultural_parcel_history.py index 1a64f0db..0815954c 100644 --- a/backend/tests/test_sprint205_agricultural_parcel_history.py +++ b/backend/tests/test_sprint205_agricultural_parcel_history.py @@ -1,7 +1,6 @@ from __future__ import annotations import importlib.util -import json import sys import zipfile from pathlib import Path diff --git a/backend/tests/test_sprint213_thematic_rasters.py b/backend/tests/test_sprint213_thematic_rasters.py index 168e730d..5040ecf1 100644 --- a/backend/tests/test_sprint213_thematic_rasters.py +++ b/backend/tests/test_sprint213_thematic_rasters.py @@ -7,7 +7,6 @@ from uuid import uuid4 import numpy as np import pytest -import rasterio from fastapi.testclient import TestClient from pyproj import Transformer from rasterio.io import MemoryFile diff --git a/backend/tests/test_sprint223_governed_grb_refresh.py b/backend/tests/test_sprint223_governed_grb_refresh.py index 1232767a..82141d00 100644 --- a/backend/tests/test_sprint223_governed_grb_refresh.py +++ b/backend/tests/test_sprint223_governed_grb_refresh.py @@ -252,7 +252,6 @@ def test_refresh_api_and_frontend_remain_explicit_only() -> None: def test_map_theme_ranking_prefers_newer_observation_over_feature_count() -> None: - root = Path(__file__).resolve().parents[2] workspace = read_map_workspace() observed_sort = workspace.index("const observedAtDifference") feature_tiebreaker = workspace.index("right.feature_count", observed_sort) @@ -262,7 +261,6 @@ def test_map_theme_ranking_prefers_newer_observation_over_feature_count() -> Non def test_map_workspace_restores_theme_from_selected_dataset() -> None: - root = Path(__file__).resolve().parents[2] workspace = read_map_workspace() assert "function themeIdForDataset(" in workspace assert "useState(() =>" in workspace diff --git a/backend/tests/test_sprint236_bathymetry_expansion.py b/backend/tests/test_sprint236_bathymetry_expansion.py index 7e8e8369..a7dc7d65 100644 --- a/backend/tests/test_sprint236_bathymetry_expansion.py +++ b/backend/tests/test_sprint236_bathymetry_expansion.py @@ -1,7 +1,6 @@ from __future__ import annotations from datetime import UTC, datetime -import json from pathlib import Path import ssl import sys @@ -31,7 +30,7 @@ if str(SCRIPTS) not in sys.path: sys.path.insert(0, str(SCRIPTS)) import provision_flanders_geographic_scope as flanders_scope # noqa: E402 -from tests.frontend_contract import read_map_workspace, read_feature +from tests.frontend_contract import read_map_workspace, read_feature # noqa: E402 class BinaryResponse: @@ -136,6 +135,32 @@ def test_mdk_probe_parses_capabilities_without_enabling_acquisition() -> None: assert seen["timeout"] == 20 +def test_mdk_probe_default_opener_uses_guarded_strict_tls_path(monkeypatch) -> None: + seen = {} + + def guarded_factory(expected_url): + seen["expected_url"] = expected_url + + def open_request(request, timeout): + seen["request_url"] = request.full_url + seen["timeout"] = timeout + return BinaryResponse(capabilities_xml()) + + return open_request + + monkeypatch.setattr( + "app.services.mdk_bathymetry_probe_service.guarded_opener", + guarded_factory, + ) + + result = MdkBathymetryProbeService.probe(settings=Settings(_env_file=None)) + + assert result["status"] == "reachable" + assert seen["expected_url"] == seen["request_url"] + assert seen["expected_url"].startswith("https://") + assert seen["timeout"] == 20 + + def test_mdk_probe_reports_tls_failure_and_never_uses_insecure_fallback() -> None: calls = 0 diff --git a/backend/tests/test_sprint237_flanders_thematic_on_demand.py b/backend/tests/test_sprint237_flanders_thematic_on_demand.py index 03965a09..6f89f5cc 100644 --- a/backend/tests/test_sprint237_flanders_thematic_on_demand.py +++ b/backend/tests/test_sprint237_flanders_thematic_on_demand.py @@ -1,5 +1,5 @@ from pathlib import Path -from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace, read_feature +from tests.frontend_contract import assert_wired, read_map_workspace, read_feature ROOT = Path(__file__).resolve().parents[2] diff --git a/backend/tests/test_sprint239_bounded_grb_acquisition.py b/backend/tests/test_sprint239_bounded_grb_acquisition.py index f7e4e719..f33f4801 100644 --- a/backend/tests/test_sprint239_bounded_grb_acquisition.py +++ b/backend/tests/test_sprint239_bounded_grb_acquisition.py @@ -19,7 +19,7 @@ from app.models import Area, Dataset, Job, Project from app.schemas.grb import GrbAcquireRequest from app.services.dataset_service import DatasetService from app.services.grb_acquisition_service import GrbAcquisitionService -from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace, read_feature +from tests.frontend_contract import assert_wired, read_map_workspace, read_feature ROOT = Path(__file__).resolve().parents[2] diff --git a/backend/tests/test_sprint241_spw_bathymetry_raster.py b/backend/tests/test_sprint241_spw_bathymetry_raster.py index a1fb8a57..4366368b 100644 --- a/backend/tests/test_sprint241_spw_bathymetry_raster.py +++ b/backend/tests/test_sprint241_spw_bathymetry_raster.py @@ -1,7 +1,6 @@ from __future__ import annotations import importlib.util -import io from pathlib import Path import sys import zipfile diff --git a/backend/tests/test_sprint7a_persistence_foundation.py b/backend/tests/test_sprint7a_persistence_foundation.py index b25be683..c431f834 100644 --- a/backend/tests/test_sprint7a_persistence_foundation.py +++ b/backend/tests/test_sprint7a_persistence_foundation.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import json from pathlib import Path from uuid import uuid4 @@ -187,23 +188,31 @@ def test_dataset_upload_persists_vector_features(monkeypatch, tmp_path) -> None: filename = "reference.geojson" content_type = "application/geo+json" - async def read(self) -> bytes: + def __init__(self) -> None: import json - return json.dumps(payload).encode("utf-8") + self._content = json.dumps(payload).encode("utf-8") + + async def read(self, size: int) -> bytes: + chunk, self._content = self._content[:size], self._content[size:] + return chunk storage_path = tmp_path / "reference.geojson" - storage_path.write_text("{}", encoding="utf-8") - monkeypatch.setattr( - "app.services.dataset_service.StorageService.persist_dataset_file", - lambda **_kwargs: { + storage_path.write_text(json.dumps(payload), encoding="utf-8") + + async def persist_upload_file(**_kwargs): + return { "storage_path": str(storage_path), "original_filename": "reference.geojson", "stored_filename": "reference.geojson", "content_type": "application/geo+json", - "size_bytes": 2, + "size_bytes": storage_path.stat().st_size, "checksum_sha256": "0" * 64, - }, + } + + monkeypatch.setattr( + "app.services.dataset_service.StorageService.persist_upload_file", + persist_upload_file, ) result = asyncio.run( @@ -236,19 +245,28 @@ def test_dataset_upload_rolls_back_dataset_and_file_when_vector_indexing_fails(m filename = "invalid.geojson" content_type = "application/geo+json" - async def read(self) -> bytes: - return b'{"type":"FeatureCollection","features":[]}' + def __init__(self) -> None: + self._content = b'{"type":"FeatureCollection","features":[]}' - monkeypatch.setattr( - "app.services.dataset_service.StorageService.persist_dataset_file", - lambda **_kwargs: { + async def read(self, size: int) -> bytes: + chunk, self._content = self._content[:size], self._content[size:] + return chunk + + storage_path.write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8") + + async def persist_upload_file(**_kwargs): + return { "storage_path": str(storage_path), "original_filename": "invalid.geojson", "stored_filename": "invalid.geojson", "content_type": "application/geo+json", - "size_bytes": 2, + "size_bytes": storage_path.stat().st_size, "checksum_sha256": "0" * 64, - }, + } + + monkeypatch.setattr( + "app.services.dataset_service.StorageService.persist_upload_file", + persist_upload_file, ) monkeypatch.setattr( VectorFeatureService, diff --git a/backend/tests/test_sprint8b_yolo_foundation.py b/backend/tests/test_sprint8b_yolo_foundation.py index cbbc25bb..4bcd77f9 100644 --- a/backend/tests/test_sprint8b_yolo_foundation.py +++ b/backend/tests/test_sprint8b_yolo_foundation.py @@ -13,12 +13,23 @@ from shapely.geometry import box, mapping from app.core.config import Settings from app.core.errors import AppError -from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project, SourceRegistry, SourceSnapshot +from app.models import ( + AnalysisRun, + Area, + Dataset, + DatasetVersion, + Detection, + Job, + Project, + SourceRegistry, + SourceSnapshot, +) from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon from app.services.detection_service import DetectionService from app.services.model_registry_service import ModelRegistryService from app.services.model_validation_scope_service import ModelValidationScopeService from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService +from app.services.tile_manifest_service import TileManifestService from app.services.yolo_adapter import YoloDetectionAdapter ROOT = Path(__file__).resolve().parents[2] @@ -189,6 +200,8 @@ def _project_and_dataset(dataset_type: str = "raster"): source_name="test-derived-raster", storage_path="storage/uploads/source.tif", checksum_sha256=checksum, + crs="EPSG:4326", + bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0}, source_registry_id=source_registry_id, source_snapshot_id=source_snapshot_id, data_contract_key="geointel.raster.geotiff", @@ -201,6 +214,9 @@ def _project_and_dataset(dataset_type: str = "raster"): ) dataset.source_registry = source_registry dataset.source_snapshot = source_snapshot + dataset.versions.append( + DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1, checksum_sha256=checksum) + ) db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset}) return db, project_id, dataset_id @@ -309,28 +325,34 @@ def _write_model_sidecar( ) -def _manifest(tmp_path: Path, tile_count: int = 1) -> Path: +def _manifest(tmp_path: Path, tile_count: int = 1, dataset: Dataset | None = None) -> Path: tiles = [] for index in range(tile_count): tile_path = tmp_path / f"tile_{index:04d}.tif" tile_path.write_bytes(b"fixture") - tiles.append( - { + tile = { "path": str(tile_path), "pixel_window": [0, 0, 100, 100], "bounds": [4.0, 51.0, 5.0, 52.0], "transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01], "crs": "EPSG:4326", "index": index, + **TileManifestService.tile_integrity(tile_path), } - ) + tiles.append(tile) + binding = ( + TileManifestService.dataset_binding(SimpleNamespace(get=lambda *_args: None), dataset) + if dataset is not None + else {} + ) manifest_path = tmp_path / "manifest.json" manifest_path.write_text( json.dumps( { + **binding, "tile_set_id": "tiles-fixture", - "source_dataset_id": str(uuid4()), - "source_raster_id": str(uuid4()), + "source_dataset_id": binding.get("source_dataset_id", str(uuid4())), + "source_raster_id": binding.get("source_raster_id", str(uuid4())), "crs": "EPSG:4326", "bounds": [4.0, 51.0, 5.0, 52.0], "tile_size": 100, @@ -514,7 +536,7 @@ def test_yolo_run_fails_closed_before_adapter_load_without_sidecar(tmp_path: Pat dataset_id=dataset_id, model_id="yolo-configured", confidence_threshold=0.5, - tile_manifest_path=str(_manifest(tmp_path)), + tile_manifest_path=str(_manifest(tmp_path, dataset=db.get(Dataset, dataset_id))), settings=settings, yolo_adapter_class=AvailableAdapter, ) @@ -530,7 +552,7 @@ def test_yolo_run_rejects_manifest_over_tile_limit(tmp_path: Path) -> None: model_path.write_bytes(b"local weights") settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_max_tiles=1) _write_model_sidecar(model_path, settings, db=db) - manifest_path = _manifest(tmp_path, tile_count=2) + manifest_path = _manifest(tmp_path, tile_count=2, dataset=db.get(Dataset, dataset_id)) result = DetectionService.run_detection( db=db, @@ -609,7 +631,7 @@ def test_yolo_run_rejects_unbound_model_snapshot_before_adapter_load(tmp_path: P dataset_id=dataset_id, model_id="yolo-configured", confidence_threshold=0.5, - tile_manifest_path=str(_manifest(tmp_path)), + tile_manifest_path=str(_manifest(tmp_path, dataset=db.get(Dataset, dataset_id))), settings=settings, yolo_adapter_class=NeverLoadUnboundModelAdapter, ) @@ -638,7 +660,7 @@ def test_yolo_run_persists_mocked_georeferenced_detections(tmp_path: Path) -> No model_path.write_bytes(b"local weights") settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_model_version="local-test") _write_model_sidecar(model_path, settings, db=db) - manifest_path = _manifest(tmp_path, tile_count=1) + manifest_path = _manifest(tmp_path, tile_count=1, dataset=db.get(Dataset, dataset_id)) result = DetectionService.run_detection( db=db, @@ -678,7 +700,7 @@ def test_yolo_class_filter_is_case_insensitive_and_persists_canonical_class(tmp_ model_path.write_bytes(b"local weights") settings = _settings(tmp_path, yolo_model_path=str(model_path)) _write_model_sidecar(model_path, settings, db=db) - manifest_path = _manifest(tmp_path, tile_count=1) + manifest_path = _manifest(tmp_path, tile_count=1, dataset=db.get(Dataset, dataset_id)) result = DetectionService.run_detection( db=db, @@ -706,7 +728,7 @@ def test_yolo_run_suppresses_cross_tile_duplicate_detections(tmp_path: Path) -> model_path.write_bytes(b"local weights") settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_duplicate_iou_threshold=0.5) _write_model_sidecar(model_path, settings, db=db) - manifest_path = _manifest(tmp_path, tile_count=2) + manifest_path = _manifest(tmp_path, tile_count=2, dataset=db.get(Dataset, dataset_id)) result = DetectionService.run_detection( db=db, diff --git a/backend/tests/test_sprint93_export_handoff_completion.py b/backend/tests/test_sprint93_export_handoff_completion.py index 8d210ba7..57bd3a39 100644 --- a/backend/tests/test_sprint93_export_handoff_completion.py +++ b/backend/tests/test_sprint93_export_handoff_completion.py @@ -1,5 +1,5 @@ from pathlib import Path -from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_feature +from tests.frontend_contract import assert_mentions, read_feature ROOT = Path(__file__).resolve().parents[2] diff --git a/backend/tests/test_storage_service.py b/backend/tests/test_storage_service.py index de289492..80517ac4 100644 --- a/backend/tests/test_storage_service.py +++ b/backend/tests/test_storage_service.py @@ -1,6 +1,12 @@ +import asyncio +from hashlib import sha256 from pathlib import Path from types import SimpleNamespace +import pytest + +from app.core.errors import AppError +from app.services.dataset_service import DatasetService from app.services.storage_service import StorageService @@ -26,3 +32,111 @@ def test_persist_dataset_file_records_metadata(monkeypatch, tmp_path) -> None: assert len(metadata["checksum_sha256"]) == 64 assert Path(metadata["storage_path"]).exists() assert str(Path(tmp_path, "uploads", "project-123", "vector", "dataset-456")) in metadata["storage_path"] + + +class _ChunkedUpload: + def __init__(self, content: bytes) -> None: + self.content = content + self.requested_sizes: list[int] = [] + + async def read(self, size: int) -> bytes: + self.requested_sizes.append(size) + chunk, self.content = self.content[:size], self.content[size:] + return chunk + + +def test_persist_upload_file_streams_bounded_chunks(monkeypatch, tmp_path) -> None: + monkeypatch.setattr(StorageService, "_base_dir", staticmethod(lambda: tmp_path)) + upload = _ChunkedUpload(b"abcdefghijk") + + metadata = asyncio.run( + StorageService.persist_upload_file( + project_id="project", + dataset_id="dataset", + dataset_type="raster", + original_filename="source.tif", + upload=upload, + content_type="image/tiff", + max_bytes=32, + chunk_size=4, + ) + ) + + stored = Path(metadata["storage_path"]) + assert upload.requested_sizes == [4, 4, 4, 4] + assert stored.read_bytes() == b"abcdefghijk" + assert metadata["size_bytes"] == 11 + assert metadata["checksum_sha256"] == sha256(b"abcdefghijk").hexdigest() + + +def test_persist_upload_file_rejects_oversize_and_removes_partial_file(monkeypatch, tmp_path) -> None: + monkeypatch.setattr(StorageService, "_base_dir", staticmethod(lambda: tmp_path)) + upload = _ChunkedUpload(b"0123456789") + expected_path = Path( + StorageService.dataset_file_path( + "project", + "dataset", + "vector", + "source.geojson", + ) + ) + + with pytest.raises(AppError) as exc_info: + asyncio.run( + StorageService.persist_upload_file( + project_id="project", + dataset_id="dataset", + dataset_type="vector", + original_filename="source.geojson", + upload=upload, + content_type="application/geo+json", + max_bytes=8, + chunk_size=3, + ) + ) + + assert exc_info.value.code == "UPLOAD_TOO_LARGE" + assert exc_info.value.status_code == 413 + assert not expected_path.exists() + + +def test_vector_staging_uses_lower_in_memory_limit(monkeypatch) -> None: + captured = {} + + async def persist_upload_file(**kwargs): + captured.update(kwargs) + return {"storage_path": "unused"} + + monkeypatch.setattr( + "app.services.dataset_service.get_settings", + lambda: SimpleNamespace(max_upload_mb=500, max_in_memory_vector_mb=32), + ) + monkeypatch.setattr(StorageService, "persist_upload_file", persist_upload_file) + + asyncio.run( + DatasetService._stage_upload( + project_id="project", + dataset_id="dataset", + dataset_type="vector", + filename="source.geojson", + file=SimpleNamespace(content_type="application/geo+json"), + ) + ) + + assert captured["max_bytes"] == 32 * 1024 * 1024 + + +def test_staged_vector_read_is_bounded_before_loading_file(monkeypatch, tmp_path) -> None: + source = tmp_path / "large.geojson" + source.write_bytes(b"x" * (1024 * 1024 + 1)) + monkeypatch.setattr( + "app.services.dataset_service.get_settings", + lambda: SimpleNamespace(max_upload_mb=500, max_in_memory_vector_mb=1), + ) + + with pytest.raises(AppError) as exc_info: + DatasetService._read_staged_vector_bytes({"storage_path": str(source)}) + + assert exc_info.value.code == "UPLOAD_TOO_LARGE" + assert exc_info.value.status_code == 413 + assert not source.exists() diff --git a/backend/tests/test_tile_manifest_binding.py b/backend/tests/test_tile_manifest_binding.py new file mode 100644 index 00000000..f85346d9 --- /dev/null +++ b/backend/tests/test_tile_manifest_binding.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +import json +from pathlib import Path +from uuid import uuid4 + +from geoalchemy2.shape import from_shape +import pytest +from shapely.geometry import box + +from app.core.config import Settings +from app.core.errors import AppError +from app.models import Area, Dataset, DatasetVersion, SourceRegistry, SourceSnapshot +from app.services.tile_manifest_service import TileManifestService, canonical_manifest_json + + +class FakeSession: + def __init__(self, objects=None) -> None: + self.objects = objects or {} + + def get(self, model, item_id): + return self.objects.get((model, item_id)) + + +def _dataset_and_session(*, with_area: bool = True): + project_id = uuid4() + dataset_id = uuid4() + registry_id = uuid4() + snapshot_id = uuid4() + checksum = "a" * 64 + area = None + area_id = uuid4() if with_area else None + if area_id is not None: + area = Area( + id=area_id, + project_id=project_id, + name="Inference AOI", + geometry=from_shape(box(4.0, 51.0, 5.0, 52.0), srid=4326), + ) + registry = SourceRegistry( + id=registry_id, + source_key="governed-test-raster", + display_name="Governed test raster", + classification="derived", + authority_name="GeoIntel", + ) + snapshot = SourceSnapshot( + id=snapshot_id, + source_registry_id=registry_id, + snapshot_key="snapshot-1", + checksum_sha256=checksum, + ingest_status="ingested", + freshness_status="current", + ) + dataset = Dataset( + id=dataset_id, + project_id=project_id, + area_id=area_id, + name="orthophoto.tif", + dataset_type="raster", + source="governed-test-raster", + source_name="governed-test-raster", + storage_path="storage/uploads/orthophoto.tif", + size_bytes=123, + checksum_sha256=checksum, + crs="EPSG:4326", + bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0}, + source_registry_id=registry_id, + source_snapshot_id=snapshot_id, + data_contract_key="geointel.raster.geotiff", + data_contract_version="1.0.0", + validation_status="passed", + provenance_status="complete", + lineage_status="not_applicable", + quarantine_status="not_quarantined", + status="ready", + ) + dataset.source_registry = registry + dataset.source_snapshot = snapshot + version = DatasetVersion( + id=uuid4(), + dataset_id=dataset_id, + version=3, + checksum_sha256=checksum, + ) + dataset.versions.append(version) + objects = {(Dataset, dataset_id): dataset} + if area is not None: + objects[(Area, area.id)] = area + return FakeSession(objects), dataset, area + + +def _manifest(tmp_path: Path, db: FakeSession, dataset: Dataset) -> Path: + tile_path = tmp_path / "tile_0000.tif" + tile_path.write_bytes(b"immutable tile") + payload = { + **TileManifestService.dataset_binding(db, dataset), + "tile_set_id": "tile-set-1", + "crs": "EPSG:4326", + "source_crs": "EPSG:4326", + "dataset_crs": "EPSG:4326", + "bounds": [4.0, 51.0, 5.0, 52.0], + "count": 1, + "tiles": [ + { + "path": str(tile_path), + "bounds": [4.0, 51.0, 5.0, 52.0], + "crs": "EPSG:4326", + "index": 0, + **TileManifestService.tile_integrity(tile_path), + } + ], + } + manifest_path = tmp_path / "manifest.json" + manifest_path.write_text(canonical_manifest_json(payload), encoding="utf-8") + return manifest_path + + +def _validate(tmp_path: Path, db: FakeSession, dataset: Dataset, *, prefix: str = "DETECTION"): + manifest_path = tmp_path / "manifest.json" + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + return TileManifestService.validate_for_inference( + db, + dataset, + payload, + manifest_path=manifest_path, + settings=Settings(_env_file=None, storage_root=str(tmp_path)), + error_prefix=prefix, + ) + + +def test_versioned_tile_manifest_binds_dataset_version_snapshot_area_and_tile_bytes(tmp_path: Path) -> None: + db, dataset, _area = _dataset_and_session() + manifest_path = _manifest(tmp_path, db, dataset) + + evidence = _validate(tmp_path, db, dataset) + + assert evidence["manifest_path"] == str(manifest_path.resolve()) + assert evidence["source_dataset_id"] == str(dataset.id) + assert evidence["source_snapshot_id"] == str(dataset.source_snapshot_id) + assert evidence["dataset_version_id"] == str(dataset.versions[0].id) + assert evidence["source_area_id"] == str(dataset.area_id) + assert evidence["tile_count"] == 1 + assert evidence["tile_union_bounds_epsg4326"] == pytest.approx([4.0, 51.0, 5.0, 52.0]) + + +@pytest.mark.parametrize( + ("field", "value", "expected_code"), + [ + ("source_dataset_id", lambda: str(uuid4()), "DETECTION_TILE_MANIFEST_DATASET_MISMATCH"), + ("source_dataset_checksum_sha256", lambda: "b" * 64, "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"), + ("source_snapshot_id", lambda: str(uuid4()), "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"), + ("dataset_version", lambda: 99, "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"), + ("source_area_geometry_sha256", lambda: "c" * 64, "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"), + ], +) +def test_tile_manifest_rejects_dataset_or_provenance_mismatch( + tmp_path: Path, + field: str, + value, + expected_code: str, +) -> None: + db, dataset, _area = _dataset_and_session() + manifest_path = _manifest(tmp_path, db, dataset) + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + payload[field] = value() + manifest_path.write_text(canonical_manifest_json(payload), encoding="utf-8") + + with pytest.raises(AppError) as exc_info: + _validate(tmp_path, db, dataset) + + assert exc_info.value.code == expected_code + + +def test_tile_manifest_rejects_tile_bytes_changed_after_manifest_creation(tmp_path: Path) -> None: + db, dataset, _area = _dataset_and_session() + manifest_path = _manifest(tmp_path, db, dataset) + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + Path(payload["tiles"][0]["path"]).write_bytes(b"tampered tile") + + with pytest.raises(AppError) as exc_info: + _validate(tmp_path, db, dataset) + + assert exc_info.value.code == "DETECTION_TILE_MANIFEST_TILE_INTEGRITY_MISMATCH" + + +def test_tile_manifest_rejects_dataset_without_version_binding(tmp_path: Path) -> None: + db, dataset, _area = _dataset_and_session() + _manifest(tmp_path, db, dataset) + dataset.versions.clear() + + with pytest.raises(AppError) as exc_info: + _validate(tmp_path, db, dataset) + + assert exc_info.value.code == "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH" + assert "dataset_version_id" in exc_info.value.details["missing_fields"] + + +def test_segmentation_tile_manifest_rejects_union_outside_dataset_scope(tmp_path: Path) -> None: + db, dataset, _area = _dataset_and_session(with_area=False) + manifest_path = _manifest(tmp_path, db, dataset) + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + payload["bounds"] = [4.0, 51.0, 6.0, 52.0] + payload["tiles"][0]["bounds"] = [4.0, 51.0, 6.0, 52.0] + manifest_path.write_text(canonical_manifest_json(payload), encoding="utf-8") + + with pytest.raises(AppError) as exc_info: + _validate(tmp_path, db, dataset, prefix="SEGMENTATION") + + assert exc_info.value.code == "SEGMENTATION_TILE_MANIFEST_SCOPE_MISMATCH" diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index 5ab67060..40860b1a 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -13,8 +13,8 @@ This document freezes the first API shape. Codex may add implementation details uses the canonical `{"data": ...}` envelope. Readiness runs an OpenAPI audit that rejects free-form dictionary responses and envelope drift. - The only successful non-envelope responses are `/health`, `/health/live`, - `/health/ready`, the four documented persisted-raster PNG endpoints and the - export artifact download endpoint. + `/health/ready`, the two documented Authentik redirects, the documented + persisted-raster PNG endpoints and the export artifact download endpoint. ## Shared schemas @@ -63,7 +63,7 @@ one client/username combination within five minutes temporarily return HTTP Optional guest access is a configuration-gated demonstration mode. It creates a shorter signed session with role `guest`, scopes that session to the -idempotently seeded demo project and blocks mutating operator routes. Project +idempotently seeded demo project and blocks administrative operator routes. Project listing is filtered to the bound demo project. The frontend exposes the same exploration, assistant, model-selection, analysis, QA and export workspaces as an operator. Model catalogs are globally readable; every run, result and export @@ -73,13 +73,33 @@ administrative mutations remain unavailable. This is deliberately **not** a substitute for user accounts, authorization or tenant isolation; expose it only on a dedicated demo installation without private or operational data. +The functional demo boundary includes bounded, project-path-scoped official +source acquisition, persisted bbox selections, temporal comparisons and change +detection. Change detection resolves the source dataset first, requires its +project to equal the signed guest-project UUID and validates the target dataset +against that same project before the synchronous job starts. Cross-project +datasets therefore fail before comparison. Generic uploads, arbitrary +clip/buffer/intersect operations, forceful project management and evidence +adjudication remain blocked. + +The login/demo seed remains offline and retains its synthetic raster as an +explicit `fixture` for UI context and fixture QA only. That raster is excluded +from configured detection and segmentation selectors and the production +consumption gate continues to reject it. A guest starts real model inference +by drawing a bounded map selection; GeoIntel then acquires an official regional +orthophoto on demand inside the signed demo project, persists its contextual +source registry/snapshot/checksum evidence, and only then tiles and runs the +configured model. No network acquisition occurs merely by logging in. + ### GET `/api/v1/auth/session` Public session probe used by the frontend before it mounts the workbench. When authentication is disabled, `authenticated` is true and `authentication_required` is false so local development retains its existing direct workflow. `guest_access_enabled` tells the landing page whether it may -show the guest action. +show the guest action. `authentik_enabled` indicates that the additive +Authentik operator flow is fully configured; the local operator login remains +available as a recovery path. ```json { @@ -90,6 +110,7 @@ show the guest action. "expires_at": null, "role": null, "guest_access_enabled": true, + "authentik_enabled": false, "guest_project_id": null } } @@ -112,6 +133,23 @@ Successful login sets the session cookie and returns the authenticated session shape. Invalid credentials return HTTP 401 `INVALID_CREDENTIALS`; username existence is not disclosed. +### GET `/api/v1/auth/authentik/start` + +Starts an authorization-code OIDC flow with PKCE, signed state and nonce when +all Authentik settings are present. Discovery, token and JWKS requests are +restricted to the configured HTTPS issuer origin, reject redirects and enforce +a bounded JSON response size. The flow cookie is HttpOnly, Secure, ten minutes +or less and scoped to the Authentik callback path. + +### GET `/api/v1/auth/authentik/callback` + +Validates issuer, audience, signature, expiry, state, nonce and the exact +configured verified e-mail address. Success creates the same operator session +as local login and clears the one-use flow cookie. Failure clears that cookie +and redirects to the landing page with a generic error marker; token or +identity details are never returned to the browser. These two redirect routes +are the only additional non-envelope authentication responses. + ### POST `/api/v1/auth/guest` No request body is required. The endpoint is available only when both @@ -130,6 +168,12 @@ is available for that bound demo project. Unscoped analysis routes require the same UUID as a `project_id` query parameter; cross-project values fail before route execution. Coverage resolution additionally verifies the `project_id` in the request body against the guest-session scope. +Raster tiling for an already-persisted raster in the bound demo project is an +explicitly allowed preparation step for detection and segmentation. It creates +only integrity-bound inference tiles and a job record; uploads, source +acquisition, model management and arbitrary derived-dataset writes remain +operator-only. Guest tiling is server-capped by the configured inference tile +limit before any tile bytes are written. ### POST `/api/v1/auth/logout` @@ -286,11 +330,13 @@ Request: Backend responsibilities: -- Validate geometry. -- Repair trivial polygon issues if safe. -- Store geometry in PostGIS. -- Calculate area in square meters using projected CRS. -- Store bbox. +- Accept only a finite, valid `Polygon` or `MultiPolygon` in the declared CRS. +- Reject unknown/non-2D CRS definitions and geometry outside the Belgium and + Belgian North Sea workbench domain. +- Transform the geometry to EPSG:4326 before PostGIS persistence while + retaining the exact declared CRS in `original_crs`. +- Calculate `area_m2` in Belgian Lambert 72 (`EPSG:31370`) and derive the + persisted EPSG:4326 bbox from the normalized geometry. ### GET `/api/v1/projects/{project_id}/areas/{area_id}` @@ -299,8 +345,12 @@ area list endpoint and includes persisted GeoJSON geometry for map display. ### PATCH `/api/v1/projects/{project_id}/areas/{area_id}` -Updates the area name and/or geometry. Geometry updates follow the same -validation, repair and metric-calculation rules as area creation. +Updates the area name and/or geometry. A replacement geometry follows the same +strict CRS validation, EPSG:4326 normalization and metric-calculation rules as +area creation and atomically recomputes `geometry`, `bbox`, `area_m2` and +`original_crs`. A PATCH containing `crs` without `geometry` fails with +`INVALID_AREA_CRS_UPDATE`; omitted CRS on replacement geometry means +EPSG:4326. ### GET `/api/v1/projects/{project_id}/areas/municipalities` @@ -409,10 +459,22 @@ validation report/status, provenance/lineage status, quarantine status and an idempotent ingest key. A malformed or doubtful artifact is retained as `status=quarantined`; it is not silently discarded or made ready. -Vector uploads remain stored as original files and are also persisted into -`vector_features` as queryable PostGIS state only after their contract passes. -Non-EPSG:4326 vector coordinates are explicitly transformed before canonical -feature persistence; relabelling Lambert coordinates as EPSG:4326 is rejected. +The backend streams uploads to governed storage in bounded 8 MiB reads while +calculating size and SHA-256. `GEOINTEL_MAX_UPLOAD_MB` (legacy alias +`MAX_UPLOAD_MB`, default 500, allowed range 1–2,048) is enforced by the backend even when a request +bypasses the reverse proxy. The first byte beyond the configured limit aborts +the ingest, removes the partial file and returns HTTP 413 with +`UPLOAD_TOO_LARGE`. Because GeoJSON validation currently requires an in-memory +parse, vector uploads have the additional lower +`GEOINTEL_MAX_IN_MEMORY_VECTOR_MB` limit (default 64 MiB, maximum 256 MiB). +This limit is enforced during streaming and again with a bounded read before +parsing; larger vector sources must use a governed batch-import workflow. + +Vector uploads are also persisted into `vector_features` as queryable PostGIS +state only after their contract passes. Non-EPSG:4326 source bytes are retained +as provenance evidence while the normal Dataset `storage_path` points to the +deterministic EPSG:4326 artifact. Relabelling Lambert coordinates as EPSG:4326 +is rejected. ### GET `/api/v1/source-registry` @@ -1053,7 +1115,13 @@ Failure modes: ### POST `/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/tile` -Generate raster tiles and a manifest for downstream processing. Returns a job payload with `tile_set_id` and manifest metadata. +Generate raster tiles and a versioned manifest for downstream processing. +Returns a job payload with `tile_set_id` and manifest metadata. Contract +`geointel.raster.tile-manifest@2.0.0` binds the tile set to the exact source +Dataset, DatasetVersion, source registry/snapshot and checksum values that are +available in persistence, plus the source Area and Area-geometry checksum when +scoped. Every tile records immutable byte size/SHA-256, explicit CRS and bounds; +the manifest records its source extent and tile count. If raster processing dependencies are unavailable: @@ -1594,13 +1662,16 @@ label or model filename. ### GET `/api/v1/detection/model-assets` -Returns governed local runtime model files. This is a read-only catalog. +Returns configured local runtime model files. This is a read-only catalog; +catalog visibility establishes byte identity and runtime availability only. GeoIntel never downloads, creates, mutates or deletes model weights from this endpoint. The backend scans `YOLO_MODELS_DIR` (default `/app/models`). When `YOLO_MODEL_PATH` resolves to an existing file, production catalog output is -restricted to that explicitly approved active model. When no active model is +restricted to that active runtime model. Active/available is a runtime-selection +state only: it does not assert governed validation, human review, national +coverage or promotion. When no active model is configured, supported `.pt`, `.onnx` and `.engine` files remain visible for development/operator discovery but cannot make the configured detector ready. @@ -1620,8 +1691,12 @@ Response data: "size_bytes": 123456, "sha256": "sha256hex", "active": true, - "status": "approved", - "limitation_message": "Approved local runtime model asset. GeoIntel will not download or mutate model weights.", + "runtime_available": true, + "runtime_status": "active", + "governed_validation_status": "not_verified_by_catalog", + "promotion_status": "not_verified_by_catalog", + "status": "runtime_active", + "limitation_message": "Active local runtime model asset. Runtime selection is not evidence of governed validation or promotion.", "will_download_models": false } ], @@ -1732,10 +1807,21 @@ governed, runtime-produced artifact the persistence model requires. Rejection is `STORAGE_PATH_OUTSIDE_ROOT`; `GEOINTEL_ALLOW_EXTERNAL_ARTIFACT_PATHS` opts out for provisioning workflows that stage tiles before ingest. +Configured detection and segmentation revalidate the v2 contract before model +loading. The requested Dataset identity, checksum, latest DatasetVersion, +source snapshot and Area binding must still match; every tile checksum is +recomputed; and the EPSG:4326 tile union must remain inside both the declared +manifest extent and persisted raster extent and intersect its Area where one is +bound. Validation fails closed with the typed suffixes +`TILE_MANIFEST_DATASET_MISMATCH`, `TILE_MANIFEST_PROVENANCE_MISMATCH`, +`TILE_MANIFEST_TILE_INTEGRITY_MISMATCH` or +`TILE_MANIFEST_SCOPE_MISMATCH`, prefixed with `DETECTION_` or +`SEGMENTATION_` for the requesting task. + The manifest must carry explicit CRS metadata (`crs`, `source_crs` or -`dataset_crs`). A manifest without it fails with -`DETECTION_TILE_MANIFEST_INVALID` rather than being georeferenced against an -assumed EPSG:4326, which would place detections plausibly but wrongly. +`dataset_crs`). A manifest without it fails rather than being georeferenced +against an assumed EPSG:4326, which would place detections plausibly but +wrongly. Tiles are read with rasterio: the visible RGB bands are selected explicitly and percentile-stretched to 8-bit, so 16-bit and 4-band (RGB + NIR) orthophotos @@ -2095,6 +2181,11 @@ to the synchronous route. A configured model requires an existing `SEGMENTATION_ACCELERATOR_MISCONFIGURED` when CUDA is required. A valid zero-polygon run is shown as an empty model result, never as proof that the AOI contains no relevant objects. +When a raster is selected but no manifest exists, the normal frontend flow +first requests project-scoped 512 px tiles with 64 px overlap and forwards the +server-generated manifest. Inference then applies the complete v2 provenance, +integrity and scope validation before model load; manual server-path entry is +an operator-only advanced control. Validation errors: diff --git a/scripts/audit_api_contracts.py b/scripts/audit_api_contracts.py index bfb31236..01242df2 100644 --- a/scripts/audit_api_contracts.py +++ b/scripts/audit_api_contracts.py @@ -13,6 +13,8 @@ ALLOWED_NON_ENVELOPE_ENDPOINTS = { ("GET", "/health"), ("GET", "/health/live"), ("GET", "/health/ready"), + ("GET", "/api/v1/auth/authentik/start"), + ("GET", "/api/v1/auth/authentik/callback"), ("GET", "/api/v1/exports/{export_id}/download"), ("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/image"), ("GET", "/api/v1/projects/{project_id}/datasets/{dataset_id}/raster/terrain/image"), diff --git a/scripts/smoke_contracts.py b/scripts/smoke_contracts.py index aaf10575..cbf62a35 100644 --- a/scripts/smoke_contracts.py +++ b/scripts/smoke_contracts.py @@ -1,5 +1,11 @@ from pathlib import Path + ROOT = Path(__file__).resolve().parents[1] -missing=[d for d in ["contracts/api","contracts/database","contracts/events"] if not (ROOT/d).exists()] -if missing: raise SystemExit("Missing contract directories: "+", ".join(missing)) +missing = [ + d + for d in ["contracts/api", "contracts/database", "contracts/events"] + if not (ROOT / d).exists() +] +if missing: + raise SystemExit("Missing contract directories: " + ", ".join(missing)) print("Contracts smoke OK")