fix(platform): govern geospatial analysis and raster handoffs
This commit is contained in:
@@ -14,7 +14,7 @@ from uuid import uuid4
|
||||
import pytest
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import AnalysisRun, Detection, Job
|
||||
from app.models import Job
|
||||
from app.services.analysis_job_worker import AnalysisJobWorker
|
||||
from app.services.detection_service import DetectionService
|
||||
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
from geoalchemy2.shape import from_shape, to_shape
|
||||
from pyproj import Transformer
|
||||
import pytest
|
||||
from shapely.geometry import Polygon, mapping
|
||||
from shapely.ops import transform
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Project
|
||||
from app.schemas.area import AreaCreate, AreaUpdate
|
||||
from app.services.area_service import AreaService
|
||||
from app.utils.geometry import area_m2, normalize_area_to_epsg4326
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, objects=None) -> None:
|
||||
self.objects = objects or {}
|
||||
self.added = []
|
||||
self.commits = 0
|
||||
self.refreshes = []
|
||||
|
||||
def get(self, model, item_id):
|
||||
return self.objects.get((model, item_id))
|
||||
|
||||
def add(self, item) -> None:
|
||||
self.added.append(item)
|
||||
|
||||
def commit(self) -> None:
|
||||
self.commits += 1
|
||||
|
||||
def refresh(self, item) -> None:
|
||||
self.refreshes.append(item)
|
||||
|
||||
|
||||
def _wgs84_polygon(offset: float = 0.0) -> Polygon:
|
||||
return Polygon(
|
||||
[
|
||||
(5.00 + offset, 51.00),
|
||||
(5.01 + offset, 51.00),
|
||||
(5.01 + offset, 51.01),
|
||||
(5.00 + offset, 51.01),
|
||||
(5.00 + offset, 51.00),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _to_lambert(geometry: Polygon) -> Polygon:
|
||||
transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
||||
return transform(transformer.transform, geometry)
|
||||
|
||||
|
||||
def test_create_area_transforms_declared_lambert_geometry_before_storage() -> None:
|
||||
project_id = uuid4()
|
||||
db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")})
|
||||
source = _wgs84_polygon()
|
||||
|
||||
area = AreaService.create_area(
|
||||
db,
|
||||
project_id,
|
||||
AreaCreate(name="Lambert AOI", geometry=mapping(_to_lambert(source)), crs="EPSG:31370"),
|
||||
)
|
||||
|
||||
stored = to_shape(area.geometry)
|
||||
assert stored.bounds == pytest.approx(source.bounds, abs=1e-7)
|
||||
assert area.original_crs == "EPSG:31370"
|
||||
assert area.area_m2 == pytest.approx(area_m2(normalize_area_to_epsg4326(mapping(source), "EPSG:4326")[0]))
|
||||
assert area.area_m2 and area.area_m2 > 0
|
||||
assert to_shape(area.bbox).bounds == pytest.approx(source.bounds, abs=1e-7)
|
||||
|
||||
|
||||
def test_patch_area_replaces_geometry_and_recomputes_all_spatial_fields() -> None:
|
||||
area_id = uuid4()
|
||||
project_id = uuid4()
|
||||
original = _wgs84_polygon()
|
||||
normalized, _ = normalize_area_to_epsg4326(mapping(original), "EPSG:4326")
|
||||
area = Area(
|
||||
id=area_id,
|
||||
project_id=project_id,
|
||||
name="Original",
|
||||
geometry=from_shape(normalized, srid=4326),
|
||||
bbox=from_shape(normalized.envelope, srid=4326),
|
||||
original_crs="EPSG:4326",
|
||||
area_m2=area_m2(normalized),
|
||||
)
|
||||
db = FakeSession({(Area, area_id): area})
|
||||
replacement = _wgs84_polygon(offset=0.05)
|
||||
|
||||
updated = AreaService.update_area(
|
||||
db,
|
||||
area_id,
|
||||
AreaUpdate(
|
||||
name="Replacement",
|
||||
geometry=mapping(_to_lambert(replacement)),
|
||||
crs="EPSG:31370",
|
||||
),
|
||||
)
|
||||
|
||||
assert updated.name == "Replacement"
|
||||
assert updated.original_crs == "EPSG:31370"
|
||||
assert to_shape(updated.geometry).bounds == pytest.approx(replacement.bounds, abs=1e-7)
|
||||
assert to_shape(updated.bbox).bounds == pytest.approx(replacement.bounds, abs=1e-7)
|
||||
assert updated.area_m2 and updated.area_m2 > 0
|
||||
assert db.commits == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("geometry", "crs", "message_fragment"),
|
||||
[
|
||||
(mapping(_wgs84_polygon()), "EPSG:not-real", "unknown or invalid"),
|
||||
(mapping(_wgs84_polygon()), "EPSG:4979", "exactly two spatial axes"),
|
||||
(
|
||||
{
|
||||
"type": "Polygon",
|
||||
"coordinates": [[[5.0, 51.0], [float("nan"), 51.0], [5.1, 51.1], [5.0, 51.0]]],
|
||||
},
|
||||
"EPSG:4326",
|
||||
"finite",
|
||||
),
|
||||
(mapping(Polygon([(10.0, 51.0), (10.1, 51.0), (10.1, 51.1), (10.0, 51.0)])), "EPSG:4326", "workbench domain"),
|
||||
(
|
||||
{
|
||||
"type": "Polygon",
|
||||
"coordinates": [[[5.0, 51.0], [5.1, 51.1], [5.1, 51.0], [5.0, 51.1], [5.0, 51.0]]],
|
||||
},
|
||||
"EPSG:4326",
|
||||
"invalid",
|
||||
),
|
||||
({"type": "Point", "coordinates": [5.0, 51.0]}, "EPSG:4326", "Polygon or MultiPolygon"),
|
||||
],
|
||||
)
|
||||
def test_create_area_rejects_invalid_crs_nonfinite_and_out_of_domain_geometry(
|
||||
geometry: dict,
|
||||
crs: str,
|
||||
message_fragment: str,
|
||||
) -> None:
|
||||
project_id = uuid4()
|
||||
db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")})
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
AreaService.create_area(db, project_id, AreaCreate(name="Invalid", geometry=geometry, crs=crs))
|
||||
|
||||
assert exc_info.value.code == "INVALID_GEOMETRY"
|
||||
assert message_fragment in exc_info.value.message
|
||||
assert db.commits == 0
|
||||
|
||||
|
||||
def test_patch_area_rejects_crs_without_replacement_geometry() -> None:
|
||||
area_id = uuid4()
|
||||
area = Area(id=area_id, project_id=uuid4(), name="AOI", original_crs="EPSG:4326")
|
||||
db = FakeSession({(Area, area_id): area})
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
AreaService.update_area(db, area_id, AreaUpdate(crs="EPSG:31370"))
|
||||
|
||||
assert exc_info.value.code == "INVALID_AREA_CRS_UPDATE"
|
||||
assert db.commits == 0
|
||||
@@ -190,6 +190,17 @@ def test_passed_manual_or_experimental_dataset_cannot_cross_production_boundary(
|
||||
assert "experimental_source_not_allowed_for_purpose" in exc_info.value.details["reasons"]
|
||||
|
||||
|
||||
def test_fully_governed_demo_fixture_still_cannot_enter_production_inference() -> None:
|
||||
fixture = _governed_dataset(source_key="fixture", classification="experimental")
|
||||
fixture.source_metadata = {"fixture": True, "usage": "offline demo raster workflow only"}
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetConsumptionGate.assert_eligible(fixture, purpose="production_inference")
|
||||
|
||||
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert "experimental_source_not_allowed_for_purpose" in exc_info.value.details["reasons"]
|
||||
|
||||
|
||||
def test_reference_validation_requires_authoritative_ground_truth_reference() -> None:
|
||||
reference = _governed_dataset()
|
||||
reference.dataset_type = "vector"
|
||||
|
||||
@@ -17,7 +17,9 @@ import pytest
|
||||
|
||||
np = pytest.importorskip("numpy")
|
||||
|
||||
from app.services.flood_hazard_analysis_service import FloodHazardCellStatistics
|
||||
from app.services.flood_hazard_analysis_service import ( # noqa: E402 - optional NumPy gate precedes service import
|
||||
FloodHazardCellStatistics,
|
||||
)
|
||||
|
||||
|
||||
NODATA = -9999.0
|
||||
|
||||
@@ -11,10 +11,20 @@ from fastapi.testclient import TestClient
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import AppError
|
||||
from app.main import app
|
||||
from app.models import AnalysisRun, Dataset, Detection, Job, Project, SourceRegistry, SourceSnapshot
|
||||
from app.models import (
|
||||
AnalysisRun,
|
||||
Dataset,
|
||||
DatasetVersion,
|
||||
Detection,
|
||||
Job,
|
||||
Project,
|
||||
SourceRegistry,
|
||||
SourceSnapshot,
|
||||
)
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||
from app.services.tile_manifest_service import TileManifestService
|
||||
|
||||
|
||||
class FakeSession:
|
||||
@@ -98,6 +108,8 @@ def _project_and_raster_dataset():
|
||||
source_name="test-derived-raster",
|
||||
storage_path="storage/uploads/source.tif",
|
||||
checksum_sha256=checksum,
|
||||
crs="EPSG:4326",
|
||||
bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0},
|
||||
source_registry_id=source_registry_id,
|
||||
source_snapshot_id=source_snapshot_id,
|
||||
data_contract_key="geointel.raster.geotiff",
|
||||
@@ -110,17 +122,22 @@ def _project_and_raster_dataset():
|
||||
)
|
||||
dataset.source_registry = source_registry
|
||||
dataset.source_snapshot = source_snapshot
|
||||
dataset.versions.append(
|
||||
DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1, checksum_sha256=checksum)
|
||||
)
|
||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||
return db, project_id, dataset_id
|
||||
|
||||
|
||||
def _manifest(tmp_path: Path) -> Path:
|
||||
def _manifest(tmp_path: Path, db: FakeSession, dataset: Dataset) -> Path:
|
||||
tile_path = tmp_path / "tile_0000.tif"
|
||||
tile_path.write_bytes(b"tile")
|
||||
binding = TileManifestService.dataset_binding(db, dataset)
|
||||
manifest_path = tmp_path / "manifest.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
**binding,
|
||||
"tile_set_id": "tiles-fixture",
|
||||
"count": 1,
|
||||
"crs": "EPSG:4326",
|
||||
@@ -133,6 +150,7 @@ def _manifest(tmp_path: Path) -> Path:
|
||||
"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01],
|
||||
"crs": "EPSG:4326",
|
||||
"index": 0,
|
||||
**TileManifestService.tile_integrity(tile_path),
|
||||
}
|
||||
],
|
||||
}
|
||||
@@ -226,7 +244,11 @@ def test_model_asset_catalog_lists_supported_local_model_files(tmp_path: Path) -
|
||||
assert asset.size_bytes == len(b"local model")
|
||||
assert len(asset.sha256) == 64
|
||||
assert asset.active is True
|
||||
assert asset.status == "approved"
|
||||
assert asset.runtime_available is True
|
||||
assert asset.runtime_status == "active"
|
||||
assert asset.governed_validation_status == "not_verified_by_catalog"
|
||||
assert asset.promotion_status == "not_verified_by_catalog"
|
||||
assert asset.status == "runtime_active"
|
||||
assert asset.will_download_models is False
|
||||
|
||||
|
||||
@@ -257,7 +279,10 @@ def test_model_asset_catalog_only_exposes_explicit_active_asset_in_runtime(tmp_p
|
||||
assert response.total == 1
|
||||
assert response.items[0].filename == active_file.name
|
||||
assert response.items[0].active is True
|
||||
assert response.items[0].status == "approved"
|
||||
assert response.items[0].runtime_status == "active"
|
||||
assert response.items[0].governed_validation_status == "not_verified_by_catalog"
|
||||
assert response.items[0].promotion_status == "not_verified_by_catalog"
|
||||
assert response.items[0].status == "runtime_active"
|
||||
|
||||
|
||||
def test_model_asset_catalog_rejects_unknown_asset(tmp_path: Path) -> None:
|
||||
@@ -284,6 +309,9 @@ def test_model_assets_api_returns_canonical_envelope(monkeypatch, tmp_path: Path
|
||||
assert payload["data"]["total"] == 1
|
||||
assert payload["data"]["items"][0]["model_asset_id"] == "building-detector-pt"
|
||||
assert payload["data"]["items"][0]["active"] is True
|
||||
assert payload["data"]["items"][0]["runtime_status"] == "active"
|
||||
assert payload["data"]["items"][0]["governed_validation_status"] == "not_verified_by_catalog"
|
||||
assert payload["data"]["items"][0]["promotion_status"] == "not_verified_by_catalog"
|
||||
assert payload["data"]["items"][0]["will_download_models"] is False
|
||||
|
||||
|
||||
@@ -309,7 +337,7 @@ def test_detection_run_persists_selected_model_asset_parameters(tmp_path, monkey
|
||||
model_id="yolo-configured",
|
||||
model_asset_id="building-detector-pt",
|
||||
confidence_threshold=0.5,
|
||||
tile_manifest_path=str(_manifest(tmp_path)),
|
||||
tile_manifest_path=str(_manifest(tmp_path, db, db.get(Dataset, dataset_id))),
|
||||
settings=settings,
|
||||
yolo_adapter_class=MockYoloAdapter,
|
||||
)
|
||||
|
||||
@@ -18,10 +18,10 @@ import pytest
|
||||
np = pytest.importorskip("numpy")
|
||||
rasterio = pytest.importorskip("rasterio")
|
||||
|
||||
from rasterio.transform import from_origin
|
||||
from shapely.geometry import box
|
||||
from rasterio.transform import from_origin # noqa: E402 - optional rasterio gate precedes imports
|
||||
from shapely.geometry import box # noqa: E402 - optional rasterio gate precedes imports
|
||||
|
||||
from app.services.raster_cell_selection import select_cells
|
||||
from app.services.raster_cell_selection import select_cells # noqa: E402 - optional rasterio gate precedes service import
|
||||
|
||||
|
||||
# 100 m cells, origin at the top-left corner of a 3x3 grid.
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import ModuleType, SimpleNamespace
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
from pathlib import Path
|
||||
import importlib
|
||||
from hashlib import sha256
|
||||
|
||||
from geoalchemy2.shape import from_shape
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset, DatasetVersion
|
||||
from app.services.raster_operations_service import RasterOperationsService
|
||||
from app.services.storage_service import StorageService
|
||||
from app.api.routes.datasets import _run_job_sync
|
||||
from shapely.geometry import box
|
||||
import pytest
|
||||
@@ -688,6 +690,104 @@ def test_raster_tile_returns_manifest_payload(monkeypatch, tmp_path) -> None:
|
||||
assert payload["manifest"]["tile_server"] is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("dimension", "expected"),
|
||||
[
|
||||
(512, [0]),
|
||||
(513, [0, 1]),
|
||||
(960, [0, 448]),
|
||||
(961, [0, 448, 449]),
|
||||
],
|
||||
)
|
||||
def test_raster_tile_offsets_use_full_tiles_and_one_unique_edge_start(dimension, expected) -> None:
|
||||
assert RasterOperationsService._tile_offsets(dimension, tile_size=512, step=448) == expected
|
||||
|
||||
|
||||
def test_raster_tile_rejects_limit_before_creating_output(monkeypatch, tmp_path) -> None:
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
source = tmp_path / "large-raster.tif"
|
||||
source.write_bytes(b"source")
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name="large-raster.tif",
|
||||
dataset_type="raster",
|
||||
source="user_upload",
|
||||
storage_path=str(source),
|
||||
original_filename="large-raster.tif",
|
||||
stored_filename="large-raster.tif",
|
||||
content_type="image/tiff",
|
||||
size_bytes=6,
|
||||
)
|
||||
db = FakeSession([dataset])
|
||||
|
||||
class FakeSource:
|
||||
width = 2048
|
||||
height = 2048
|
||||
count = 1
|
||||
crs = None
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return None
|
||||
|
||||
fake_rasterio = SimpleNamespace(open=lambda _path: FakeSource())
|
||||
monkeypatch.setattr(
|
||||
"app.services.raster_operations_service._import_rasterio",
|
||||
lambda: (fake_rasterio, SimpleNamespace()),
|
||||
)
|
||||
tile_root = tmp_path / "tiles-that-must-not-exist"
|
||||
monkeypatch.setattr(
|
||||
StorageService,
|
||||
"raster_tiles_root",
|
||||
staticmethod(lambda *_args: tile_root),
|
||||
)
|
||||
|
||||
with pytest.raises(AppError) as error:
|
||||
RasterOperationsService.tile(db, dataset_id, tile_size=512, overlap=64, max_tiles=1)
|
||||
|
||||
assert error.value.code == "RASTER_TILE_LIMIT_EXCEEDED"
|
||||
assert error.value.details == {"expected_tile_count": 25, "max_tiles": 1}
|
||||
assert not tile_root.exists()
|
||||
|
||||
|
||||
def test_raster_tile_rejects_changed_source_bytes_before_creating_output(monkeypatch, tmp_path) -> None:
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
source = tmp_path / "changed-raster.tif"
|
||||
source.write_bytes(b"changed")
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name="changed-raster.tif",
|
||||
dataset_type="raster",
|
||||
source="user_upload",
|
||||
storage_path=str(source),
|
||||
original_filename="changed-raster.tif",
|
||||
stored_filename="changed-raster.tif",
|
||||
content_type="image/tiff",
|
||||
size_bytes=7,
|
||||
checksum_sha256=sha256(b"original").hexdigest(),
|
||||
data_contract_key="raster.generic",
|
||||
)
|
||||
db = FakeSession([dataset])
|
||||
tile_root = tmp_path / "tiles-that-must-not-exist"
|
||||
monkeypatch.setattr(
|
||||
StorageService,
|
||||
"raster_tiles_root",
|
||||
staticmethod(lambda *_args: tile_root),
|
||||
)
|
||||
|
||||
with pytest.raises(AppError) as error:
|
||||
RasterOperationsService.tile(db, dataset_id)
|
||||
|
||||
assert error.value.code == "DATASET_STORAGE_CHECKSUM_MISMATCH"
|
||||
assert not tile_root.exists()
|
||||
|
||||
|
||||
def test_raster_clip_persists_derived_dataset(monkeypatch, tmp_path) -> None:
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
@@ -1393,4 +1493,3 @@ def test_run_job_sync_serializes_index_job_output_dataset_id(monkeypatch) -> Non
|
||||
assert result["job_type"] == "raster.ndvi"
|
||||
assert result["output_dataset_id"] == str(output_dataset_id)
|
||||
assert result["result_json"]["output_dataset_id"] == str(output_dataset_id)
|
||||
|
||||
|
||||
@@ -9,11 +9,12 @@ import pytest
|
||||
from geoalchemy2.shape import to_shape
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.models import Dataset, Project, Segmentation, SourceRegistry, SourceSnapshot
|
||||
from app.models import Dataset, DatasetVersion, Project, Segmentation, SourceRegistry, SourceSnapshot
|
||||
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
|
||||
from app.services.model_registry_service import ModelRegistryService
|
||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||
from app.services.segmentation_service import SegmentationService
|
||||
from app.services.tile_manifest_service import TileManifestService
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
@@ -122,6 +123,8 @@ def _project_and_dataset(dataset_type: str = "raster"):
|
||||
source_name="digitaal_vlaanderen_orthophoto",
|
||||
storage_path="storage/uploads/ortho.tif",
|
||||
checksum_sha256=checksum,
|
||||
crs="EPSG:4326",
|
||||
bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0},
|
||||
source_registry_id=source_id,
|
||||
source_snapshot_id=snapshot_id,
|
||||
data_contract_key="geointel.raster.geotiff",
|
||||
@@ -134,6 +137,9 @@ def _project_and_dataset(dataset_type: str = "raster"):
|
||||
)
|
||||
dataset.source_registry = source
|
||||
dataset.source_snapshot = snapshot
|
||||
dataset.versions.append(
|
||||
DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1, checksum_sha256=checksum)
|
||||
)
|
||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||
return db, project_id, dataset_id
|
||||
|
||||
@@ -249,28 +255,36 @@ def _write_configured_model_sidecars(
|
||||
)
|
||||
|
||||
|
||||
def _manifest(tmp_path: Path, tile_count: int = 1) -> Path:
|
||||
def _manifest(
|
||||
tmp_path: Path,
|
||||
tile_count: int = 1,
|
||||
*,
|
||||
db: FakeSession | None = None,
|
||||
dataset: Dataset | None = None,
|
||||
) -> Path:
|
||||
tiles = []
|
||||
for index in range(tile_count):
|
||||
tile_path = tmp_path / f"tile_{index:04d}.tif"
|
||||
tile_path.write_bytes(b"fixture")
|
||||
tiles.append(
|
||||
{
|
||||
tile = {
|
||||
"path": str(tile_path),
|
||||
"pixel_window": [0, 0, 100, 100],
|
||||
"bounds": [4.0, 51.0, 5.0, 52.0],
|
||||
"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01],
|
||||
"crs": "EPSG:4326",
|
||||
"index": index,
|
||||
**TileManifestService.tile_integrity(tile_path),
|
||||
}
|
||||
)
|
||||
tiles.append(tile)
|
||||
binding = TileManifestService.dataset_binding(db or FakeSession(), dataset) if dataset is not None else {}
|
||||
manifest_path = tmp_path / "manifest.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
**binding,
|
||||
"tile_set_id": "tiles-fixture",
|
||||
"source_dataset_id": str(uuid4()),
|
||||
"source_raster_id": str(uuid4()),
|
||||
"source_dataset_id": binding.get("source_dataset_id", str(uuid4())),
|
||||
"source_raster_id": binding.get("source_raster_id", str(uuid4())),
|
||||
"crs": "EPSG:4326",
|
||||
"bounds": [4.0, 51.0, 5.0, 52.0],
|
||||
"tile_size": 100,
|
||||
@@ -424,7 +438,9 @@ def test_configured_segmentation_rejects_unbound_model_snapshot_before_adapter_l
|
||||
dataset_id=dataset_id,
|
||||
model_id="yolo-seg-configured",
|
||||
confidence_threshold=0.5,
|
||||
tile_manifest_path=str(_manifest(tmp_path)),
|
||||
tile_manifest_path=str(
|
||||
_manifest(tmp_path, db=db, dataset=db.get(Dataset, dataset_id))
|
||||
),
|
||||
settings=settings,
|
||||
yolo_seg_adapter_class=NeverLoadSegAdapter,
|
||||
sam_adapter_class=ClassAgnosticSamAdapter,
|
||||
@@ -440,7 +456,7 @@ def test_configured_yolo_seg_run_persists_georeferenced_masks(tmp_path: Path) ->
|
||||
db, project_id, dataset_id = _project_and_dataset()
|
||||
settings = _settings(tmp_path)
|
||||
_write_configured_model_sidecars(tmp_path, settings, include_sam=False, db=db)
|
||||
manifest_path = _manifest(tmp_path)
|
||||
manifest_path = _manifest(tmp_path, db=db, dataset=db.get(Dataset, dataset_id))
|
||||
|
||||
response = SegmentationService.run_segmentation(
|
||||
db=db,
|
||||
@@ -479,7 +495,7 @@ def test_configured_sam_run_is_class_agnostic(tmp_path: Path) -> None:
|
||||
db, project_id, dataset_id = _project_and_dataset()
|
||||
settings = _settings(tmp_path)
|
||||
_write_configured_model_sidecars(tmp_path, settings, include_yolo=False, db=db)
|
||||
manifest_path = _manifest(tmp_path)
|
||||
manifest_path = _manifest(tmp_path, db=db, dataset=db.get(Dataset, dataset_id))
|
||||
|
||||
response = SegmentationService.run_segmentation(
|
||||
db=db,
|
||||
|
||||
@@ -15,14 +15,18 @@ import pytest
|
||||
np = pytest.importorskip("numpy")
|
||||
rasterio = pytest.importorskip("rasterio")
|
||||
|
||||
from pyproj import Transformer
|
||||
from rasterio.transform import from_origin
|
||||
from pyproj import Transformer # noqa: E402 - optional rasterio gate precedes geospatial imports
|
||||
from rasterio.transform import from_origin # noqa: E402 - optional rasterio gate precedes geospatial imports
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.models import Dataset
|
||||
from app.schemas.flood_hazard import FloodHazardSelectionRequest
|
||||
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
|
||||
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
|
||||
from app.core.config import Settings # noqa: E402 - optional rasterio gate precedes app imports
|
||||
from app.models import Dataset # noqa: E402 - optional rasterio gate precedes app imports
|
||||
from app.schemas.flood_hazard import FloodHazardSelectionRequest # noqa: E402 - optional rasterio gate precedes app imports
|
||||
from app.services.flood_hazard_acquisition_service import ( # noqa: E402 - optional rasterio gate precedes app imports
|
||||
FloodHazardAcquisitionService,
|
||||
)
|
||||
from app.services.flood_hazard_analysis_service import ( # noqa: E402 - optional rasterio gate precedes app imports
|
||||
FloodHazardAnalysisService,
|
||||
)
|
||||
|
||||
|
||||
TO_4326 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
|
||||
|
||||
@@ -14,7 +14,10 @@ def test_raster_tile_manifest_can_handoff_to_segmentation_lab() -> None:
|
||||
|
||||
assert "segmentationTileManifestPath" in hook
|
||||
assert "setSegmentationTileManifestPath" in hook
|
||||
assert "tile_manifest_path: segmentationTileManifestPath.trim() || null" in hook
|
||||
assert "let manifestPath = segmentationTileManifestPath.trim()" in hook
|
||||
assert "datasetsApi.rasterInspect(projectId, datasetId)" in hook
|
||||
assert "datasetsApi.rasterTile(projectId, datasetId" in hook
|
||||
assert "tile_manifest_path: manifestPath" in hook
|
||||
assert "segmentationTileManifestPath={segmentationTileManifestPath}" in app
|
||||
assert "onSetTileManifestPath={setSegmentationTileManifestPath}" in app
|
||||
assert "onUseTileManifestForSegmentation: useRasterTileManifestForSegmentation" in app
|
||||
@@ -24,4 +27,3 @@ def test_raster_tile_manifest_can_handoff_to_segmentation_lab() -> None:
|
||||
assert "Gebruik voor segmentatie" in raster_controls
|
||||
assert "disabled={!latestRasterTileManifestPath}" in raster_controls
|
||||
assert "Beeldtegelmanifest" in segmentation_lab
|
||||
assert "Beeldtegelmanifest" in segmentation_lab
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
@@ -11,7 +10,7 @@ from shapely.geometry import Polygon, box
|
||||
from app.core.errors import AppError
|
||||
from app.models import Dataset, VectorFeature
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace, read_feature
|
||||
from tests.frontend_contract import assert_calls, assert_wired, read_map_workspace, read_feature
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
@@ -12,8 +12,12 @@ class FakeUploadFile:
|
||||
filename = "real-orthophoto.tif"
|
||||
content_type = "image/tiff"
|
||||
|
||||
async def read(self) -> bytes:
|
||||
return b"fake-raster"
|
||||
def __init__(self) -> None:
|
||||
self._content = b"fake-raster"
|
||||
|
||||
async def read(self, size: int) -> bytes:
|
||||
chunk, self._content = self._content[:size], self._content[size:]
|
||||
return chunk
|
||||
|
||||
|
||||
class FakeSession:
|
||||
@@ -40,16 +44,19 @@ def test_raster_upload_maps_metadata_bounds_resolution_and_bands(monkeypatch) ->
|
||||
project_id = uuid4()
|
||||
db = FakeSession(project_id)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||
lambda **_: {
|
||||
async def persist_upload_file(**_kwargs):
|
||||
return {
|
||||
"storage_path": "/tmp/real-orthophoto.tif",
|
||||
"original_filename": "real-orthophoto.tif",
|
||||
"stored_filename": "real-orthophoto.tif",
|
||||
"content_type": "image/tiff",
|
||||
"size_bytes": 11,
|
||||
"checksum_sha256": "checksum",
|
||||
},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_upload_file",
|
||||
persist_upload_file,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.extract_raster_metadata",
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
|
||||
from pathlib import Path
|
||||
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace, read_feature
|
||||
from tests.frontend_contract import assert_calls, assert_wired, read_map_workspace, read_feature
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from pathlib import Path
|
||||
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace, read_feature
|
||||
from tests.frontend_contract import assert_calls, read_map_workspace, read_feature
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
@@ -251,7 +251,8 @@ def test_end_user_dataset_sources_are_human_readable() -> None:
|
||||
assert "statbel: 'Statbel'" in display
|
||||
assert "getDatasetSourceDisplayName(activeThemeDataset)" in workspace
|
||||
assert "getDatasetSourceDisplayName(resultDataset)" in workspace
|
||||
assert "Zoek optioneel een gemeente" in workspace
|
||||
assert 'aria-label="Optioneel een gemeente zoeken"' in workspace
|
||||
assert 'placeholder="Gemeentenaam of NIS-code"' in workspace
|
||||
assert "latestDatasetBySeries" in catalog
|
||||
assert "Historische meetmomenten" in catalog
|
||||
assert "getDatasetSourceDisplayName(dataset)" in catalog
|
||||
|
||||
@@ -20,6 +20,7 @@ from app.db.session import get_db
|
||||
from app.main import app
|
||||
from app.models import Area, Dataset, DatasetVersion, Job, Project, SourceRegistry, SourceSnapshot
|
||||
from app.schemas.orthophoto import OrthophotoAcquireRequest
|
||||
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
|
||||
from tests.frontend_contract import read_feature
|
||||
|
||||
@@ -262,6 +263,9 @@ def test_regional_orthophoto_products_bind_provider_and_governed_scope(
|
||||
assert snapshot.checksum_sha256 == dataset.checksum_sha256
|
||||
assert snapshot.ingest_status == "ingested"
|
||||
assert snapshot.freshness_status == "current"
|
||||
dataset.source_registry = source
|
||||
dataset.source_snapshot = snapshot
|
||||
assert DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference").eligible is True
|
||||
|
||||
prepared = OrthophotoAcquisitionService._prepared_request(_selection_payload(product_key="1971"), settings)
|
||||
assert prepared["params"]["LAYERS"] == "OKZPAN71VL"
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from pathlib import Path
|
||||
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace
|
||||
from tests.frontend_contract import assert_wired, read_map_workspace
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
@@ -7,7 +7,6 @@ from uuid import uuid4
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import rasterio
|
||||
from fastapi.testclient import TestClient
|
||||
from pyproj import Transformer
|
||||
from rasterio.io import MemoryFile
|
||||
|
||||
@@ -252,7 +252,6 @@ def test_refresh_api_and_frontend_remain_explicit_only() -> None:
|
||||
|
||||
|
||||
def test_map_theme_ranking_prefers_newer_observation_over_feature_count() -> None:
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
workspace = read_map_workspace()
|
||||
observed_sort = workspace.index("const observedAtDifference")
|
||||
feature_tiebreaker = workspace.index("right.feature_count", observed_sort)
|
||||
@@ -262,7 +261,6 @@ def test_map_theme_ranking_prefers_newer_observation_over_feature_count() -> Non
|
||||
|
||||
|
||||
def test_map_workspace_restores_theme_from_selected_dataset() -> None:
|
||||
root = Path(__file__).resolve().parents[2]
|
||||
workspace = read_map_workspace()
|
||||
assert "function themeIdForDataset(" in workspace
|
||||
assert "useState<DataThemeId>(() =>" in workspace
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
import json
|
||||
from pathlib import Path
|
||||
import ssl
|
||||
import sys
|
||||
@@ -31,7 +30,7 @@ if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
import provision_flanders_geographic_scope as flanders_scope # noqa: E402
|
||||
from tests.frontend_contract import read_map_workspace, read_feature
|
||||
from tests.frontend_contract import read_map_workspace, read_feature # noqa: E402
|
||||
|
||||
|
||||
class BinaryResponse:
|
||||
@@ -136,6 +135,32 @@ def test_mdk_probe_parses_capabilities_without_enabling_acquisition() -> None:
|
||||
assert seen["timeout"] == 20
|
||||
|
||||
|
||||
def test_mdk_probe_default_opener_uses_guarded_strict_tls_path(monkeypatch) -> None:
|
||||
seen = {}
|
||||
|
||||
def guarded_factory(expected_url):
|
||||
seen["expected_url"] = expected_url
|
||||
|
||||
def open_request(request, timeout):
|
||||
seen["request_url"] = request.full_url
|
||||
seen["timeout"] = timeout
|
||||
return BinaryResponse(capabilities_xml())
|
||||
|
||||
return open_request
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.mdk_bathymetry_probe_service.guarded_opener",
|
||||
guarded_factory,
|
||||
)
|
||||
|
||||
result = MdkBathymetryProbeService.probe(settings=Settings(_env_file=None))
|
||||
|
||||
assert result["status"] == "reachable"
|
||||
assert seen["expected_url"] == seen["request_url"]
|
||||
assert seen["expected_url"].startswith("https://")
|
||||
assert seen["timeout"] == 20
|
||||
|
||||
|
||||
def test_mdk_probe_reports_tls_failure_and_never_uses_insecure_fallback() -> None:
|
||||
calls = 0
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from pathlib import Path
|
||||
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace, read_feature
|
||||
from tests.frontend_contract import assert_wired, read_map_workspace, read_feature
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
@@ -19,7 +19,7 @@ from app.models import Area, Dataset, Job, Project
|
||||
from app.schemas.grb import GrbAcquireRequest
|
||||
from app.services.dataset_service import DatasetService
|
||||
from app.services.grb_acquisition_service import GrbAcquisitionService
|
||||
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_map_workspace, read_feature
|
||||
from tests.frontend_contract import assert_wired, read_map_workspace, read_feature
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import io
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import zipfile
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
@@ -187,23 +188,31 @@ def test_dataset_upload_persists_vector_features(monkeypatch, tmp_path) -> None:
|
||||
filename = "reference.geojson"
|
||||
content_type = "application/geo+json"
|
||||
|
||||
async def read(self) -> bytes:
|
||||
def __init__(self) -> None:
|
||||
import json
|
||||
|
||||
return json.dumps(payload).encode("utf-8")
|
||||
self._content = json.dumps(payload).encode("utf-8")
|
||||
|
||||
async def read(self, size: int) -> bytes:
|
||||
chunk, self._content = self._content[:size], self._content[size:]
|
||||
return chunk
|
||||
|
||||
storage_path = tmp_path / "reference.geojson"
|
||||
storage_path.write_text("{}", encoding="utf-8")
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||
lambda **_kwargs: {
|
||||
storage_path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
|
||||
async def persist_upload_file(**_kwargs):
|
||||
return {
|
||||
"storage_path": str(storage_path),
|
||||
"original_filename": "reference.geojson",
|
||||
"stored_filename": "reference.geojson",
|
||||
"content_type": "application/geo+json",
|
||||
"size_bytes": 2,
|
||||
"size_bytes": storage_path.stat().st_size,
|
||||
"checksum_sha256": "0" * 64,
|
||||
},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_upload_file",
|
||||
persist_upload_file,
|
||||
)
|
||||
|
||||
result = asyncio.run(
|
||||
@@ -236,19 +245,28 @@ def test_dataset_upload_rolls_back_dataset_and_file_when_vector_indexing_fails(m
|
||||
filename = "invalid.geojson"
|
||||
content_type = "application/geo+json"
|
||||
|
||||
async def read(self) -> bytes:
|
||||
return b'{"type":"FeatureCollection","features":[]}'
|
||||
def __init__(self) -> None:
|
||||
self._content = b'{"type":"FeatureCollection","features":[]}'
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||
lambda **_kwargs: {
|
||||
async def read(self, size: int) -> bytes:
|
||||
chunk, self._content = self._content[:size], self._content[size:]
|
||||
return chunk
|
||||
|
||||
storage_path.write_text('{"type":"FeatureCollection","features":[]}', encoding="utf-8")
|
||||
|
||||
async def persist_upload_file(**_kwargs):
|
||||
return {
|
||||
"storage_path": str(storage_path),
|
||||
"original_filename": "invalid.geojson",
|
||||
"stored_filename": "invalid.geojson",
|
||||
"content_type": "application/geo+json",
|
||||
"size_bytes": 2,
|
||||
"size_bytes": storage_path.stat().st_size,
|
||||
"checksum_sha256": "0" * 64,
|
||||
},
|
||||
}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_upload_file",
|
||||
persist_upload_file,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
VectorFeatureService,
|
||||
|
||||
@@ -13,12 +13,23 @@ from shapely.geometry import box, mapping
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import AppError
|
||||
from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project, SourceRegistry, SourceSnapshot
|
||||
from app.models import (
|
||||
AnalysisRun,
|
||||
Area,
|
||||
Dataset,
|
||||
DatasetVersion,
|
||||
Detection,
|
||||
Job,
|
||||
Project,
|
||||
SourceRegistry,
|
||||
SourceSnapshot,
|
||||
)
|
||||
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.model_registry_service import ModelRegistryService
|
||||
from app.services.model_validation_scope_service import ModelValidationScopeService
|
||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||
from app.services.tile_manifest_service import TileManifestService
|
||||
from app.services.yolo_adapter import YoloDetectionAdapter
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
@@ -189,6 +200,8 @@ def _project_and_dataset(dataset_type: str = "raster"):
|
||||
source_name="test-derived-raster",
|
||||
storage_path="storage/uploads/source.tif",
|
||||
checksum_sha256=checksum,
|
||||
crs="EPSG:4326",
|
||||
bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0},
|
||||
source_registry_id=source_registry_id,
|
||||
source_snapshot_id=source_snapshot_id,
|
||||
data_contract_key="geointel.raster.geotiff",
|
||||
@@ -201,6 +214,9 @@ def _project_and_dataset(dataset_type: str = "raster"):
|
||||
)
|
||||
dataset.source_registry = source_registry
|
||||
dataset.source_snapshot = source_snapshot
|
||||
dataset.versions.append(
|
||||
DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1, checksum_sha256=checksum)
|
||||
)
|
||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||
return db, project_id, dataset_id
|
||||
|
||||
@@ -309,28 +325,34 @@ def _write_model_sidecar(
|
||||
)
|
||||
|
||||
|
||||
def _manifest(tmp_path: Path, tile_count: int = 1) -> Path:
|
||||
def _manifest(tmp_path: Path, tile_count: int = 1, dataset: Dataset | None = None) -> Path:
|
||||
tiles = []
|
||||
for index in range(tile_count):
|
||||
tile_path = tmp_path / f"tile_{index:04d}.tif"
|
||||
tile_path.write_bytes(b"fixture")
|
||||
tiles.append(
|
||||
{
|
||||
tile = {
|
||||
"path": str(tile_path),
|
||||
"pixel_window": [0, 0, 100, 100],
|
||||
"bounds": [4.0, 51.0, 5.0, 52.0],
|
||||
"transform": [4.0, 0.01, 0.0, 52.0, 0.0, -0.01],
|
||||
"crs": "EPSG:4326",
|
||||
"index": index,
|
||||
**TileManifestService.tile_integrity(tile_path),
|
||||
}
|
||||
)
|
||||
tiles.append(tile)
|
||||
binding = (
|
||||
TileManifestService.dataset_binding(SimpleNamespace(get=lambda *_args: None), dataset)
|
||||
if dataset is not None
|
||||
else {}
|
||||
)
|
||||
manifest_path = tmp_path / "manifest.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
**binding,
|
||||
"tile_set_id": "tiles-fixture",
|
||||
"source_dataset_id": str(uuid4()),
|
||||
"source_raster_id": str(uuid4()),
|
||||
"source_dataset_id": binding.get("source_dataset_id", str(uuid4())),
|
||||
"source_raster_id": binding.get("source_raster_id", str(uuid4())),
|
||||
"crs": "EPSG:4326",
|
||||
"bounds": [4.0, 51.0, 5.0, 52.0],
|
||||
"tile_size": 100,
|
||||
@@ -514,7 +536,7 @@ def test_yolo_run_fails_closed_before_adapter_load_without_sidecar(tmp_path: Pat
|
||||
dataset_id=dataset_id,
|
||||
model_id="yolo-configured",
|
||||
confidence_threshold=0.5,
|
||||
tile_manifest_path=str(_manifest(tmp_path)),
|
||||
tile_manifest_path=str(_manifest(tmp_path, dataset=db.get(Dataset, dataset_id))),
|
||||
settings=settings,
|
||||
yolo_adapter_class=AvailableAdapter,
|
||||
)
|
||||
@@ -530,7 +552,7 @@ def test_yolo_run_rejects_manifest_over_tile_limit(tmp_path: Path) -> None:
|
||||
model_path.write_bytes(b"local weights")
|
||||
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_max_tiles=1)
|
||||
_write_model_sidecar(model_path, settings, db=db)
|
||||
manifest_path = _manifest(tmp_path, tile_count=2)
|
||||
manifest_path = _manifest(tmp_path, tile_count=2, dataset=db.get(Dataset, dataset_id))
|
||||
|
||||
result = DetectionService.run_detection(
|
||||
db=db,
|
||||
@@ -609,7 +631,7 @@ def test_yolo_run_rejects_unbound_model_snapshot_before_adapter_load(tmp_path: P
|
||||
dataset_id=dataset_id,
|
||||
model_id="yolo-configured",
|
||||
confidence_threshold=0.5,
|
||||
tile_manifest_path=str(_manifest(tmp_path)),
|
||||
tile_manifest_path=str(_manifest(tmp_path, dataset=db.get(Dataset, dataset_id))),
|
||||
settings=settings,
|
||||
yolo_adapter_class=NeverLoadUnboundModelAdapter,
|
||||
)
|
||||
@@ -638,7 +660,7 @@ def test_yolo_run_persists_mocked_georeferenced_detections(tmp_path: Path) -> No
|
||||
model_path.write_bytes(b"local weights")
|
||||
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_model_version="local-test")
|
||||
_write_model_sidecar(model_path, settings, db=db)
|
||||
manifest_path = _manifest(tmp_path, tile_count=1)
|
||||
manifest_path = _manifest(tmp_path, tile_count=1, dataset=db.get(Dataset, dataset_id))
|
||||
|
||||
result = DetectionService.run_detection(
|
||||
db=db,
|
||||
@@ -678,7 +700,7 @@ def test_yolo_class_filter_is_case_insensitive_and_persists_canonical_class(tmp_
|
||||
model_path.write_bytes(b"local weights")
|
||||
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
||||
_write_model_sidecar(model_path, settings, db=db)
|
||||
manifest_path = _manifest(tmp_path, tile_count=1)
|
||||
manifest_path = _manifest(tmp_path, tile_count=1, dataset=db.get(Dataset, dataset_id))
|
||||
|
||||
result = DetectionService.run_detection(
|
||||
db=db,
|
||||
@@ -706,7 +728,7 @@ def test_yolo_run_suppresses_cross_tile_duplicate_detections(tmp_path: Path) ->
|
||||
model_path.write_bytes(b"local weights")
|
||||
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_duplicate_iou_threshold=0.5)
|
||||
_write_model_sidecar(model_path, settings, db=db)
|
||||
manifest_path = _manifest(tmp_path, tile_count=2)
|
||||
manifest_path = _manifest(tmp_path, tile_count=2, dataset=db.get(Dataset, dataset_id))
|
||||
|
||||
result = DetectionService.run_detection(
|
||||
db=db,
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from pathlib import Path
|
||||
from tests.frontend_contract import assert_calls, assert_mentions, assert_wired, read_feature
|
||||
from tests.frontend_contract import assert_mentions, read_feature
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
import asyncio
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.services.dataset_service import DatasetService
|
||||
from app.services.storage_service import StorageService
|
||||
|
||||
|
||||
@@ -26,3 +32,111 @@ def test_persist_dataset_file_records_metadata(monkeypatch, tmp_path) -> None:
|
||||
assert len(metadata["checksum_sha256"]) == 64
|
||||
assert Path(metadata["storage_path"]).exists()
|
||||
assert str(Path(tmp_path, "uploads", "project-123", "vector", "dataset-456")) in metadata["storage_path"]
|
||||
|
||||
|
||||
class _ChunkedUpload:
|
||||
def __init__(self, content: bytes) -> None:
|
||||
self.content = content
|
||||
self.requested_sizes: list[int] = []
|
||||
|
||||
async def read(self, size: int) -> bytes:
|
||||
self.requested_sizes.append(size)
|
||||
chunk, self.content = self.content[:size], self.content[size:]
|
||||
return chunk
|
||||
|
||||
|
||||
def test_persist_upload_file_streams_bounded_chunks(monkeypatch, tmp_path) -> None:
|
||||
monkeypatch.setattr(StorageService, "_base_dir", staticmethod(lambda: tmp_path))
|
||||
upload = _ChunkedUpload(b"abcdefghijk")
|
||||
|
||||
metadata = asyncio.run(
|
||||
StorageService.persist_upload_file(
|
||||
project_id="project",
|
||||
dataset_id="dataset",
|
||||
dataset_type="raster",
|
||||
original_filename="source.tif",
|
||||
upload=upload,
|
||||
content_type="image/tiff",
|
||||
max_bytes=32,
|
||||
chunk_size=4,
|
||||
)
|
||||
)
|
||||
|
||||
stored = Path(metadata["storage_path"])
|
||||
assert upload.requested_sizes == [4, 4, 4, 4]
|
||||
assert stored.read_bytes() == b"abcdefghijk"
|
||||
assert metadata["size_bytes"] == 11
|
||||
assert metadata["checksum_sha256"] == sha256(b"abcdefghijk").hexdigest()
|
||||
|
||||
|
||||
def test_persist_upload_file_rejects_oversize_and_removes_partial_file(monkeypatch, tmp_path) -> None:
|
||||
monkeypatch.setattr(StorageService, "_base_dir", staticmethod(lambda: tmp_path))
|
||||
upload = _ChunkedUpload(b"0123456789")
|
||||
expected_path = Path(
|
||||
StorageService.dataset_file_path(
|
||||
"project",
|
||||
"dataset",
|
||||
"vector",
|
||||
"source.geojson",
|
||||
)
|
||||
)
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
asyncio.run(
|
||||
StorageService.persist_upload_file(
|
||||
project_id="project",
|
||||
dataset_id="dataset",
|
||||
dataset_type="vector",
|
||||
original_filename="source.geojson",
|
||||
upload=upload,
|
||||
content_type="application/geo+json",
|
||||
max_bytes=8,
|
||||
chunk_size=3,
|
||||
)
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "UPLOAD_TOO_LARGE"
|
||||
assert exc_info.value.status_code == 413
|
||||
assert not expected_path.exists()
|
||||
|
||||
|
||||
def test_vector_staging_uses_lower_in_memory_limit(monkeypatch) -> None:
|
||||
captured = {}
|
||||
|
||||
async def persist_upload_file(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return {"storage_path": "unused"}
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.get_settings",
|
||||
lambda: SimpleNamespace(max_upload_mb=500, max_in_memory_vector_mb=32),
|
||||
)
|
||||
monkeypatch.setattr(StorageService, "persist_upload_file", persist_upload_file)
|
||||
|
||||
asyncio.run(
|
||||
DatasetService._stage_upload(
|
||||
project_id="project",
|
||||
dataset_id="dataset",
|
||||
dataset_type="vector",
|
||||
filename="source.geojson",
|
||||
file=SimpleNamespace(content_type="application/geo+json"),
|
||||
)
|
||||
)
|
||||
|
||||
assert captured["max_bytes"] == 32 * 1024 * 1024
|
||||
|
||||
|
||||
def test_staged_vector_read_is_bounded_before_loading_file(monkeypatch, tmp_path) -> None:
|
||||
source = tmp_path / "large.geojson"
|
||||
source.write_bytes(b"x" * (1024 * 1024 + 1))
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.get_settings",
|
||||
lambda: SimpleNamespace(max_upload_mb=500, max_in_memory_vector_mb=1),
|
||||
)
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetService._read_staged_vector_bytes({"storage_path": str(source)})
|
||||
|
||||
assert exc_info.value.code == "UPLOAD_TOO_LARGE"
|
||||
assert exc_info.value.status_code == 413
|
||||
assert not source.exists()
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from geoalchemy2.shape import from_shape
|
||||
import pytest
|
||||
from shapely.geometry import box
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset, DatasetVersion, SourceRegistry, SourceSnapshot
|
||||
from app.services.tile_manifest_service import TileManifestService, canonical_manifest_json
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, objects=None) -> None:
|
||||
self.objects = objects or {}
|
||||
|
||||
def get(self, model, item_id):
|
||||
return self.objects.get((model, item_id))
|
||||
|
||||
|
||||
def _dataset_and_session(*, with_area: bool = True):
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
registry_id = uuid4()
|
||||
snapshot_id = uuid4()
|
||||
checksum = "a" * 64
|
||||
area = None
|
||||
area_id = uuid4() if with_area else None
|
||||
if area_id is not None:
|
||||
area = Area(
|
||||
id=area_id,
|
||||
project_id=project_id,
|
||||
name="Inference AOI",
|
||||
geometry=from_shape(box(4.0, 51.0, 5.0, 52.0), srid=4326),
|
||||
)
|
||||
registry = SourceRegistry(
|
||||
id=registry_id,
|
||||
source_key="governed-test-raster",
|
||||
display_name="Governed test raster",
|
||||
classification="derived",
|
||||
authority_name="GeoIntel",
|
||||
)
|
||||
snapshot = SourceSnapshot(
|
||||
id=snapshot_id,
|
||||
source_registry_id=registry_id,
|
||||
snapshot_key="snapshot-1",
|
||||
checksum_sha256=checksum,
|
||||
ingest_status="ingested",
|
||||
freshness_status="current",
|
||||
)
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
area_id=area_id,
|
||||
name="orthophoto.tif",
|
||||
dataset_type="raster",
|
||||
source="governed-test-raster",
|
||||
source_name="governed-test-raster",
|
||||
storage_path="storage/uploads/orthophoto.tif",
|
||||
size_bytes=123,
|
||||
checksum_sha256=checksum,
|
||||
crs="EPSG:4326",
|
||||
bounds_json={"min_x": 4.0, "min_y": 51.0, "max_x": 5.0, "max_y": 52.0},
|
||||
source_registry_id=registry_id,
|
||||
source_snapshot_id=snapshot_id,
|
||||
data_contract_key="geointel.raster.geotiff",
|
||||
data_contract_version="1.0.0",
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="not_applicable",
|
||||
quarantine_status="not_quarantined",
|
||||
status="ready",
|
||||
)
|
||||
dataset.source_registry = registry
|
||||
dataset.source_snapshot = snapshot
|
||||
version = DatasetVersion(
|
||||
id=uuid4(),
|
||||
dataset_id=dataset_id,
|
||||
version=3,
|
||||
checksum_sha256=checksum,
|
||||
)
|
||||
dataset.versions.append(version)
|
||||
objects = {(Dataset, dataset_id): dataset}
|
||||
if area is not None:
|
||||
objects[(Area, area.id)] = area
|
||||
return FakeSession(objects), dataset, area
|
||||
|
||||
|
||||
def _manifest(tmp_path: Path, db: FakeSession, dataset: Dataset) -> Path:
|
||||
tile_path = tmp_path / "tile_0000.tif"
|
||||
tile_path.write_bytes(b"immutable tile")
|
||||
payload = {
|
||||
**TileManifestService.dataset_binding(db, dataset),
|
||||
"tile_set_id": "tile-set-1",
|
||||
"crs": "EPSG:4326",
|
||||
"source_crs": "EPSG:4326",
|
||||
"dataset_crs": "EPSG:4326",
|
||||
"bounds": [4.0, 51.0, 5.0, 52.0],
|
||||
"count": 1,
|
||||
"tiles": [
|
||||
{
|
||||
"path": str(tile_path),
|
||||
"bounds": [4.0, 51.0, 5.0, 52.0],
|
||||
"crs": "EPSG:4326",
|
||||
"index": 0,
|
||||
**TileManifestService.tile_integrity(tile_path),
|
||||
}
|
||||
],
|
||||
}
|
||||
manifest_path = tmp_path / "manifest.json"
|
||||
manifest_path.write_text(canonical_manifest_json(payload), encoding="utf-8")
|
||||
return manifest_path
|
||||
|
||||
|
||||
def _validate(tmp_path: Path, db: FakeSession, dataset: Dataset, *, prefix: str = "DETECTION"):
|
||||
manifest_path = tmp_path / "manifest.json"
|
||||
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
return TileManifestService.validate_for_inference(
|
||||
db,
|
||||
dataset,
|
||||
payload,
|
||||
manifest_path=manifest_path,
|
||||
settings=Settings(_env_file=None, storage_root=str(tmp_path)),
|
||||
error_prefix=prefix,
|
||||
)
|
||||
|
||||
|
||||
def test_versioned_tile_manifest_binds_dataset_version_snapshot_area_and_tile_bytes(tmp_path: Path) -> None:
|
||||
db, dataset, _area = _dataset_and_session()
|
||||
manifest_path = _manifest(tmp_path, db, dataset)
|
||||
|
||||
evidence = _validate(tmp_path, db, dataset)
|
||||
|
||||
assert evidence["manifest_path"] == str(manifest_path.resolve())
|
||||
assert evidence["source_dataset_id"] == str(dataset.id)
|
||||
assert evidence["source_snapshot_id"] == str(dataset.source_snapshot_id)
|
||||
assert evidence["dataset_version_id"] == str(dataset.versions[0].id)
|
||||
assert evidence["source_area_id"] == str(dataset.area_id)
|
||||
assert evidence["tile_count"] == 1
|
||||
assert evidence["tile_union_bounds_epsg4326"] == pytest.approx([4.0, 51.0, 5.0, 52.0])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "value", "expected_code"),
|
||||
[
|
||||
("source_dataset_id", lambda: str(uuid4()), "DETECTION_TILE_MANIFEST_DATASET_MISMATCH"),
|
||||
("source_dataset_checksum_sha256", lambda: "b" * 64, "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"),
|
||||
("source_snapshot_id", lambda: str(uuid4()), "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"),
|
||||
("dataset_version", lambda: 99, "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"),
|
||||
("source_area_geometry_sha256", lambda: "c" * 64, "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"),
|
||||
],
|
||||
)
|
||||
def test_tile_manifest_rejects_dataset_or_provenance_mismatch(
|
||||
tmp_path: Path,
|
||||
field: str,
|
||||
value,
|
||||
expected_code: str,
|
||||
) -> None:
|
||||
db, dataset, _area = _dataset_and_session()
|
||||
manifest_path = _manifest(tmp_path, db, dataset)
|
||||
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
payload[field] = value()
|
||||
manifest_path.write_text(canonical_manifest_json(payload), encoding="utf-8")
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
_validate(tmp_path, db, dataset)
|
||||
|
||||
assert exc_info.value.code == expected_code
|
||||
|
||||
|
||||
def test_tile_manifest_rejects_tile_bytes_changed_after_manifest_creation(tmp_path: Path) -> None:
|
||||
db, dataset, _area = _dataset_and_session()
|
||||
manifest_path = _manifest(tmp_path, db, dataset)
|
||||
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
Path(payload["tiles"][0]["path"]).write_bytes(b"tampered tile")
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
_validate(tmp_path, db, dataset)
|
||||
|
||||
assert exc_info.value.code == "DETECTION_TILE_MANIFEST_TILE_INTEGRITY_MISMATCH"
|
||||
|
||||
|
||||
def test_tile_manifest_rejects_dataset_without_version_binding(tmp_path: Path) -> None:
|
||||
db, dataset, _area = _dataset_and_session()
|
||||
_manifest(tmp_path, db, dataset)
|
||||
dataset.versions.clear()
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
_validate(tmp_path, db, dataset)
|
||||
|
||||
assert exc_info.value.code == "DETECTION_TILE_MANIFEST_PROVENANCE_MISMATCH"
|
||||
assert "dataset_version_id" in exc_info.value.details["missing_fields"]
|
||||
|
||||
|
||||
def test_segmentation_tile_manifest_rejects_union_outside_dataset_scope(tmp_path: Path) -> None:
|
||||
db, dataset, _area = _dataset_and_session(with_area=False)
|
||||
manifest_path = _manifest(tmp_path, db, dataset)
|
||||
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
payload["bounds"] = [4.0, 51.0, 6.0, 52.0]
|
||||
payload["tiles"][0]["bounds"] = [4.0, 51.0, 6.0, 52.0]
|
||||
manifest_path.write_text(canonical_manifest_json(payload), encoding="utf-8")
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
_validate(tmp_path, db, dataset, prefix="SEGMENTATION")
|
||||
|
||||
assert exc_info.value.code == "SEGMENTATION_TILE_MANIFEST_SCOPE_MISMATCH"
|
||||
Reference in New Issue
Block a user