504 lines
19 KiB
Python
504 lines
19 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from uuid import uuid4
|
|
|
|
import numpy as np
|
|
import pytest
|
|
import rasterio
|
|
from fastapi.testclient import TestClient
|
|
from geoalchemy2.shape import from_shape
|
|
from pyproj import Transformer
|
|
from rasterio.io import MemoryFile
|
|
from rasterio.transform import from_origin
|
|
from shapely.geometry import MultiPolygon, box
|
|
|
|
from app.core.config import Settings
|
|
from app.core.errors import AppError
|
|
from app.db.session import get_db
|
|
from app.main import app
|
|
from app.models import Area, Dataset, DatasetVersion, Job, Project
|
|
from app.schemas.orthophoto import OrthophotoAcquireRequest
|
|
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
class FakeSession:
|
|
def __init__(self, rows: dict[tuple[type, object], object] | None = None, query_result=None):
|
|
self.rows = rows or {}
|
|
self.query_result = query_result
|
|
self.added: list[object] = []
|
|
|
|
def get(self, model, row_id):
|
|
row = self.rows.get((model, row_id))
|
|
if row is not None:
|
|
return row
|
|
return next((item for item in self.added if isinstance(item, model) and item.id == row_id), None)
|
|
|
|
def add(self, row):
|
|
self.added.append(row)
|
|
|
|
def commit(self):
|
|
return None
|
|
|
|
def rollback(self):
|
|
return None
|
|
|
|
def refresh(self, row):
|
|
return row
|
|
|
|
def query(self, _model):
|
|
return FakeQuery(self.query_result)
|
|
|
|
|
|
class FakeQuery:
|
|
def __init__(self, result):
|
|
self.result = result
|
|
|
|
def filter(self, *_args):
|
|
return self
|
|
|
|
def order_by(self, *_args):
|
|
return self
|
|
|
|
def first(self):
|
|
return self.result
|
|
|
|
|
|
class FakeImageResponse:
|
|
def __init__(self, content: bytes):
|
|
self.content = content
|
|
self.headers = {
|
|
"Content-Type": "image/tiff",
|
|
"Content-Length": str(len(content)),
|
|
}
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
return None
|
|
|
|
def read(self, limit: int) -> bytes:
|
|
return self.content[:limit]
|
|
|
|
|
|
def _selection_payload(
|
|
*,
|
|
side_m: float = 512.0,
|
|
force_refresh: bool = True,
|
|
area_id=None,
|
|
product_key: str = "most_recent",
|
|
resolution_m: float | None = None,
|
|
) -> OrthophotoAcquireRequest:
|
|
west, south = 199_000.0, 210_000.0
|
|
transformer = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
|
|
min_lon, min_lat = transformer.transform(west, south)
|
|
max_lon, max_lat = transformer.transform(west + side_m, south + side_m)
|
|
return OrthophotoAcquireRequest(
|
|
bbox={
|
|
"min_x": min_lon,
|
|
"min_y": min_lat,
|
|
"max_x": max_lon,
|
|
"max_y": max_lat,
|
|
"crs": "EPSG:4326",
|
|
},
|
|
area_id=area_id,
|
|
product_key=product_key,
|
|
force_refresh=force_refresh,
|
|
resolution_m=resolution_m,
|
|
)
|
|
|
|
|
|
def _source_tiff(width: int, height: int) -> bytes:
|
|
pixels = np.zeros((3, height, width), dtype=np.uint8)
|
|
pixels[0, :, :] = 92
|
|
pixels[1, :, :] = 126
|
|
pixels[2, :, :] = 84
|
|
with MemoryFile() as memory:
|
|
with memory.open(
|
|
driver="GTiff",
|
|
width=width,
|
|
height=height,
|
|
count=3,
|
|
dtype="uint8",
|
|
transform=from_origin(0, height, 1, 1),
|
|
) as output:
|
|
output.write(pixels)
|
|
return memory.read()
|
|
|
|
|
|
def test_orthophoto_request_is_bounded_and_uses_official_wms_contract() -> None:
|
|
settings = Settings(_env_file=None)
|
|
prepared = OrthophotoAcquisitionService._prepared_request(_selection_payload(), settings)
|
|
|
|
# A north-up WGS84 rectangle becomes slightly wider after the bounded
|
|
# EPSG:31370 transform; the service must still keep it near the requested scale.
|
|
assert 500 <= prepared["width"] <= 540
|
|
assert 500 <= prepared["height"] <= 540
|
|
assert prepared["params"]["CRS"] == "EPSG:31370"
|
|
assert prepared["params"]["LAYERS"] == "Ortho"
|
|
assert "geo.api.vlaanderen.be/OMWRGBMRVL/wms" in prepared["request_url"]
|
|
assert len(prepared["request_hash"]) == 64
|
|
|
|
|
|
def test_training_request_can_use_native_resolution_but_not_oversample_source() -> None:
|
|
settings = Settings(_env_file=None)
|
|
prepared = OrthophotoAcquisitionService._prepared_request(
|
|
_selection_payload(product_key="wallonia_latest", resolution_m=0.25), settings
|
|
)
|
|
assert 2_000 <= prepared["width"] <= 2_120
|
|
assert prepared["resolution_m"] == 0.25
|
|
|
|
with pytest.raises(AppError) as exc_info:
|
|
OrthophotoAcquisitionService._prepared_request(
|
|
_selection_payload(product_key="wallonia_latest", resolution_m=0.1), settings
|
|
)
|
|
assert exc_info.value.code == "ORTHOPHOTO_RESOLUTION_EXCEEDS_SOURCE"
|
|
|
|
|
|
def test_orthophoto_product_registry_exposes_only_governed_official_layers() -> None:
|
|
settings = Settings(_env_file=None)
|
|
products = OrthophotoAcquisitionService.list_products(settings)
|
|
keys = [item["key"] for item in products]
|
|
|
|
assert keys[0] == "most_recent"
|
|
assert {"2025", "2012", "2008_2011", "2000_2003", "1979_1990", "1971"}.issubset(keys)
|
|
assert next(item for item in products if item["key"] == "most_recent")["supports_detection"] is True
|
|
detection_keys = {item["key"] for item in products if item["supports_detection"]}
|
|
assert {"most_recent", "wallonia_latest", "wallonia_2024", "wallonia_2023", "brussels_latest", "brussels_2025", "2025"} <= detection_keys
|
|
by_key = {item["key"]: item for item in products}
|
|
assert by_key["wallonia_latest"]["provider"] == "spw_orthophoto"
|
|
assert by_key["wallonia_latest"]["coverage_zone"] == "wallonia"
|
|
assert by_key["brussels_latest"]["provider"] == "urbis_orthophoto"
|
|
assert by_key["brussels_latest"]["coverage_zone"] == "brussels"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("product_key", "provider", "layer", "coverage_zone"),
|
|
[
|
|
("wallonia_latest", "spw_orthophoto", "0", "wallonia"),
|
|
("brussels_latest", "urbis_orthophoto", "Ortho", "brussels"),
|
|
],
|
|
)
|
|
def test_regional_orthophoto_products_bind_provider_and_governed_scope(
|
|
tmp_path, product_key: str, provider: str, layer: str, coverage_zone: str
|
|
) -> None:
|
|
project_id = uuid4()
|
|
payload = _selection_payload(product_key=product_key)
|
|
scope = Area(
|
|
id=uuid4(),
|
|
project_id=project_id,
|
|
name="Wallonia" if coverage_zone == "wallonia" else "Brussels-Capital Region",
|
|
geometry=from_shape(
|
|
MultiPolygon(
|
|
[
|
|
box(
|
|
payload.bbox.min_x - 0.01,
|
|
payload.bbox.min_y - 0.01,
|
|
payload.bbox.max_x + 0.01,
|
|
payload.bbox.max_y + 0.01,
|
|
)
|
|
]
|
|
),
|
|
srid=4326,
|
|
),
|
|
)
|
|
db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")}, query_result=scope)
|
|
settings = Settings(_env_file=None, storage_root=str(tmp_path), orthophoto_resolution_m=1.0)
|
|
prepared = OrthophotoAcquisitionService._prepared_request(payload, settings)
|
|
|
|
result = OrthophotoAcquisitionService.acquire(
|
|
db,
|
|
project_id,
|
|
payload,
|
|
settings=settings,
|
|
opener=lambda *_args, **_kwargs: FakeImageResponse(
|
|
_source_tiff(prepared["width"], prepared["height"])
|
|
),
|
|
)
|
|
|
|
dataset = next(row for row in db.added if isinstance(row, Dataset))
|
|
assert result["provider"] == provider
|
|
assert result["layer"] == layer
|
|
assert dataset.source_name == provider
|
|
assert dataset.source_metadata["coverage_zone"] == coverage_zone
|
|
assert dataset.source_metadata["license_note"]
|
|
assert dataset.provenance_metadata["request_url"].startswith(prepared["product"].wms_url)
|
|
|
|
prepared = OrthophotoAcquisitionService._prepared_request(_selection_payload(product_key="1971"), settings)
|
|
assert prepared["params"]["LAYERS"] == "OKZPAN71VL"
|
|
assert prepared["wms_url"] == "https://geo.api.vlaanderen.be/OKZ/wms"
|
|
|
|
with pytest.raises(AppError) as exc_info:
|
|
OrthophotoAcquisitionService._prepared_request(_selection_payload(product_key="arbitrary-layer"), settings)
|
|
assert exc_info.value.code == "ORTHOPHOTO_PRODUCT_NOT_SUPPORTED"
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("side_m", "expected_code"),
|
|
[(64.0, "ORTHOPHOTO_SELECTION_TOO_SMALL"), (1_200.0, "ORTHOPHOTO_SELECTION_TOO_LARGE")],
|
|
)
|
|
def test_orthophoto_request_rejects_unsafe_selection_sizes(side_m: float, expected_code: str) -> None:
|
|
with pytest.raises(AppError) as exc_info:
|
|
OrthophotoAcquisitionService._prepared_request(_selection_payload(side_m=side_m), Settings(_env_file=None))
|
|
|
|
assert exc_info.value.code == expected_code
|
|
|
|
|
|
def test_orthophoto_acquisition_persists_georeferenced_raster_and_provenance(tmp_path) -> None:
|
|
project_id = uuid4()
|
|
area_id = uuid4()
|
|
payload = _selection_payload(area_id=area_id)
|
|
area_geometry = MultiPolygon(
|
|
[
|
|
box(
|
|
payload.bbox.min_x - 0.01,
|
|
payload.bbox.min_y - 0.01,
|
|
payload.bbox.max_x + 0.01,
|
|
payload.bbox.max_y + 0.01,
|
|
)
|
|
]
|
|
)
|
|
db = FakeSession(
|
|
{
|
|
(Project, project_id): Project(id=project_id, name="Mol operationele werkruimte"),
|
|
(Area, area_id): Area(
|
|
id=area_id,
|
|
project_id=project_id,
|
|
name="Gemeente Mol",
|
|
geometry=from_shape(area_geometry, srid=4326),
|
|
),
|
|
}
|
|
)
|
|
settings = Settings(_env_file=None, storage_root=str(tmp_path), orthophoto_resolution_m=1.0)
|
|
prepared = OrthophotoAcquisitionService._prepared_request(payload, settings)
|
|
response = FakeImageResponse(_source_tiff(prepared["width"], prepared["height"]))
|
|
|
|
result = OrthophotoAcquisitionService.acquire(
|
|
db,
|
|
project_id,
|
|
payload,
|
|
settings=settings,
|
|
opener=lambda *_args, **_kwargs: response,
|
|
)
|
|
|
|
datasets = [row for row in db.added if isinstance(row, Dataset)]
|
|
versions = [row for row in db.added if isinstance(row, DatasetVersion)]
|
|
assert len(datasets) == 1
|
|
assert len(versions) == 1
|
|
dataset = datasets[0]
|
|
assert result["output_dataset_id"] == str(dataset.id)
|
|
assert result["reused"] is False
|
|
assert dataset.project_id == project_id
|
|
assert dataset.area_id == area_id
|
|
assert dataset.dataset_type == "raster"
|
|
assert dataset.dataset_role == "source"
|
|
assert dataset.source_name == "digitaal_vlaanderen_orthophoto"
|
|
assert dataset.crs == "EPSG:31370"
|
|
assert dataset.provenance_metadata["acquisition"] == "explicit_bounded_map_selection"
|
|
assert dataset.provenance_metadata["request_hash"] == prepared["request_hash"]
|
|
assert dataset.source_metadata["attribution"].startswith("Bron: Orthofotomozaiek Vlaanderen")
|
|
assert dataset.storage_path is not None
|
|
with rasterio.open(dataset.storage_path) as stored:
|
|
assert stored.crs.to_epsg() == 31370
|
|
assert stored.count == 3
|
|
assert stored.width == prepared["width"]
|
|
assert stored.height == prepared["height"]
|
|
assert list(stored.bounds) == pytest.approx(prepared["bbox_epsg31370"], abs=0.01)
|
|
|
|
|
|
def test_orthophoto_acquisition_rejects_selection_outside_persisted_area() -> None:
|
|
project_id = uuid4()
|
|
area_id = uuid4()
|
|
payload = _selection_payload(area_id=area_id)
|
|
db = FakeSession(
|
|
{
|
|
(Project, project_id): Project(id=project_id, name="Mol"),
|
|
(Area, area_id): Area(
|
|
id=area_id,
|
|
project_id=project_id,
|
|
name="Unrelated area",
|
|
geometry=from_shape(MultiPolygon([box(3.0, 50.0, 3.1, 50.1)]), srid=4326),
|
|
),
|
|
}
|
|
)
|
|
|
|
with pytest.raises(AppError) as exc_info:
|
|
OrthophotoAcquisitionService.acquire(db, project_id, payload, settings=Settings(_env_file=None))
|
|
|
|
assert exc_info.value.code == "ORTHOPHOTO_SELECTION_OUTSIDE_AREA"
|
|
|
|
|
|
def test_orthophoto_acquisition_reuses_fresh_exact_request_without_provider_call(tmp_path) -> None:
|
|
project_id = uuid4()
|
|
payload = _selection_payload(force_refresh=False)
|
|
prepared = OrthophotoAcquisitionService._prepared_request(payload, Settings(_env_file=None))
|
|
stored_path = tmp_path / "cached.tif"
|
|
stored_path.write_bytes(b"persisted")
|
|
cached = Dataset(
|
|
id=uuid4(),
|
|
project_id=project_id,
|
|
name=f"orthofoto_most_recent_{prepared['request_hash'][:12]}.tif",
|
|
dataset_type="raster",
|
|
source="Digitaal Vlaanderen",
|
|
source_name="digitaal_vlaanderen_orthophoto",
|
|
status="ready",
|
|
storage_path=str(stored_path),
|
|
imported_at=datetime.now(UTC),
|
|
)
|
|
db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")}, query_result=cached)
|
|
|
|
result = OrthophotoAcquisitionService.acquire(
|
|
db,
|
|
project_id,
|
|
payload,
|
|
settings=Settings(_env_file=None),
|
|
opener=lambda *_args, **_kwargs: pytest.fail("fresh cached request must not call the provider"),
|
|
)
|
|
|
|
assert result["output_dataset_id"] == str(cached.id)
|
|
assert result["reused"] is True
|
|
assert db.added == []
|
|
|
|
|
|
def test_orthophoto_provider_rejects_non_image_response() -> None:
|
|
response = FakeImageResponse(b"<ServiceException>invalid layer</ServiceException>")
|
|
response.headers["Content-Type"] = "text/xml"
|
|
|
|
with pytest.raises(AppError) as exc_info:
|
|
OrthophotoAcquisitionService._fetch(
|
|
"https://geo.api.vlaanderen.be/OMWRGBMRVL/wms",
|
|
Settings(_env_file=None),
|
|
opener=lambda *_args, **_kwargs: response,
|
|
)
|
|
|
|
assert exc_info.value.code == "ORTHOPHOTO_PROVIDER_INVALID_RESPONSE"
|
|
|
|
|
|
def test_historical_orthophoto_persists_temporal_product_provenance(tmp_path) -> None:
|
|
project_id = uuid4()
|
|
payload = _selection_payload(product_key="2020")
|
|
db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")})
|
|
settings = Settings(_env_file=None, storage_root=str(tmp_path), orthophoto_resolution_m=1.0)
|
|
prepared = OrthophotoAcquisitionService._prepared_request(payload, settings)
|
|
response = FakeImageResponse(_source_tiff(prepared["width"], prepared["height"]))
|
|
|
|
result = OrthophotoAcquisitionService.acquire(
|
|
db,
|
|
project_id,
|
|
payload,
|
|
settings=settings,
|
|
opener=lambda *_args, **_kwargs: response,
|
|
)
|
|
|
|
dataset = next(row for row in db.added if isinstance(row, Dataset))
|
|
assert result["product_key"] == "2020"
|
|
assert result["supports_detection"] is False
|
|
assert dataset.observed_at.year == 2020
|
|
assert dataset.temporal_granularity == "year"
|
|
assert dataset.source_metadata["layer"] == "OMWRGB20VL"
|
|
assert dataset.source_metadata["product_key"] == "2020"
|
|
assert dataset.provenance_metadata["spatial_hash"] == prepared["spatial_hash"]
|
|
|
|
|
|
def test_persisted_orthophoto_renders_browser_png(tmp_path) -> None:
|
|
project_id = uuid4()
|
|
dataset_id = uuid4()
|
|
path = tmp_path / "ortho.tif"
|
|
path.write_bytes(_source_tiff(32, 24))
|
|
dataset = Dataset(
|
|
id=dataset_id,
|
|
project_id=project_id,
|
|
name="ortho.tif",
|
|
dataset_type="raster",
|
|
source="Digitaal Vlaanderen",
|
|
source_name="digitaal_vlaanderen_orthophoto",
|
|
status="ready",
|
|
storage_path=str(path),
|
|
)
|
|
db = FakeSession({(Dataset, dataset_id): dataset})
|
|
|
|
png = OrthophotoAcquisitionService.render_png(db, project_id, dataset_id)
|
|
|
|
assert png.startswith(b"\x89PNG\r\n\x1a\n")
|
|
|
|
|
|
def test_orthophoto_endpoint_returns_canonical_job_envelope(monkeypatch) -> None:
|
|
project_id = uuid4()
|
|
output_dataset_id = uuid4()
|
|
db = FakeSession()
|
|
monkeypatch.setattr(
|
|
OrthophotoAcquisitionService,
|
|
"acquire",
|
|
lambda *_args, **_kwargs: {
|
|
"output_dataset_id": str(output_dataset_id),
|
|
"reused": False,
|
|
"provider": "digitaal_vlaanderen_orthophoto",
|
|
},
|
|
)
|
|
payload = _selection_payload(force_refresh=False).model_dump(mode="json")
|
|
app.dependency_overrides[get_db] = lambda: db
|
|
try:
|
|
response = TestClient(app).post(f"/api/v1/projects/{project_id}/datasets/orthophoto/acquire", json=payload)
|
|
finally:
|
|
app.dependency_overrides.clear()
|
|
|
|
assert response.status_code == 200
|
|
body = response.json()
|
|
assert set(body) == {"data"}
|
|
assert body["data"]["status"] == "success"
|
|
assert body["data"]["job_type"] == "raster.orthophoto.acquire"
|
|
assert body["data"]["output_dataset_id"] == str(output_dataset_id)
|
|
assert body["data"]["result_json"]["provider"] == "digitaal_vlaanderen_orthophoto"
|
|
assert any(isinstance(row, Job) for row in db.added)
|
|
|
|
|
|
def test_orthophoto_product_endpoint_returns_canonical_envelope() -> None:
|
|
project_id = uuid4()
|
|
db = FakeSession({(Project, project_id): Project(id=project_id, name="Mol")})
|
|
app.dependency_overrides[get_db] = lambda: db
|
|
try:
|
|
response = TestClient(app).get(f"/api/v1/projects/{project_id}/datasets/orthophoto/products")
|
|
finally:
|
|
app.dependency_overrides.clear()
|
|
|
|
assert response.status_code == 200
|
|
body = response.json()
|
|
assert set(body) == {"data"}
|
|
assert body["data"]["total"] == len(body["data"]["items"])
|
|
assert body["data"]["items"][0]["key"] == "most_recent"
|
|
|
|
|
|
def test_frontend_connects_map_selection_to_existing_detection_and_qa_flows() -> None:
|
|
app_source = (ROOT / "frontend" / "src" / "App.tsx").read_text(encoding="utf-8")
|
|
hook_source = (ROOT / "frontend" / "src" / "hooks" / "useMapOrthophotoAnalysis.ts").read_text(encoding="utf-8")
|
|
map_source = (ROOT / "frontend" / "src" / "components" / "map" / "MapWorkspace.tsx").read_text(encoding="utf-8")
|
|
|
|
assert "useMapOrthophotoAnalysis" in app_source
|
|
assert "onRunOrthophotoAnalysis={mapOrthophotoAnalysis.run}" in app_source
|
|
assert "datasetsApi.acquireOrthophoto" in hook_source
|
|
assert "prepareAndRunDetection(datasetId)" in hook_source
|
|
assert "compareDetectionRunWithReference" in hook_source
|
|
assert "compareDetectionRunWithReference(analysisRunId, referenceDatasetId, false, iouThreshold)" in app_source
|
|
assert "MAP_BUILDING_QA_IOU_THRESHOLD = 0.25" in hook_source
|
|
assert "vervoerregio|operationele grens" in app_source
|
|
assert "selectionBbox: mapSelectionBbox" in app_source
|
|
assert "Maak de rechthoek minstens 128 bij 128 meter groot." in hook_source
|
|
assert "Herken gebouwen" in map_source
|
|
assert "Officieel luchtbeeld, lokaal AI-model" in map_source
|
|
|
|
|
|
def test_unraid_runtime_exposes_bounded_orthophoto_settings() -> None:
|
|
compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8")
|
|
runner = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(encoding="utf-8")
|
|
template = (ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml").read_text(encoding="utf-8")
|
|
|
|
for name in ("ORTHOPHOTO_ENABLED", "ORTHOPHOTO_WMS_URL", "ORTHOPHOTO_RESOLUTION_M", "ORTHOPHOTO_MAX_SIDE_M"):
|
|
assert name in compose
|
|
assert name in runner
|
|
assert name in template
|