Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
425 lines
15 KiB
Python
425 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
from urllib.parse import parse_qs, urlparse
|
|
from uuid import uuid4
|
|
|
|
import numpy as np
|
|
import pytest
|
|
from fastapi.testclient import TestClient
|
|
from geoalchemy2.shape import from_shape
|
|
from rasterio.io import MemoryFile
|
|
from rasterio.transform import from_origin
|
|
from shapely.geometry import MultiPolygon, Polygon
|
|
|
|
from app.core.config import Settings
|
|
from app.core.errors import AppError
|
|
from app.db.session import get_db
|
|
from app.main import app
|
|
from app.models import Area, Dataset, Job, Project
|
|
from app.schemas.official_vector import OfficialVectorAcquireRequest
|
|
from app.services.dataset_service import DatasetService
|
|
from app.services.official_vector_acquisition_service import OfficialVectorAcquisitionService
|
|
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
|
|
from tests.frontend_contract import read_feature
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
|
|
|
|
class FakeQuery:
|
|
def __init__(self, result=None):
|
|
self.result = result
|
|
|
|
def filter(self, *_args):
|
|
return self
|
|
|
|
def order_by(self, *_args):
|
|
return self
|
|
|
|
def all(self):
|
|
return self.result if isinstance(self.result, list) else []
|
|
|
|
|
|
class FakeSession:
|
|
def __init__(self, rows=None, query_result=None):
|
|
self.rows = rows or {}
|
|
self.query_result = query_result
|
|
self.added = []
|
|
|
|
def get(self, model, row_id):
|
|
row = self.rows.get((model, row_id))
|
|
if row is not None:
|
|
return row
|
|
return next((item for item in self.added if isinstance(item, model) and item.id == row_id), None)
|
|
|
|
def add(self, row):
|
|
self.added.append(row)
|
|
|
|
def commit(self):
|
|
return None
|
|
|
|
def rollback(self):
|
|
return None
|
|
|
|
def refresh(self, row):
|
|
return row
|
|
|
|
def query(self, _model):
|
|
return FakeQuery(self.query_result)
|
|
|
|
|
|
class JsonResponse:
|
|
def __init__(self, payload):
|
|
self.content = json.dumps(payload).encode("utf-8")
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
return False
|
|
|
|
def read(self, size=-1):
|
|
return self.content if size < 0 else self.content[:size]
|
|
|
|
|
|
def request(product_key: str, *, area_id=None) -> OfficialVectorAcquireRequest:
|
|
return OfficialVectorAcquireRequest(
|
|
bbox={
|
|
"min_x": 5.15,
|
|
"min_y": 51.18,
|
|
"max_x": 5.17,
|
|
"max_y": 51.20,
|
|
"crs": "EPSG:4326",
|
|
},
|
|
area_id=area_id,
|
|
product_key=product_key,
|
|
force_refresh=True,
|
|
)
|
|
|
|
|
|
def polygon_feature(feature_id: str, *, properties=None) -> dict:
|
|
return {
|
|
"type": "Feature",
|
|
"id": feature_id,
|
|
"geometry": {
|
|
"type": "Polygon",
|
|
"coordinates": [[
|
|
[5.155, 51.185],
|
|
[5.175, 51.185],
|
|
[5.175, 51.195],
|
|
[5.155, 51.195],
|
|
[5.155, 51.185],
|
|
]],
|
|
},
|
|
"properties": properties or {},
|
|
}
|
|
|
|
|
|
def test_product_registries_expose_honest_forest_agriculture_nature_and_soil() -> None:
|
|
raster = {item["key"]: item for item in ThematicRasterAcquisitionService.list_products()}
|
|
vector = {item["key"]: item for item in OfficialVectorAcquisitionService.list_products()}
|
|
|
|
assert raster["forest_land_use_2025"]["included_source_values"] == [12]
|
|
assert raster["agricultural_land_use_2025"]["included_source_values"] == [13, 14]
|
|
assert "geen juridische bosgrens" in raster["forest_land_use_2025"]["limitation_message"].lower()
|
|
assert "geen alz-perceelaangifte" in raster["agricultural_land_use_2025"]["limitation_message"].lower()
|
|
assert {
|
|
"bwk_natura2000_2025",
|
|
"dov_soil_types",
|
|
"spw_picc_buildings",
|
|
"spw_picc_roads",
|
|
"spw_picc_waterways",
|
|
"spw_picc_water_surfaces",
|
|
"spw_flood_hazard_2021",
|
|
"urbis_buildings",
|
|
"urbis_cadastral_parcels",
|
|
"urbis_street_axes",
|
|
"urbis_land_cover_blocks",
|
|
"urbis_forest_parks",
|
|
"urbis_water_surfaces",
|
|
} == set(vector)
|
|
assert vector["bwk_natura2000_2025"]["authority_level"] == "authoritative"
|
|
assert vector["dov_soil_types"]["authority_level"] == "authoritative_historical_baseline"
|
|
assert "1949-1971" in vector["dov_soil_types"]["observation_label"]
|
|
|
|
|
|
def test_land_use_classes_are_converted_to_binary_masks_without_nodata_cast_warning() -> None:
|
|
values = np.asarray([[12.0, 13.0], [14.0, -9999.0]], dtype="float32")
|
|
with MemoryFile() as source_memory:
|
|
with source_memory.open(
|
|
driver="GTiff",
|
|
width=2,
|
|
height=2,
|
|
count=1,
|
|
dtype="float32",
|
|
crs="EPSG:31370",
|
|
transform=from_origin(200_000, 210_020, 10, 10),
|
|
nodata=-9999.0,
|
|
) as source:
|
|
source.write(values, 1)
|
|
from pyproj import Transformer
|
|
to_wgs84 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
|
|
scope = Polygon([
|
|
to_wgs84.transform(200_000, 210_000),
|
|
to_wgs84.transform(200_020, 210_000),
|
|
to_wgs84.transform(200_020, 210_020),
|
|
to_wgs84.transform(200_000, 210_020),
|
|
to_wgs84.transform(200_000, 210_000),
|
|
])
|
|
content, validation = ThematicRasterAcquisitionService._normalize_raster(
|
|
source_memory.read(),
|
|
scope,
|
|
{
|
|
"product": ThematicRasterAcquisitionService._product("forest_land_use_2025"),
|
|
"width": 2,
|
|
"height": 2,
|
|
"bbox_epsg31370": [200_000, 210_000, 200_020, 210_020],
|
|
},
|
|
)
|
|
with MemoryFile(content) as normalized_memory:
|
|
with normalized_memory.open() as normalized:
|
|
output = normalized.read(1, masked=True)
|
|
|
|
assert output.compressed().tolist() == [1.0, 0.0, 0.0]
|
|
assert validation["included_source_values"] == [12]
|
|
assert validation["source_minimum_value"] == 12.0
|
|
assert validation["source_maximum_value"] == 14.0
|
|
|
|
|
|
def test_bwk_wfs_pagination_clips_geometry_and_preserves_semantics() -> None:
|
|
product = OfficialVectorAcquisitionService._product("bwk_natura2000_2025")
|
|
scope_wgs84 = Polygon([
|
|
(5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)
|
|
])
|
|
from shapely.ops import transform
|
|
from app.services.official_vector_acquisition_service import _TO_LAMBERT72
|
|
|
|
scope_metric = transform(_TO_LAMBERT72.transform, scope_wgs84)
|
|
calls = []
|
|
|
|
def opener(raw_request, timeout):
|
|
assert timeout == 180
|
|
calls.append(raw_request.full_url)
|
|
query = parse_qs(urlparse(raw_request.full_url).query)
|
|
assert query["typeNames"] == ["BWK:Bwkhab"]
|
|
assert query["sortBy"] == ["UIDN"]
|
|
feature = polygon_feature(
|
|
"Bwkhab.1",
|
|
properties={"UIDN": 42, "EVAL": "z", "HAB1": "2310", "PHAB1": 60},
|
|
)
|
|
if query.get("startIndex") == ["1"]:
|
|
return JsonResponse({
|
|
"type": "FeatureCollection",
|
|
"numberReturned": 0,
|
|
"features": [],
|
|
})
|
|
return JsonResponse({
|
|
"type": "FeatureCollection",
|
|
"numberReturned": 1,
|
|
"features": [feature],
|
|
})
|
|
|
|
features, transfer = OfficialVectorAcquisitionService._fetch_features(
|
|
product,
|
|
scope_wgs84,
|
|
scope_metric,
|
|
"bounded_selection",
|
|
Settings(_env_file=None, OFFICIAL_VECTOR_PAGE_SIZE=1),
|
|
opener,
|
|
)
|
|
|
|
assert len(calls) == 2
|
|
assert transfer["reference_truncated"] is False
|
|
assert features[0]["id"] == "BWK:Bwkhab:42"
|
|
assert features[0]["properties"]["bwk_evaluation_code"] == "z"
|
|
assert features[0]["properties"]["natura2000_share_percent"] == 60
|
|
assert features[0]["properties"]["geometry_clipped_to_selection"] is True
|
|
|
|
|
|
def test_bwk_rejects_a_non_https_configured_endpoint_before_network_access() -> None:
|
|
product = OfficialVectorAcquisitionService._product("bwk_natura2000_2025")
|
|
scope_wgs84 = Polygon([
|
|
(5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)
|
|
])
|
|
from shapely.ops import transform
|
|
from app.services.official_vector_acquisition_service import _TO_LAMBERT72
|
|
|
|
scope_metric = transform(_TO_LAMBERT72.transform, scope_wgs84)
|
|
|
|
def opener(_request, timeout):
|
|
del _request, timeout
|
|
raise AssertionError("network access must not occur")
|
|
|
|
with pytest.raises(AppError) as exc_info:
|
|
OfficialVectorAcquisitionService._fetch_features(
|
|
product,
|
|
scope_wgs84,
|
|
scope_metric,
|
|
"bounded_selection",
|
|
Settings(_env_file=None, BWK_WFS_URL="http://example.invalid/wfs"),
|
|
opener,
|
|
)
|
|
|
|
assert exc_info.value.code == "OFFICIAL_VECTOR_PROVIDER_INVALID_PAGINATION"
|
|
|
|
|
|
def test_dov_wfs_uses_stable_complete_pagination_and_historical_fields() -> None:
|
|
product = OfficialVectorAcquisitionService._product("dov_soil_types")
|
|
scope_wgs84 = Polygon([
|
|
(5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)
|
|
])
|
|
from shapely.ops import transform
|
|
from app.services.official_vector_acquisition_service import _TO_LAMBERT72
|
|
|
|
scope_metric = transform(_TO_LAMBERT72.transform, scope_wgs84)
|
|
|
|
def opener(raw_request, timeout):
|
|
assert timeout == 180
|
|
query = parse_qs(urlparse(raw_request.full_url).query)
|
|
assert query["typeNames"] == ["bodemkaart:bodemtypes"]
|
|
assert query["sortBy"] == ["gid"]
|
|
return JsonResponse({
|
|
"type": "FeatureCollection",
|
|
"numberMatched": 1,
|
|
"numberReturned": 1,
|
|
"features": [polygon_feature(
|
|
"bodemtypes.7",
|
|
properties={
|
|
"gid": 7,
|
|
"Bodemtype": "Zcg",
|
|
"Gegeneraliseerde_legende": "Droog zand",
|
|
"Drainageklasse": "Matig droog",
|
|
},
|
|
)],
|
|
})
|
|
|
|
features, transfer = OfficialVectorAcquisitionService._fetch_features(
|
|
product,
|
|
scope_wgs84,
|
|
scope_metric,
|
|
"bounded_selection",
|
|
Settings(_env_file=None),
|
|
opener,
|
|
)
|
|
|
|
assert transfer["page_count"] == 1
|
|
assert transfer["candidate_feature_count"] == 1
|
|
assert features[0]["properties"]["soil_type_code"] == "Zcg"
|
|
assert features[0]["properties"]["soil_generalized_legend"] == "Droog zand"
|
|
assert features[0]["properties"]["survey_period"] == "1949-1971"
|
|
|
|
|
|
def test_nature_acquisition_persists_only_through_dataset_service(monkeypatch) -> None:
|
|
project_id, area_id, dataset_id = uuid4(), uuid4(), uuid4()
|
|
municipality = MultiPolygon([Polygon([
|
|
(5.15, 51.18), (5.17, 51.18), (5.17, 51.20), (5.15, 51.20), (5.15, 51.18)
|
|
])])
|
|
db = FakeSession({
|
|
(Project, project_id): Project(id=project_id, name="Vlaanderen"),
|
|
(Area, area_id): Area(
|
|
id=area_id,
|
|
project_id=project_id,
|
|
name="Gemeente Mol",
|
|
geometry=from_shape(municipality, srid=4326),
|
|
),
|
|
})
|
|
captured = {}
|
|
|
|
def opener(_request, timeout):
|
|
del timeout
|
|
return JsonResponse({
|
|
"type": "FeatureCollection",
|
|
"features": [polygon_feature(
|
|
"Bwkhab.1",
|
|
properties={"UIDN": 42, "EVAL": "w", "HAB1": "rbbmr", "PHAB1": 100},
|
|
)],
|
|
"links": [],
|
|
})
|
|
|
|
def persist(_db, **kwargs):
|
|
captured.update(kwargs)
|
|
dataset = Dataset(
|
|
id=dataset_id,
|
|
project_id=project_id,
|
|
area_id=area_id,
|
|
name=kwargs["filename"],
|
|
dataset_type="vector",
|
|
source=kwargs["source"],
|
|
dataset_role=kwargs["dataset_role"],
|
|
source_name=kwargs["source_name"],
|
|
reference_layer_name=kwargs["reference_layer_name"],
|
|
observed_at=kwargs["observed_at"],
|
|
source_version=kwargs["source_version"],
|
|
source_metadata=kwargs["source_metadata"],
|
|
provenance_metadata=kwargs["provenance_metadata"],
|
|
metadata_json={"feature_count": 1},
|
|
status="ready",
|
|
)
|
|
db.rows[(Dataset, dataset_id)] = dataset
|
|
return SimpleNamespace(id=dataset_id)
|
|
|
|
monkeypatch.setattr(DatasetService, "import_vector_bytes", persist)
|
|
result = OfficialVectorAcquisitionService.acquire(
|
|
db,
|
|
project_id,
|
|
request("bwk_natura2000_2025", area_id=area_id),
|
|
settings=Settings(_env_file=None),
|
|
opener=opener,
|
|
)
|
|
|
|
assert result["output_dataset_id"] == str(dataset_id)
|
|
assert captured["dataset_role"] == "reference"
|
|
assert captured["source_name"] == "inbo_bwk_natura2000"
|
|
assert captured["reference_layer_name"] == "nature_value"
|
|
assert captured["source_metadata"]["selection_aggregation"]["metric_key"] == "nature_mapped_area"
|
|
assert captured["source_metadata"]["selection_metrics"][4]["is_estimate"] is True
|
|
assert captured["provenance_metadata"]["reference_truncated"] is False
|
|
assert json.loads(captured["content"])["features"][0]["properties"]["coverage_scope"] == "municipality"
|
|
|
|
|
|
def test_official_vector_routes_and_frontend_use_canonical_backend_path(monkeypatch) -> None:
|
|
project_id, dataset_id = uuid4(), uuid4()
|
|
db = FakeSession({(Project, project_id): Project(id=project_id, name="Vlaanderen")})
|
|
monkeypatch.setattr(
|
|
OfficialVectorAcquisitionService,
|
|
"acquire",
|
|
lambda *_args, **_kwargs: {
|
|
"output_dataset_id": str(dataset_id),
|
|
"product_key": "bwk_natura2000_2025",
|
|
"feature_count": 1,
|
|
},
|
|
)
|
|
app.dependency_overrides[get_db] = lambda: db
|
|
try:
|
|
client = TestClient(app)
|
|
products_response = client.get(
|
|
f"/api/v1/projects/{project_id}/datasets/official-vector/products"
|
|
)
|
|
acquire_response = client.post(
|
|
f"/api/v1/projects/{project_id}/datasets/official-vector/acquire",
|
|
json=request("bwk_natura2000_2025").model_dump(mode="json"),
|
|
)
|
|
finally:
|
|
app.dependency_overrides.clear()
|
|
|
|
assert products_response.status_code == 200
|
|
assert set(products_response.json()) == {"data"}
|
|
assert products_response.json()["data"]["total"] == 13
|
|
assert acquire_response.status_code == 200
|
|
assert set(acquire_response.json()) == {"data"}
|
|
assert acquire_response.json()["data"]["job_type"] == "vector.official.acquire"
|
|
assert any(isinstance(item, Job) for item in db.added)
|
|
|
|
selection_hook = read_feature("map_workspace")
|
|
catalog_hook = read_feature("map_workspace")
|
|
workspace = read_feature("map_workspace")
|
|
assert "datasetsApi.acquireOfficialVector" in selection_hook
|
|
assert "datasetsApi.listOfficialVectorProducts" in catalog_hook
|
|
assert "officialMapProducts.officialVector" in workspace
|
|
assert "officialMapProducts.thematic" in workspace
|
|
assert "result[product.theme] = null" in workspace
|
|
assert "geo.api.vlaanderen.be" not in workspace
|