505 lines
18 KiB
Python
505 lines
18 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
import ssl
|
|
import sys
|
|
from types import SimpleNamespace
|
|
from urllib.error import URLError
|
|
from uuid import uuid4
|
|
|
|
from fastapi.testclient import TestClient
|
|
from geoalchemy2.shape import from_shape
|
|
import pytest
|
|
from shapely.geometry import MultiPolygon, Polygon
|
|
|
|
from app.core.config import Settings
|
|
from app.core.errors import AppError
|
|
from app.db.session import get_db
|
|
from app.main import app
|
|
from app.models import Area, Dataset, DatasetVersion, Project
|
|
from app.schemas.bathymetry import BathymetryPartitionFinalizeRequest
|
|
from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
|
|
from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService
|
|
from app.services.vector_feature_service import VectorFeatureService
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
SCRIPTS = ROOT / "scripts"
|
|
if str(SCRIPTS) not in sys.path:
|
|
sys.path.insert(0, str(SCRIPTS))
|
|
|
|
import provision_flanders_geographic_scope as flanders_scope # noqa: E402
|
|
from tests.frontend_contract import read_map_workspace, read_feature # noqa: E402
|
|
|
|
|
|
class BinaryResponse:
|
|
def __init__(self, content: bytes, *, content_type: str = "application/xml"):
|
|
self.content = content
|
|
self.headers = {"Content-Type": content_type}
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *_args):
|
|
return False
|
|
|
|
def read(self, size=-1):
|
|
return self.content if size < 0 else self.content[:size]
|
|
|
|
|
|
class JsonResponse:
|
|
def __init__(self, payload, *, url="https://geo.api.vlaanderen.be/VRBG/items"):
|
|
self.payload = payload
|
|
self.url = url
|
|
|
|
def raise_for_status(self):
|
|
return None
|
|
|
|
def json(self):
|
|
return self.payload
|
|
|
|
|
|
class SourceSession:
|
|
def __init__(self, payload):
|
|
self.payload = payload
|
|
|
|
def get(self, *_args, **_kwargs):
|
|
return JsonResponse(self.payload)
|
|
|
|
|
|
class VersionQuery:
|
|
def __init__(self, versions):
|
|
self.versions = versions
|
|
|
|
def filter(self, *_args):
|
|
return self
|
|
|
|
def all(self):
|
|
return self.versions
|
|
|
|
|
|
class FinalizeSession:
|
|
def __init__(self, rows, versions=None):
|
|
self.rows = rows
|
|
self.versions = versions or []
|
|
self.commit_count = 0
|
|
|
|
def get(self, model, row_id):
|
|
return self.rows.get((model, row_id))
|
|
|
|
def query(self, model):
|
|
assert model is DatasetVersion
|
|
return VersionQuery(self.versions)
|
|
|
|
def commit(self):
|
|
self.commit_count += 1
|
|
|
|
|
|
def capabilities_xml() -> bytes:
|
|
return b"""<?xml version="1.0"?>
|
|
<WCS_Capabilities version="1.0.0" xmlns="http://www.opengis.net/wcs">
|
|
<ContentMetadata>
|
|
<CoverageOfferingBrief>
|
|
<name>EL.GridCoverage</name>
|
|
<label>Belgian Continental Shelf bathymetry</label>
|
|
<lonLatEnvelope srsName="urn:ogc:def:crs:OGC:1.3:CRS84" />
|
|
</CoverageOfferingBrief>
|
|
</ContentMetadata>
|
|
<Capability><Request><GetCoverage><resultFormat><formats>GeoTIFF</formats></resultFormat></GetCoverage></Request></Capability>
|
|
</WCS_Capabilities>"""
|
|
|
|
|
|
def test_mdk_probe_parses_capabilities_without_enabling_acquisition() -> None:
|
|
seen = {}
|
|
|
|
def opener(request, timeout):
|
|
seen["url"] = request.full_url
|
|
seen["timeout"] = timeout
|
|
return BinaryResponse(capabilities_xml())
|
|
|
|
result = MdkBathymetryProbeService.probe(
|
|
settings=Settings(_env_file=None),
|
|
opener=opener,
|
|
checked_at=datetime(2026, 7, 17, tzinfo=UTC),
|
|
)
|
|
|
|
assert result["status"] == "reachable"
|
|
assert result["tls_verified"] is True
|
|
assert result["capabilities_reachable"] is True
|
|
assert result["acquisition_supported"] is False
|
|
assert result["coverage_identifiers"] == ["EL.GridCoverage"]
|
|
assert result["advertised_formats"] == ["GeoTIFF"]
|
|
assert result["response_sha256"]
|
|
assert "request=GetCapabilities" in seen["url"]
|
|
assert seen["timeout"] == 20
|
|
|
|
|
|
def test_mdk_probe_default_opener_uses_guarded_strict_tls_path(monkeypatch) -> None:
|
|
seen = {}
|
|
|
|
def guarded_factory(expected_url):
|
|
seen["expected_url"] = expected_url
|
|
|
|
def open_request(request, timeout):
|
|
seen["request_url"] = request.full_url
|
|
seen["timeout"] = timeout
|
|
return BinaryResponse(capabilities_xml())
|
|
|
|
return open_request
|
|
|
|
monkeypatch.setattr(
|
|
"app.services.mdk_bathymetry_probe_service.guarded_opener",
|
|
guarded_factory,
|
|
)
|
|
|
|
result = MdkBathymetryProbeService.probe(settings=Settings(_env_file=None))
|
|
|
|
assert result["status"] == "reachable"
|
|
assert seen["expected_url"] == seen["request_url"]
|
|
assert seen["expected_url"].startswith("https://")
|
|
assert seen["timeout"] == 20
|
|
|
|
|
|
def test_mdk_probe_reports_tls_failure_and_never_uses_insecure_fallback() -> None:
|
|
calls = 0
|
|
|
|
def opener(_request, timeout):
|
|
nonlocal calls
|
|
assert timeout == 20
|
|
calls += 1
|
|
raise URLError(ssl.SSLCertVerificationError("hostname mismatch"))
|
|
|
|
result = MdkBathymetryProbeService.probe(
|
|
settings=Settings(_env_file=None),
|
|
opener=opener,
|
|
)
|
|
|
|
assert calls == 1
|
|
assert result["status"] == "tls_error"
|
|
assert result["tls_verified"] is False
|
|
assert result["capabilities_reachable"] is False
|
|
assert "insecure fallback is prohibited" in result["message"]
|
|
|
|
|
|
def test_mdk_readiness_api_uses_canonical_envelope(monkeypatch) -> None:
|
|
project_id = uuid4()
|
|
db = SimpleNamespace(get=lambda model, row_id: Project(id=project_id, name="Mol") if model is Project else None)
|
|
monkeypatch.setattr(
|
|
MdkBathymetryProbeService,
|
|
"probe",
|
|
lambda: {
|
|
"source_key": "mdk_bcp_bathymetry",
|
|
"status": "tls_error",
|
|
"configured_url": "https://example.invalid/wcs",
|
|
"tls_verified": False,
|
|
"capabilities_reachable": False,
|
|
"acquisition_supported": False,
|
|
"checked_at": "2026-07-18T00:00:00Z",
|
|
"message": "TLS validation failed.",
|
|
"limitation_message": "No insecure fallback is permitted.",
|
|
},
|
|
)
|
|
app.dependency_overrides[get_db] = lambda: db
|
|
try:
|
|
response = TestClient(app).get(
|
|
f"/api/v1/projects/{project_id}/datasets/bathymetry/sources/mdk_bcp_bathymetry/readiness"
|
|
)
|
|
finally:
|
|
app.dependency_overrides.clear()
|
|
|
|
assert response.status_code == 200
|
|
assert set(response.json()) == {"data"}
|
|
assert response.json()["data"]["status"] == "tls_error"
|
|
assert response.json()["data"]["acquisition_supported"] is False
|
|
|
|
|
|
def test_partition_finalization_requires_complete_area_accounting_and_updates_versions() -> None:
|
|
project_id = uuid4()
|
|
area_ids = [uuid4(), uuid4()]
|
|
dataset_id = uuid4()
|
|
project = Project(id=project_id, name="Flanders")
|
|
areas = [
|
|
Area(id=area_ids[0], project_id=project_id, name="Gemeente Mol - officiële grens"),
|
|
Area(id=area_ids[1], project_id=project_id, name="Gemeente Geel - officiële grens"),
|
|
]
|
|
dataset = Dataset(
|
|
id=dataset_id,
|
|
project_id=project_id,
|
|
area_id=area_ids[0],
|
|
name="vha.geojson",
|
|
dataset_type="vector",
|
|
source="VHA",
|
|
source_name=BathymetryProfileAcquisitionService.PROVIDER,
|
|
source_metadata={
|
|
"profile_count": 3,
|
|
"document_count": 2,
|
|
"structured_depth_count": 1,
|
|
"measurement_date_min": "1990-01-01",
|
|
"measurement_date_max": "2020-01-01",
|
|
},
|
|
provenance_metadata={},
|
|
status="ready",
|
|
)
|
|
version = DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1)
|
|
rows = {
|
|
(Project, project_id): project,
|
|
(Area, area_ids[0]): areas[0],
|
|
(Area, area_ids[1]): areas[1],
|
|
(Dataset, dataset_id): dataset,
|
|
}
|
|
db = FinalizeSession(rows, [version])
|
|
payload = BathymetryPartitionFinalizeRequest(
|
|
partition_scope_key="flanders",
|
|
expected_area_ids=area_ids,
|
|
dataset_ids=[dataset_id],
|
|
no_profile_area_ids=[area_ids[1]],
|
|
manifest_sha256="a" * 64,
|
|
observed_at=datetime(2026, 7, 17, tzinfo=UTC),
|
|
)
|
|
|
|
result = BathymetryProfileAcquisitionService.finalize_partitions(db, project_id, payload)
|
|
|
|
assert result["regional_partitions_complete"] is True
|
|
assert result["partition_count"] == 2
|
|
assert result["data_partition_count"] == 1
|
|
assert result["no_profile_partition_count"] == 1
|
|
assert result["profile_count"] == 3
|
|
assert dataset.source_metadata["coverage_scope"] == "flanders"
|
|
assert dataset.source_metadata["municipality"] == "Mol"
|
|
assert dataset.source_metadata["partitioned_source_audit"] is True
|
|
assert version.source_metadata == dataset.source_metadata
|
|
assert version.provenance_metadata == dataset.provenance_metadata
|
|
assert db.commit_count == 1
|
|
|
|
incomplete = payload.model_copy(update={"no_profile_area_ids": []})
|
|
with pytest.raises(AppError) as exc_info:
|
|
BathymetryProfileAcquisitionService.finalize_partitions(
|
|
FinalizeSession(rows, [version]),
|
|
project_id,
|
|
incomplete,
|
|
)
|
|
assert exc_info.value.code == "BATHYMETRY_PARTITION_MANIFEST_INCOMPLETE"
|
|
|
|
|
|
def test_partition_selection_uses_latest_complete_manifest_without_duplicates() -> None:
|
|
project_id = uuid4()
|
|
source_name = BathymetryProfileAcquisitionService.PROVIDER
|
|
|
|
def partition(area_id, manifest, observed_at, data_partition_count):
|
|
return Dataset(
|
|
id=uuid4(),
|
|
project_id=project_id,
|
|
area_id=area_id,
|
|
name="vha.geojson",
|
|
dataset_type="vector",
|
|
source="VHA",
|
|
source_name=source_name,
|
|
source_metadata={
|
|
"regional_partitions_complete": True,
|
|
"partition_scope_key": "flanders",
|
|
"partition_manifest_sha256": manifest,
|
|
"partition_manifest_observed_at": observed_at,
|
|
"data_partition_count": data_partition_count,
|
|
},
|
|
status="ready",
|
|
)
|
|
|
|
old = [partition(uuid4(), "a" * 64, "2026-07-16T00:00:00+00:00", 1)]
|
|
new_area_ids = [uuid4(), uuid4()]
|
|
new = [
|
|
partition(area_id, "b" * 64, "2026-07-17T00:00:00+00:00", 2)
|
|
for area_id in new_area_ids
|
|
]
|
|
incomplete = [partition(uuid4(), "c" * 64, "2026-07-18T00:00:00+00:00", 2)]
|
|
|
|
selected = VectorFeatureService._latest_complete_partition_manifest(
|
|
[*old, *new, *incomplete],
|
|
source_name=source_name,
|
|
partition_scope_key="flanders",
|
|
)
|
|
|
|
assert {dataset.area_id for dataset in selected} == set(new_area_ids)
|
|
assert all(dataset.source_metadata["partition_manifest_sha256"] == "b" * 64 for dataset in selected)
|
|
|
|
|
|
def test_partition_selection_route_uses_exact_municipality_and_canonical_envelope(monkeypatch) -> None:
|
|
project_id, area_id, dataset_id = uuid4(), uuid4(), uuid4()
|
|
area_geometry = MultiPolygon(
|
|
[
|
|
Polygon(
|
|
[
|
|
(5.0, 51.0),
|
|
(5.2, 51.0),
|
|
(5.2, 51.2),
|
|
(5.0, 51.2),
|
|
(5.0, 51.0),
|
|
]
|
|
)
|
|
]
|
|
)
|
|
area = Area(
|
|
id=area_id,
|
|
project_id=project_id,
|
|
name="Gemeente Mol - officiele grens",
|
|
geometry=from_shape(area_geometry, srid=4326),
|
|
)
|
|
db = SimpleNamespace(get=lambda model, row_id: area if model is Area and row_id == area_id else None)
|
|
captured = {}
|
|
|
|
def select_partitions(_db, **kwargs):
|
|
captured.update(kwargs)
|
|
return {
|
|
"selection_bbox": kwargs["bbox"],
|
|
"selection_area_id": area_id,
|
|
"feature_count": 1,
|
|
"total_feature_count": 1,
|
|
"limit": kwargs["limit"],
|
|
"truncated": False,
|
|
"geojson": {"type": "FeatureCollection", "features": []},
|
|
"partition_count": 1,
|
|
"available_partition_count": 269,
|
|
"partition_scope_key": "flanders",
|
|
"source_name": BathymetryProfileAcquisitionService.PROVIDER,
|
|
"dataset_ids": [dataset_id],
|
|
}
|
|
|
|
monkeypatch.setattr(VectorFeatureService, "select_partitioned_features_by_bbox", select_partitions)
|
|
app.dependency_overrides[get_db] = lambda: db
|
|
try:
|
|
response = TestClient(app).post(
|
|
f"/api/v1/projects/{project_id}/datasets/bathymetry/profiles/partitions/select",
|
|
json={
|
|
"bbox": {
|
|
"min_x": 5.0,
|
|
"min_y": 51.0,
|
|
"max_x": 5.2,
|
|
"max_y": 51.2,
|
|
"crs": "EPSG:4326",
|
|
},
|
|
"area_id": str(area_id),
|
|
"limit": 1000,
|
|
},
|
|
)
|
|
finally:
|
|
app.dependency_overrides.clear()
|
|
|
|
assert response.status_code == 200
|
|
assert set(response.json()) == {"data"}
|
|
assert response.json()["data"]["partition_count"] == 1
|
|
assert response.json()["data"]["available_partition_count"] == 269
|
|
assert captured["partition_area_id"] == area_id
|
|
assert captured["source_name"] == BathymetryProfileAcquisitionService.PROVIDER
|
|
assert captured["partition_scope_key"] == "flanders"
|
|
|
|
|
|
def test_flanders_scope_discovery_uses_complete_unique_vrbg_inventory() -> None:
|
|
features = [
|
|
{
|
|
"type": "Feature",
|
|
"id": f"Refgem.{index:05d}",
|
|
"properties": {"NISCODE": f"{index:05d}", "NAAM": f"Gemeente {index:03d}"},
|
|
"geometry": {
|
|
"type": "Polygon",
|
|
"coordinates": [[[4.0, 50.5], [4.1, 50.5], [4.1, 50.6], [4.0, 50.5]]],
|
|
},
|
|
}
|
|
for index in range(1, 286)
|
|
]
|
|
scope, selected, source_url = flanders_scope.discover_flanders_scope(
|
|
SourceSession({"features": list(reversed(features))}),
|
|
timeout=30,
|
|
min_municipalities=270,
|
|
max_municipalities=300,
|
|
)
|
|
|
|
assert scope.key == "flanders"
|
|
assert scope.project_name == "Flanders Regional Workbench"
|
|
assert len(scope.members) == 285
|
|
assert len(set(scope.nis_codes)) == 285
|
|
assert [item["properties"]["NISCODE"] for item in selected] == sorted(scope.nis_codes)
|
|
assert source_url.startswith("https://geo.api.vlaanderen.be/")
|
|
|
|
|
|
def test_expansion_scripts_are_packaged_and_readiness_checked() -> None:
|
|
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
|
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
|
for script in (
|
|
"provision_flanders_geographic_scope.py",
|
|
"provision_flanders_bathymetry_profiles.py",
|
|
"probe_mdk_bathymetry.py",
|
|
):
|
|
assert f"COPY scripts/{script}" in dockerfile
|
|
assert f"py_compile scripts/{script}" in readiness
|
|
|
|
sources = {
|
|
item["key"]: item
|
|
for item in BathymetryProfileAcquisitionService.list_sources()
|
|
}
|
|
assert sources["mdk_bcp_bathymetry"]["integration_status"] == "probe_only"
|
|
# Bounded acquisition is implemented but remains disabled by default.
|
|
assert sources["mdk_bcp_bathymetry"]["acquisition_supported"] is True
|
|
assert sources["mdk_bcp_bathymetry"]["configured"] is False
|
|
assert "EL_wcs" in sources["mdk_bcp_bathymetry"]["service_url"]
|
|
|
|
|
|
def test_frontend_exhaustively_pages_regional_area_inventory() -> None:
|
|
area_api = (ROOT / "frontend" / "src" / "services" / "api" / "areas.ts").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
assert "const AREA_PAGE_SIZE = 200" in area_api
|
|
assert "while (offset < (total ?? 0))" in area_api
|
|
assert "items.length !== total" in area_api
|
|
assert "list: listProjectAreas" in area_api
|
|
|
|
|
|
def test_frontend_bounds_large_area_and_dataset_catalogs() -> None:
|
|
area_panel = (
|
|
ROOT / "frontend" / "src" / "components" / "project" / "AreaPanel.tsx"
|
|
).read_text(encoding="utf-8")
|
|
dataset_panel = read_feature("datasets")
|
|
|
|
assert "const AREA_CATALOG_PAGE_SIZE = 12" in area_panel
|
|
assert "{catalogOpen ? (" in area_panel
|
|
assert "visibleAreas.map" in area_panel
|
|
assert "Zoek gemeente of regio" in area_panel
|
|
assert "const DATASET_CATALOG_PAGE_SIZE = 10" in dataset_panel
|
|
assert "visiblePrimaryDatasets.map" in dataset_panel
|
|
assert "Zoek in beschikbare bronnen" in dataset_panel
|
|
assert "{historyOpen ? <ul" in dataset_panel
|
|
|
|
|
|
def test_frontend_labels_flanders_scope_without_kempen_mislabeling() -> None:
|
|
dataset_display = (
|
|
ROOT / "frontend" / "src" / "lib" / "datasetDisplay.ts"
|
|
).read_text(encoding="utf-8")
|
|
|
|
assert "coverageScope === 'flanders' && layer === 'regional_boundary'" in dataset_display
|
|
assert "'Grens Vlaanderen'" in dataset_display
|
|
assert "coverageScope === 'flanders' && layer === 'municipality_boundaries'" in dataset_display
|
|
assert "'Gemeentegrenzen Vlaanderen'" in dataset_display
|
|
|
|
|
|
def test_frontend_uses_partitioned_bathymetry_selection_for_regional_scope() -> None:
|
|
app = read_feature("shell")
|
|
index = (ROOT / "frontend" / "index.html").read_text(encoding="utf-8")
|
|
focus = (
|
|
ROOT / "frontend" / "src" / "config" / "primaryFocus.ts"
|
|
).read_text(encoding="utf-8")
|
|
map_workspace = read_map_workspace()
|
|
theme_hook = read_feature("map_workspace")
|
|
dataset_api = read_feature("datasets")
|
|
|
|
assert "isPartitionedBathymetry" in map_workspace
|
|
assert "regionalPartitionedThemeActive" in map_workspace
|
|
assert "availabilityLabel" in map_workspace
|
|
assert "activeSelectionResult?.geojson ?? null" in map_workspace
|
|
assert "selectBathymetryProfilePartitions" in theme_hook
|
|
assert "datasets/bathymetry/profiles/partitions/select" in dataset_api
|
|
assert "regionalBathymetryContextActive" in app
|
|
assert "regionalBathymetryProfileCount" in app
|
|
assert "profielen · ${regionalBathymetryPartitions.length} gemeenten" in app
|
|
assert "FLANDERS_WORKSPACE_LABEL = 'Vlaanderen (285 gemeenten)'" in focus
|
|
assert "<title>GeoIntel</title>" in index
|