Initial public release

backend/tests/test_sprint236_bathymetry_expansion.py

@@ -0,0 +1,504 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from pathlib import Path
+import ssl
+import sys
+from types import SimpleNamespace
+from urllib.error import URLError
+from uuid import uuid4
+
+from fastapi.testclient import TestClient
+from geoalchemy2.shape import from_shape
+import pytest
+from shapely.geometry import MultiPolygon, Polygon
+
+from app.core.config import Settings
+from app.core.errors import AppError
+from app.db.session import get_db
+from app.main import app
+from app.models import Area, Dataset, DatasetVersion, Project
+from app.schemas.bathymetry import BathymetryPartitionFinalizeRequest
+from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
+from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService
+from app.services.vector_feature_service import VectorFeatureService
+
+
+ROOT = Path(__file__).resolve().parents[2]
+SCRIPTS = ROOT / "scripts"
+if str(SCRIPTS) not in sys.path:
+    sys.path.insert(0, str(SCRIPTS))
+
+import provision_flanders_geographic_scope as flanders_scope  # noqa: E402
+from tests.frontend_contract import read_map_workspace, read_feature  # noqa: E402
+
+
+class BinaryResponse:
+    def __init__(self, content: bytes, *, content_type: str = "application/xml"):
+        self.content = content
+        self.headers = {"Content-Type": content_type}
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_args):
+        return False
+
+    def read(self, size=-1):
+        return self.content if size < 0 else self.content[:size]
+
+
+class JsonResponse:
+    def __init__(self, payload, *, url="https://geo.api.vlaanderen.be/VRBG/items"):
+        self.payload = payload
+        self.url = url
+
+    def raise_for_status(self):
+        return None
+
+    def json(self):
+        return self.payload
+
+
+class SourceSession:
+    def __init__(self, payload):
+        self.payload = payload
+
+    def get(self, *_args, **_kwargs):
+        return JsonResponse(self.payload)
+
+
+class VersionQuery:
+    def __init__(self, versions):
+        self.versions = versions
+
+    def filter(self, *_args):
+        return self
+
+    def all(self):
+        return self.versions
+
+
+class FinalizeSession:
+    def __init__(self, rows, versions=None):
+        self.rows = rows
+        self.versions = versions or []
+        self.commit_count = 0
+
+    def get(self, model, row_id):
+        return self.rows.get((model, row_id))
+
+    def query(self, model):
+        assert model is DatasetVersion
+        return VersionQuery(self.versions)
+
+    def commit(self):
+        self.commit_count += 1
+
+
+def capabilities_xml() -> bytes:
+    return b"""<?xml version="1.0"?>
+    <WCS_Capabilities version="1.0.0" xmlns="http://www.opengis.net/wcs">
+      <ContentMetadata>
+        <CoverageOfferingBrief>
+          <name>EL.GridCoverage</name>
+          <label>Belgian Continental Shelf bathymetry</label>
+          <lonLatEnvelope srsName="urn:ogc:def:crs:OGC:1.3:CRS84" />
+        </CoverageOfferingBrief>
+      </ContentMetadata>
+      <Capability><Request><GetCoverage><resultFormat><formats>GeoTIFF</formats></resultFormat></GetCoverage></Request></Capability>
+    </WCS_Capabilities>"""
+
+
+def test_mdk_probe_parses_capabilities_without_enabling_acquisition() -> None:
+    seen = {}
+
+    def opener(request, timeout):
+        seen["url"] = request.full_url
+        seen["timeout"] = timeout
+        return BinaryResponse(capabilities_xml())
+
+    result = MdkBathymetryProbeService.probe(
+        settings=Settings(_env_file=None),
+        opener=opener,
+        checked_at=datetime(2026, 7, 17, tzinfo=UTC),
+    )
+
+    assert result["status"] == "reachable"
+    assert result["tls_verified"] is True
+    assert result["capabilities_reachable"] is True
+    assert result["acquisition_supported"] is False
+    assert result["coverage_identifiers"] == ["EL.GridCoverage"]
+    assert result["advertised_formats"] == ["GeoTIFF"]
+    assert result["response_sha256"]
+    assert "request=GetCapabilities" in seen["url"]
+    assert seen["timeout"] == 20
+
+
+def test_mdk_probe_default_opener_uses_guarded_strict_tls_path(monkeypatch) -> None:
+    seen = {}
+
+    def guarded_factory(expected_url):
+        seen["expected_url"] = expected_url
+
+        def open_request(request, timeout):
+            seen["request_url"] = request.full_url
+            seen["timeout"] = timeout
+            return BinaryResponse(capabilities_xml())
+
+        return open_request
+
+    monkeypatch.setattr(
+        "app.services.mdk_bathymetry_probe_service.guarded_opener",
+        guarded_factory,
+    )
+
+    result = MdkBathymetryProbeService.probe(settings=Settings(_env_file=None))
+
+    assert result["status"] == "reachable"
+    assert seen["expected_url"] == seen["request_url"]
+    assert seen["expected_url"].startswith("https://")
+    assert seen["timeout"] == 20
+
+
+def test_mdk_probe_reports_tls_failure_and_never_uses_insecure_fallback() -> None:
+    calls = 0
+
+    def opener(_request, timeout):
+        nonlocal calls
+        assert timeout == 20
+        calls += 1
+        raise URLError(ssl.SSLCertVerificationError("hostname mismatch"))
+
+    result = MdkBathymetryProbeService.probe(
+        settings=Settings(_env_file=None),
+        opener=opener,
+    )
+
+    assert calls == 1
+    assert result["status"] == "tls_error"
+    assert result["tls_verified"] is False
+    assert result["capabilities_reachable"] is False
+    assert "insecure fallback is prohibited" in result["message"]
+
+
+def test_mdk_readiness_api_uses_canonical_envelope(monkeypatch) -> None:
+    project_id = uuid4()
+    db = SimpleNamespace(get=lambda model, row_id: Project(id=project_id, name="Mol") if model is Project else None)
+    monkeypatch.setattr(
+        MdkBathymetryProbeService,
+        "probe",
+        lambda: {
+            "source_key": "mdk_bcp_bathymetry",
+            "status": "tls_error",
+            "configured_url": "https://example.invalid/wcs",
+            "tls_verified": False,
+            "capabilities_reachable": False,
+            "acquisition_supported": False,
+            "checked_at": "2026-07-18T00:00:00Z",
+            "message": "TLS validation failed.",
+            "limitation_message": "No insecure fallback is permitted.",
+        },
+    )
+    app.dependency_overrides[get_db] = lambda: db
+    try:
+        response = TestClient(app).get(
+            f"/api/v1/projects/{project_id}/datasets/bathymetry/sources/mdk_bcp_bathymetry/readiness"
+        )
+    finally:
+        app.dependency_overrides.clear()
+
+    assert response.status_code == 200
+    assert set(response.json()) == {"data"}
+    assert response.json()["data"]["status"] == "tls_error"
+    assert response.json()["data"]["acquisition_supported"] is False
+
+
+def test_partition_finalization_requires_complete_area_accounting_and_updates_versions() -> None:
+    project_id = uuid4()
+    area_ids = [uuid4(), uuid4()]
+    dataset_id = uuid4()
+    project = Project(id=project_id, name="Flanders")
+    areas = [
+        Area(id=area_ids[0], project_id=project_id, name="Gemeente Mol - officiële grens"),
+        Area(id=area_ids[1], project_id=project_id, name="Gemeente Geel - officiële grens"),
+    ]
+    dataset = Dataset(
+        id=dataset_id,
+        project_id=project_id,
+        area_id=area_ids[0],
+        name="vha.geojson",
+        dataset_type="vector",
+        source="VHA",
+        source_name=BathymetryProfileAcquisitionService.PROVIDER,
+        source_metadata={
+            "profile_count": 3,
+            "document_count": 2,
+            "structured_depth_count": 1,
+            "measurement_date_min": "1990-01-01",
+            "measurement_date_max": "2020-01-01",
+        },
+        provenance_metadata={},
+        status="ready",
+    )
+    version = DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1)
+    rows = {
+        (Project, project_id): project,
+        (Area, area_ids[0]): areas[0],
+        (Area, area_ids[1]): areas[1],
+        (Dataset, dataset_id): dataset,
+    }
+    db = FinalizeSession(rows, [version])
+    payload = BathymetryPartitionFinalizeRequest(
+        partition_scope_key="flanders",
+        expected_area_ids=area_ids,
+        dataset_ids=[dataset_id],
+        no_profile_area_ids=[area_ids[1]],
+        manifest_sha256="a" * 64,
+        observed_at=datetime(2026, 7, 17, tzinfo=UTC),
+    )
+
+    result = BathymetryProfileAcquisitionService.finalize_partitions(db, project_id, payload)
+
+    assert result["regional_partitions_complete"] is True
+    assert result["partition_count"] == 2
+    assert result["data_partition_count"] == 1
+    assert result["no_profile_partition_count"] == 1
+    assert result["profile_count"] == 3
+    assert dataset.source_metadata["coverage_scope"] == "flanders"
+    assert dataset.source_metadata["municipality"] == "Mol"
+    assert dataset.source_metadata["partitioned_source_audit"] is True
+    assert version.source_metadata == dataset.source_metadata
+    assert version.provenance_metadata == dataset.provenance_metadata
+    assert db.commit_count == 1
+
+    incomplete = payload.model_copy(update={"no_profile_area_ids": []})
+    with pytest.raises(AppError) as exc_info:
+        BathymetryProfileAcquisitionService.finalize_partitions(
+            FinalizeSession(rows, [version]),
+            project_id,
+            incomplete,
+        )
+    assert exc_info.value.code == "BATHYMETRY_PARTITION_MANIFEST_INCOMPLETE"
+
+
+def test_partition_selection_uses_latest_complete_manifest_without_duplicates() -> None:
+    project_id = uuid4()
+    source_name = BathymetryProfileAcquisitionService.PROVIDER
+
+    def partition(area_id, manifest, observed_at, data_partition_count):
+        return Dataset(
+            id=uuid4(),
+            project_id=project_id,
+            area_id=area_id,
+            name="vha.geojson",
+            dataset_type="vector",
+            source="VHA",
+            source_name=source_name,
+            source_metadata={
+                "regional_partitions_complete": True,
+                "partition_scope_key": "flanders",
+                "partition_manifest_sha256": manifest,
+                "partition_manifest_observed_at": observed_at,
+                "data_partition_count": data_partition_count,
+            },
+            status="ready",
+        )
+
+    old = [partition(uuid4(), "a" * 64, "2026-07-16T00:00:00+00:00", 1)]
+    new_area_ids = [uuid4(), uuid4()]
+    new = [
+        partition(area_id, "b" * 64, "2026-07-17T00:00:00+00:00", 2)
+        for area_id in new_area_ids
+    ]
+    incomplete = [partition(uuid4(), "c" * 64, "2026-07-18T00:00:00+00:00", 2)]
+
+    selected = VectorFeatureService._latest_complete_partition_manifest(
+        [*old, *new, *incomplete],
+        source_name=source_name,
+        partition_scope_key="flanders",
+    )
+
+    assert {dataset.area_id for dataset in selected} == set(new_area_ids)
+    assert all(dataset.source_metadata["partition_manifest_sha256"] == "b" * 64 for dataset in selected)
+
+
+def test_partition_selection_route_uses_exact_municipality_and_canonical_envelope(monkeypatch) -> None:
+    project_id, area_id, dataset_id = uuid4(), uuid4(), uuid4()
+    area_geometry = MultiPolygon(
+        [
+            Polygon(
+                [
+                    (5.0, 51.0),
+                    (5.2, 51.0),
+                    (5.2, 51.2),
+                    (5.0, 51.2),
+                    (5.0, 51.0),
+                ]
+            )
+        ]
+    )
+    area = Area(
+        id=area_id,
+        project_id=project_id,
+        name="Gemeente Mol - officiele grens",
+        geometry=from_shape(area_geometry, srid=4326),
+    )
+    db = SimpleNamespace(get=lambda model, row_id: area if model is Area and row_id == area_id else None)
+    captured = {}
+
+    def select_partitions(_db, **kwargs):
+        captured.update(kwargs)
+        return {
+            "selection_bbox": kwargs["bbox"],
+            "selection_area_id": area_id,
+            "feature_count": 1,
+            "total_feature_count": 1,
+            "limit": kwargs["limit"],
+            "truncated": False,
+            "geojson": {"type": "FeatureCollection", "features": []},
+            "partition_count": 1,
+            "available_partition_count": 269,
+            "partition_scope_key": "flanders",
+            "source_name": BathymetryProfileAcquisitionService.PROVIDER,
+            "dataset_ids": [dataset_id],
+        }
+
+    monkeypatch.setattr(VectorFeatureService, "select_partitioned_features_by_bbox", select_partitions)
+    app.dependency_overrides[get_db] = lambda: db
+    try:
+        response = TestClient(app).post(
+            f"/api/v1/projects/{project_id}/datasets/bathymetry/profiles/partitions/select",
+            json={
+                "bbox": {
+                    "min_x": 5.0,
+                    "min_y": 51.0,
+                    "max_x": 5.2,
+                    "max_y": 51.2,
+                    "crs": "EPSG:4326",
+                },
+                "area_id": str(area_id),
+                "limit": 1000,
+            },
+        )
+    finally:
+        app.dependency_overrides.clear()
+
+    assert response.status_code == 200
+    assert set(response.json()) == {"data"}
+    assert response.json()["data"]["partition_count"] == 1
+    assert response.json()["data"]["available_partition_count"] == 269
+    assert captured["partition_area_id"] == area_id
+    assert captured["source_name"] == BathymetryProfileAcquisitionService.PROVIDER
+    assert captured["partition_scope_key"] == "flanders"
+
+
+def test_flanders_scope_discovery_uses_complete_unique_vrbg_inventory() -> None:
+    features = [
+        {
+            "type": "Feature",
+            "id": f"Refgem.{index:05d}",
+            "properties": {"NISCODE": f"{index:05d}", "NAAM": f"Gemeente {index:03d}"},
+            "geometry": {
+                "type": "Polygon",
+                "coordinates": [[[4.0, 50.5], [4.1, 50.5], [4.1, 50.6], [4.0, 50.5]]],
+            },
+        }
+        for index in range(1, 286)
+    ]
+    scope, selected, source_url = flanders_scope.discover_flanders_scope(
+        SourceSession({"features": list(reversed(features))}),
+        timeout=30,
+        min_municipalities=270,
+        max_municipalities=300,
+    )
+
+    assert scope.key == "flanders"
+    assert scope.project_name == "Flanders Regional Workbench"
+    assert len(scope.members) == 285
+    assert len(set(scope.nis_codes)) == 285
+    assert [item["properties"]["NISCODE"] for item in selected] == sorted(scope.nis_codes)
+    assert source_url.startswith("https://geo.api.vlaanderen.be/")
+
+
+def test_expansion_scripts_are_packaged_and_readiness_checked() -> None:
+    dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
+    readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
+    for script in (
+        "provision_flanders_geographic_scope.py",
+        "provision_flanders_bathymetry_profiles.py",
+        "probe_mdk_bathymetry.py",
+    ):
+        assert f"COPY scripts/{script}" in dockerfile
+        assert f"py_compile scripts/{script}" in readiness
+
+    sources = {
+        item["key"]: item
+        for item in BathymetryProfileAcquisitionService.list_sources()
+    }
+    assert sources["mdk_bcp_bathymetry"]["integration_status"] == "probe_only"
+    # Bounded acquisition is implemented but remains disabled by default.
+    assert sources["mdk_bcp_bathymetry"]["acquisition_supported"] is True
+    assert sources["mdk_bcp_bathymetry"]["configured"] is False
+    assert "EL_wcs" in sources["mdk_bcp_bathymetry"]["service_url"]
+
+
+def test_frontend_exhaustively_pages_regional_area_inventory() -> None:
+    area_api = (ROOT / "frontend" / "src" / "services" / "api" / "areas.ts").read_text(
+        encoding="utf-8"
+    )
+
+    assert "const AREA_PAGE_SIZE = 200" in area_api
+    assert "while (offset < (total ?? 0))" in area_api
+    assert "items.length !== total" in area_api
+    assert "list: listProjectAreas" in area_api
+
+
+def test_frontend_bounds_large_area_and_dataset_catalogs() -> None:
+    area_panel = (
+        ROOT / "frontend" / "src" / "components" / "project" / "AreaPanel.tsx"
+    ).read_text(encoding="utf-8")
+    dataset_panel = read_feature("datasets")
+
+    assert "const AREA_CATALOG_PAGE_SIZE = 12" in area_panel
+    assert "{catalogOpen ? (" in area_panel
+    assert "visibleAreas.map" in area_panel
+    assert "Zoek gemeente of regio" in area_panel
+    assert "const DATASET_CATALOG_PAGE_SIZE = 10" in dataset_panel
+    assert "visiblePrimaryDatasets.map" in dataset_panel
+    assert "Zoek in beschikbare bronnen" in dataset_panel
+    assert "{historyOpen ? <ul" in dataset_panel
+
+
+def test_frontend_labels_flanders_scope_without_kempen_mislabeling() -> None:
+    dataset_display = (
+        ROOT / "frontend" / "src" / "lib" / "datasetDisplay.ts"
+    ).read_text(encoding="utf-8")
+
+    assert "coverageScope === 'flanders' && layer === 'regional_boundary'" in dataset_display
+    assert "'Grens Vlaanderen'" in dataset_display
+    assert "coverageScope === 'flanders' && layer === 'municipality_boundaries'" in dataset_display
+    assert "'Gemeentegrenzen Vlaanderen'" in dataset_display
+
+
+def test_frontend_uses_partitioned_bathymetry_selection_for_regional_scope() -> None:
+    app = read_feature("shell")
+    index = (ROOT / "frontend" / "index.html").read_text(encoding="utf-8")
+    focus = (
+        ROOT / "frontend" / "src" / "config" / "primaryFocus.ts"
+    ).read_text(encoding="utf-8")
+    map_workspace = read_map_workspace()
+    theme_hook = read_feature("map_workspace")
+    dataset_api = read_feature("datasets")
+
+    assert "isPartitionedBathymetry" in map_workspace
+    assert "regionalPartitionedThemeActive" in map_workspace
+    assert "availabilityLabel" in map_workspace
+    assert "activeSelectionResult?.geojson ?? null" in map_workspace
+    assert "selectBathymetryProfilePartitions" in theme_hook
+    assert "datasets/bathymetry/profiles/partitions/select" in dataset_api
+    assert "regionalBathymetryContextActive" in app
+    assert "regionalBathymetryProfileCount" in app
+    assert "profielen · ${regionalBathymetryPartitions.length} gemeenten" in app
+    assert "FLANDERS_WORKSPACE_LABEL = 'Vlaanderen (285 gemeenten)'" in focus
+    assert "<title>GeoIntel</title>" in index