Add Flemish bathymetry partition workflow

2026-07-17 16:15:29 +02:00
parent 8ef42a23e5
commit 132c4feed8
34 changed files with 1862 additions and 27 deletions
@@ -0,0 +1,295 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+import json
+from pathlib import Path
+import ssl
+import sys
+from types import SimpleNamespace
+from urllib.error import URLError
+from uuid import uuid4
+
+from fastapi.testclient import TestClient
+import pytest
+
+from app.core.config import Settings
+from app.core.errors import AppError
+from app.db.session import get_db
+from app.main import app
+from app.models import Area, Dataset, DatasetVersion, Project
+from app.schemas.bathymetry import BathymetryPartitionFinalizeRequest
+from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
+from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService
+
+
+ROOT = Path(__file__).resolve().parents[2]
+SCRIPTS = ROOT / "scripts"
+if str(SCRIPTS) not in sys.path:
+    sys.path.insert(0, str(SCRIPTS))
+
+import provision_flanders_geographic_scope as flanders_scope  # noqa: E402
+
+
+class BinaryResponse:
+    def __init__(self, content: bytes, *, content_type: str = "application/xml"):
+        self.content = content
+        self.headers = {"Content-Type": content_type}
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_args):
+        return False
+
+    def read(self, size=-1):
+        return self.content if size < 0 else self.content[:size]
+
+
+class JsonResponse:
+    def __init__(self, payload, *, url="https://geo.api.vlaanderen.be/VRBG/items"):
+        self.payload = payload
+        self.url = url
+
+    def raise_for_status(self):
+        return None
+
+    def json(self):
+        return self.payload
+
+
+class SourceSession:
+    def __init__(self, payload):
+        self.payload = payload
+
+    def get(self, *_args, **_kwargs):
+        return JsonResponse(self.payload)
+
+
+class VersionQuery:
+    def __init__(self, versions):
+        self.versions = versions
+
+    def filter(self, *_args):
+        return self
+
+    def all(self):
+        return self.versions
+
+
+class FinalizeSession:
+    def __init__(self, rows, versions=None):
+        self.rows = rows
+        self.versions = versions or []
+        self.commit_count = 0
+
+    def get(self, model, row_id):
+        return self.rows.get((model, row_id))
+
+    def query(self, model):
+        assert model is DatasetVersion
+        return VersionQuery(self.versions)
+
+    def commit(self):
+        self.commit_count += 1
+
+
+def capabilities_xml() -> bytes:
+    return b"""<?xml version="1.0"?>
+    <WCS_Capabilities version="1.0.0" xmlns="http://www.opengis.net/wcs">
+      <ContentMetadata>
+        <CoverageOfferingBrief>
+          <name>EL.GridCoverage</name>
+          <label>Belgian Continental Shelf bathymetry</label>
+          <lonLatEnvelope srsName="urn:ogc:def:crs:OGC:1.3:CRS84" />
+        </CoverageOfferingBrief>
+      </ContentMetadata>
+      <Capability><Request><GetCoverage><resultFormat><formats>GeoTIFF</formats></resultFormat></GetCoverage></Request></Capability>
+    </WCS_Capabilities>"""
+
+
+def test_mdk_probe_parses_capabilities_without_enabling_acquisition() -> None:
+    seen = {}
+
+    def opener(request, timeout):
+        seen["url"] = request.full_url
+        seen["timeout"] = timeout
+        return BinaryResponse(capabilities_xml())
+
+    result = MdkBathymetryProbeService.probe(
+        settings=Settings(_env_file=None),
+        opener=opener,
+        checked_at=datetime(2026, 7, 17, tzinfo=UTC),
+    )
+
+    assert result["status"] == "reachable"
+    assert result["tls_verified"] is True
+    assert result["capabilities_reachable"] is True
+    assert result["acquisition_supported"] is False
+    assert result["coverage_identifiers"] == ["EL.GridCoverage"]
+    assert result["advertised_formats"] == ["GeoTIFF"]
+    assert result["response_sha256"]
+    assert "request=GetCapabilities" in seen["url"]
+    assert seen["timeout"] == 20
+
+
+def test_mdk_probe_reports_tls_failure_and_never_uses_insecure_fallback() -> None:
+    calls = 0
+
+    def opener(_request, timeout):
+        nonlocal calls
+        assert timeout == 20
+        calls += 1
+        raise URLError(ssl.SSLCertVerificationError("hostname mismatch"))
+
+    result = MdkBathymetryProbeService.probe(
+        settings=Settings(_env_file=None),
+        opener=opener,
+    )
+
+    assert calls == 1
+    assert result["status"] == "tls_error"
+    assert result["tls_verified"] is False
+    assert result["capabilities_reachable"] is False
+    assert "insecure fallback is prohibited" in result["message"]
+
+
+def test_mdk_readiness_api_uses_canonical_envelope(monkeypatch) -> None:
+    project_id = uuid4()
+    db = SimpleNamespace(get=lambda model, row_id: Project(id=project_id, name="Mol") if model is Project else None)
+    monkeypatch.setattr(
+        MdkBathymetryProbeService,
+        "probe",
+        lambda: {
+            "source_key": "mdk_bcp_bathymetry",
+            "status": "tls_error",
+            "acquisition_supported": False,
+        },
+    )
+    app.dependency_overrides[get_db] = lambda: db
+    try:
+        response = TestClient(app).get(
+            f"/api/v1/projects/{project_id}/datasets/bathymetry/sources/mdk_bcp_bathymetry/readiness"
+        )
+    finally:
+        app.dependency_overrides.clear()
+
+    assert response.status_code == 200
+    assert set(response.json()) == {"data"}
+    assert response.json()["data"]["status"] == "tls_error"
+    assert response.json()["data"]["acquisition_supported"] is False
+
+
+def test_partition_finalization_requires_complete_area_accounting_and_updates_versions() -> None:
+    project_id = uuid4()
+    area_ids = [uuid4(), uuid4()]
+    dataset_id = uuid4()
+    project = Project(id=project_id, name="Flanders")
+    areas = [
+        Area(id=area_ids[0], project_id=project_id, name="Gemeente Mol - officiële grens"),
+        Area(id=area_ids[1], project_id=project_id, name="Gemeente Geel - officiële grens"),
+    ]
+    dataset = Dataset(
+        id=dataset_id,
+        project_id=project_id,
+        area_id=area_ids[0],
+        name="vha.geojson",
+        dataset_type="vector",
+        source="VHA",
+        source_name=BathymetryProfileAcquisitionService.PROVIDER,
+        source_metadata={
+            "profile_count": 3,
+            "document_count": 2,
+            "structured_depth_count": 1,
+            "measurement_date_min": "1990-01-01",
+            "measurement_date_max": "2020-01-01",
+        },
+        provenance_metadata={},
+        status="ready",
+    )
+    version = DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1)
+    rows = {
+        (Project, project_id): project,
+        (Area, area_ids[0]): areas[0],
+        (Area, area_ids[1]): areas[1],
+        (Dataset, dataset_id): dataset,
+    }
+    db = FinalizeSession(rows, [version])
+    payload = BathymetryPartitionFinalizeRequest(
+        partition_scope_key="flanders",
+        expected_area_ids=area_ids,
+        dataset_ids=[dataset_id],
+        no_profile_area_ids=[area_ids[1]],
+        manifest_sha256="a" * 64,
+        observed_at=datetime(2026, 7, 17, tzinfo=UTC),
+    )
+
+    result = BathymetryProfileAcquisitionService.finalize_partitions(db, project_id, payload)
+
+    assert result["regional_partitions_complete"] is True
+    assert result["partition_count"] == 2
+    assert result["data_partition_count"] == 1
+    assert result["no_profile_partition_count"] == 1
+    assert result["profile_count"] == 3
+    assert dataset.source_metadata["coverage_scope"] == "flanders"
+    assert dataset.source_metadata["municipality"] == "Mol"
+    assert dataset.source_metadata["partitioned_source_audit"] is True
+    assert version.source_metadata == dataset.source_metadata
+    assert version.provenance_metadata == dataset.provenance_metadata
+    assert db.commit_count == 1
+
+    incomplete = payload.model_copy(update={"no_profile_area_ids": []})
+    with pytest.raises(AppError) as exc_info:
+        BathymetryProfileAcquisitionService.finalize_partitions(
+            FinalizeSession(rows, [version]),
+            project_id,
+            incomplete,
+        )
+    assert exc_info.value.code == "BATHYMETRY_PARTITION_MANIFEST_INCOMPLETE"
+
+
+def test_flanders_scope_discovery_uses_complete_unique_vrbg_inventory() -> None:
+    features = [
+        {
+            "type": "Feature",
+            "id": f"Refgem.{index:05d}",
+            "properties": {"NISCODE": f"{index:05d}", "NAAM": f"Gemeente {index:03d}"},
+            "geometry": {
+                "type": "Polygon",
+                "coordinates": [[[4.0, 50.5], [4.1, 50.5], [4.1, 50.6], [4.0, 50.5]]],
+            },
+        }
+        for index in range(1, 286)
+    ]
+    scope, selected, source_url = flanders_scope.discover_flanders_scope(
+        SourceSession({"features": list(reversed(features))}),
+        timeout=30,
+        min_municipalities=270,
+        max_municipalities=300,
+    )
+
+    assert scope.key == "flanders"
+    assert scope.project_name == "Flanders Regional Workbench"
+    assert len(scope.members) == 285
+    assert len(set(scope.nis_codes)) == 285
+    assert [item["properties"]["NISCODE"] for item in selected] == sorted(scope.nis_codes)
+    assert source_url.startswith("https://geo.api.vlaanderen.be/")
+
+
+def test_expansion_scripts_are_packaged_and_readiness_checked() -> None:
+    dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
+    readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
+    for script in (
+        "provision_flanders_geographic_scope.py",
+        "provision_flanders_bathymetry_profiles.py",
+        "probe_mdk_bathymetry.py",
+    ):
+        assert f"COPY scripts/{script}" in dockerfile
+        assert f"py_compile scripts/{script}" in readiness
+
+    sources = {
+        item["key"]: item
+        for item in BathymetryProfileAcquisitionService.list_sources()
+    }
+    assert sources["mdk_bcp_bathymetry"]["integration_status"] == "probe_only"
+    assert sources["mdk_bcp_bathymetry"]["acquisition_supported"] is False
+    assert "EL_wcs" in sources["mdk_bcp_bathymetry"]["service_url"]