Federate official Belgium data sources

backend/tests/test_docker_runtime_config.py

@@ -250,6 +250,33 @@ def test_runtime_sets_writable_ultralytics_config_directory() -> None:
     assert "YOLO_CONFIG_DIR=/app/storage/ultralytics" in unraid_env
 
 
+def test_regional_official_vector_sources_are_configurable_in_every_runtime() -> None:
+    compose = (ROOT / "docker-compose.yml").read_text(encoding="utf-8")
+    unraid_compose = (ROOT / "docker-compose.unraid.yml").read_text(encoding="utf-8")
+    run_script = (ROOT / "deploy" / "unraid" / "run-dockerman-container.sh").read_text(
+        encoding="utf-8"
+    )
+    env_example = (ROOT / "deploy" / "unraid" / "geointel.env.example").read_text(
+        encoding="utf-8"
+    )
+    template = (ROOT / "deploy" / "unraid" / "geointel-unraid-template.xml").read_text(
+        encoding="utf-8"
+    )
+
+    for key in (
+        "SPW_PICC_ENABLED",
+        "SPW_PICC_MAPSERVER_URL",
+        "URBIS_ENABLED",
+        "URBIS_WFS_URL",
+    ):
+        assert key in compose
+        assert key in unraid_compose
+        assert f'{key}="${{{key}:-' in run_script
+        assert f'-e {key}="${key}"' in run_script
+        assert f"{key}=" in env_example
+        assert f'Target="{key}"' in template
+
+
 def test_docker_build_contexts_exclude_vendor_build_and_cache_outputs() -> None:
     required_patterns = {
         "node_modules",

backend/tests/test_post_rc_regional_official_vector.py

@@ -0,0 +1,308 @@
+from __future__ import annotations
+
+import json
+from types import SimpleNamespace
+from urllib.parse import parse_qs, urlparse
+from uuid import uuid4
+
+import pytest
+from geoalchemy2.shape import from_shape
+from pyproj import Transformer
+from shapely.geometry import MultiPolygon, Polygon
+
+from app.core.config import Settings
+from app.core.errors import AppError
+from app.models import Area, Dataset, Project
+from app.schemas.official_vector import OfficialVectorAcquireRequest
+from app.services.dataset_service import DatasetService
+from app.services.official_vector_acquisition_service import (
+    OfficialVectorAcquisitionService,
+    _TO_LAMBERT72,
+)
+
+
+class FakeQuery:
+    def __init__(self, result=None):
+        self.result = result
+
+    def filter(self, *_args):
+        return self
+
+    def order_by(self, *_args):
+        return self
+
+    def all(self):
+        return self.result if isinstance(self.result, list) else []
+
+
+class FakeSession:
+    def __init__(self, rows=None, query_result=None):
+        self.rows = rows or {}
+        self.query_result = query_result
+
+    def get(self, model, row_id):
+        return self.rows.get((model, row_id))
+
+    def query(self, _model):
+        return FakeQuery(self.query_result)
+
+
+class JsonResponse:
+    def __init__(self, payload, content_type="application/geo+json"):
+        self.content = json.dumps(payload).encode("utf-8")
+        self.content_type = content_type
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *_args):
+        return False
+
+    def read(self, size=-1):
+        return self.content if size < 0 else self.content[:size]
+
+    def getheader(self, name):
+        return self.content_type if name.lower() == "content-type" else None
+
+
+def request(product_key: str, bbox: tuple[float, float, float, float], area_id=None):
+    return OfficialVectorAcquireRequest(
+        bbox={
+            "min_x": bbox[0],
+            "min_y": bbox[1],
+            "max_x": bbox[2],
+            "max_y": bbox[3],
+            "crs": "EPSG:4326",
+        },
+        area_id=area_id,
+        product_key=product_key,
+        force_refresh=True,
+    )
+
+
+def area(project_id, name: str, bounds: tuple[float, float, float, float]):
+    min_x, min_y, max_x, max_y = bounds
+    geometry = MultiPolygon(
+        [
+            Polygon(
+                [
+                    (min_x, min_y),
+                    (max_x, min_y),
+                    (max_x, max_y),
+                    (min_x, max_y),
+                    (min_x, min_y),
+                ]
+            )
+        ]
+    )
+    return Area(
+        id=uuid4(),
+        project_id=project_id,
+        name=name,
+        geometry=from_shape(geometry, srid=4326),
+    )
+
+
+def test_regional_product_registry_is_explicit_and_source_specific() -> None:
+    products = {
+        item["key"]: item
+        for item in OfficialVectorAcquisitionService.list_products()
+    }
+
+    assert products["spw_picc_buildings"]["coverage_zones"] == ["wallonia"]
+    assert products["spw_picc_roads"]["geometry_types"] == [
+        "LineString",
+        "MultiLineString",
+    ]
+    assert products["spw_picc_waterways"]["collection"] == "28"
+    assert products["spw_picc_water_surfaces"]["collection"] == "30"
+    assert products["urbis_buildings"]["coverage_zones"] == ["brussels"]
+    assert products["urbis_buildings"]["license_note"] == "Buildings are published under CC0."
+    assert "FPS Finance" in products["urbis_cadastral_parcels"]["license_note"]
+
+
+def test_spw_arcgis_paging_is_bounded_stable_and_clipped() -> None:
+    product = OfficialVectorAcquisitionService._product("spw_picc_buildings")
+    scope = Polygon(
+        [(4.55, 50.58), (4.56, 50.58), (4.56, 50.59), (4.55, 50.59), (4.55, 50.58)]
+    )
+    scope_metric = Polygon(
+        [
+            _TO_LAMBERT72.transform(x, y)
+            for x, y in scope.exterior.coords
+        ]
+    )
+    offsets = []
+
+    def feature(object_id: int, min_x: float):
+        return {
+            "type": "Feature",
+            "id": object_id,
+            "geometry": {
+                "type": "Polygon",
+                "coordinates": [[
+                    [min_x, 50.581],
+                    [min_x + 0.002, 50.581],
+                    [min_x + 0.002, 50.583],
+                    [min_x, 50.583],
+                    [min_x, 50.581],
+                ]],
+            },
+            "properties": {"OBJECTID": object_id, "GEOREF_ID": f"wallonia-{object_id}"},
+        }
+
+    def opener(raw_request, timeout):
+        assert timeout == 180
+        query = parse_qs(urlparse(raw_request.full_url).query)
+        assert query["orderByFields"] == ["OBJECTID"]
+        assert query["f"] == ["geojson"]
+        offset = int(query["resultOffset"][0])
+        offsets.append(offset)
+        return JsonResponse(
+            {
+                "type": "FeatureCollection",
+                "features": [feature(offset + 1, 4.551 + offset * 0.0001)],
+                "exceededTransferLimit": offset == 0,
+            }
+        )
+
+    features, transfer = OfficialVectorAcquisitionService._fetch_features(
+        product,
+        scope,
+        scope_metric,
+        "wallonia",
+        Settings(_env_file=None, OFFICIAL_VECTOR_PAGE_SIZE=1),
+        opener,
+    )
+
+    assert offsets == [0, 1]
+    assert transfer["page_count"] == 2
+    assert transfer["reference_truncated"] is False
+    assert {item["properties"]["source_feature_id"] for item in features} == {
+        "11:wallonia-1",
+        "11:wallonia-2",
+    }
+    assert all(item["properties"]["coverage_scope"] == "wallonia" for item in features)
+    assert all(item["properties"]["clipped_area_ha"] > 0 for item in features)
+
+
+def test_regional_products_require_the_persisted_authoritative_coverage_area() -> None:
+    project_id = uuid4()
+    db = FakeSession({(Project, project_id): Project(id=project_id, name="Belgium")})
+
+    with pytest.raises(AppError) as exc_info:
+        OfficialVectorAcquisitionService.acquire(
+            db,
+            project_id,
+            request("spw_picc_buildings", (4.55, 50.58, 4.56, 50.59)),
+            settings=Settings(_env_file=None),
+        )
+
+    assert exc_info.value.code == "OFFICIAL_VECTOR_COVERAGE_NOT_READY"
+
+
+def test_urbis_wfs_transforms_lambert72_and_persists_through_dataset_service(
+    monkeypatch,
+) -> None:
+    project_id, dataset_id = uuid4(), uuid4()
+    brussels = area(project_id, "Brussels-Capital Region", (4.25, 50.75, 4.5, 50.95))
+    db = FakeSession(
+        {
+            (Project, project_id): Project(id=project_id, name="Belgium"),
+            (Area, brussels.id): brussels,
+        },
+        query_result=[brussels],
+    )
+    to_lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
+    min_x, min_y = to_lambert.transform(4.35, 50.84)
+    max_x, max_y = to_lambert.transform(4.351, 50.841)
+    captured = {}
+
+    def opener(raw_request, timeout):
+        assert timeout == 180
+        query = parse_qs(urlparse(raw_request.full_url).query)
+        assert query["typeNames"] == ["urbisvector:Buildings"]
+        assert query["srsName"] == ["EPSG:31370"]
+        assert query["sortBy"] == ["INSPIRE_ID"]
+        return JsonResponse(
+            {
+                "type": "FeatureCollection",
+                "numberMatched": 1,
+                "numberReturned": 1,
+                "features": [
+                    {
+                        "type": "Feature",
+                        "id": "Buildings.1",
+                        "geometry": {
+                            "type": "MultiPolygon",
+                            "coordinates": [[[
+                                [min_x, min_y],
+                                [max_x, min_y],
+                                [max_x, max_y],
+                                [min_x, max_y],
+                                [min_x, min_y],
+                            ]]],
+                        },
+                        "properties": {
+                            "INSPIRE_ID": "https://databrussels.be/id/building/1",
+                            "AREA": 75,
+                        },
+                    }
+                ],
+            },
+            "application/json",
+        )
+
+    def persist(_db, **kwargs):
+        captured.update(kwargs)
+        dataset = Dataset(
+            id=dataset_id,
+            project_id=project_id,
+            area_id=brussels.id,
+            name=kwargs["filename"],
+            dataset_type="vector",
+            source=kwargs["source"],
+            dataset_role=kwargs["dataset_role"],
+            source_name=kwargs["source_name"],
+            reference_layer_name=kwargs["reference_layer_name"],
+            temporal_series_key=kwargs["temporal_series_key"],
+            observed_at=kwargs["observed_at"],
+            source_version=kwargs["source_version"],
+            source_metadata=kwargs["source_metadata"],
+            provenance_metadata=kwargs["provenance_metadata"],
+            metadata_json={"feature_count": 1},
+            status="ready",
+        )
+        db.rows[(Dataset, dataset_id)] = dataset
+        return SimpleNamespace(id=dataset_id)
+
+    monkeypatch.setattr(DatasetService, "import_vector_bytes", persist)
+    result = OfficialVectorAcquisitionService.acquire(
+        db,
+        project_id,
+        request(
+            "urbis_buildings",
+            (4.349, 50.839, 4.352, 50.842),
+            area_id=brussels.id,
+        ),
+        settings=Settings(_env_file=None),
+        opener=opener,
+    )
+
+    assert result["output_dataset_id"] == str(dataset_id)
+    assert captured["source_name"] == "urbis"
+    assert captured["reference_layer_name"] == "buildings"
+    assert captured["source_metadata"]["coverage_zones"] == ["brussels"]
+    assert captured["source_metadata"]["selection_aggregation"]["metric_key"] == (
+        "building_footprint_area"
+    )
+    collection = json.loads(captured["content"])
+    geometry = collection["features"][0]["geometry"]
+    assert geometry["type"] in {"Polygon", "MultiPolygon"}
+    first_coordinate = (
+        geometry["coordinates"][0][0][0]
+        if geometry["type"] == "MultiPolygon"
+        else geometry["coordinates"][0][0]
+    )
+    assert 4.34999 <= first_coordinate[0] <= 4.35101
+    assert 50.83999 <= first_coordinate[1] <= 50.84101

backend/tests/test_rc4_national_coverage.py

@@ -12,6 +12,7 @@ from app.main import app
 from app.models import Area, Dataset, Project
 from app.schemas.coverage import CoverageBBox
 from app.services.coverage_registry_service import CoverageRegistryService, THEMES, ZONES
+from app.services.vector_feature_service import VectorFeatureService
 
 
 class FakeQuery:
@@ -72,6 +73,44 @@ def test_coverage_catalog_uses_normalized_contracts_and_does_not_change_provider
     assert response.json()["data"]["themes"] == list(THEMES)
 
 
+def test_national_and_maritime_reference_layers_are_selection_analyzable() -> None:
+    cases = (
+        ("ngi_adminvector", "belgium_municipalities", "administrative"),
+        ("rbins_marine_reporting_units", "marine_legal_scopes", "marine_environment"),
+        ("rbins_msp_2026", "marine_spatial_plan_2026", "maritime_planning"),
+    )
+    for source_name, layer_name, expected_theme in cases:
+        dataset = Dataset(
+            id=uuid4(),
+            project_id=uuid4(),
+            name=f"{layer_name}.geojson",
+            dataset_type="vector",
+            source="operator_official_import",
+            source_name=source_name,
+            reference_layer_name=layer_name,
+            source_metadata={"authority_level": "authoritative"},
+            status="ready",
+        )
+
+        assert VectorFeatureService._dataset_theme(dataset) == expected_theme
+        assert VectorFeatureService.supports_selection_summary(dataset) is True
+
+
+def test_national_scope_operator_assigns_explicit_map_themes() -> None:
+    root = Path(__file__).resolve().parents[2]
+    operator = (root / "scripts" / "provision_belgium_north_sea_scope.py").read_text(encoding="utf-8")
+    map_workspace = (root / "frontend" / "src" / "components" / "map" / "MapWorkspace.tsx").read_text(
+        encoding="utf-8"
+    )
+
+    assert '"belgium_municipalities": "administrative"' in operator
+    assert '"marine_legal_scopes": "marine_environment"' in operator
+    assert '"marine_spatial_plan_2026": "maritime_planning"' in operator
+    assert "id: 'administrative'" in map_workspace
+    assert "id: 'maritime_planning'" in map_workspace
+    assert "id: 'marine_environment'" in map_workspace
+
+
 def test_coverage_resolver_only_reports_operational_for_materialized_ready_dataset() -> None:
     project_id = uuid4()
     project = SimpleNamespace(id=project_id)

backend/tests/test_sprint194_regional_timeseries.py

@@ -55,16 +55,22 @@ def test_population_operator_filters_to_the_approved_scope() -> None:
     module = load_script("provision_mol_population_history.py")
     regional = module.GEOGRAPHIC_SCOPES["kempen-transport-region"]
     mol = module.GEOGRAPHIC_SCOPES["mol"]
+    belgium = module.GEOGRAPHIC_SCOPES["belgium"]
 
     regional_rows = module.population_rows(population_archive(), regional)
     mol_rows = module.population_rows(population_archive(), mol)
+    national_rows = module.population_rows(population_archive(), belgium)
 
     assert set(regional_rows) == {"13025A00-", "13008A00-"}
     assert regional_rows["13008A00-"]["municipality"] == "Geel"
     assert regional_rows["13008A00-"]["nis_code"] == "13008"
     assert set(mol_rows) == {"13025A00-"}
+    assert set(national_rows) == {"13025A00-", "13008A00-", "11002A00-"}
+    assert national_rows["11002A00-"]["municipality"] == "Antwerpen"
+    assert belgium.all_municipalities is True
     assert module.series_key(regional) == "statbel:population-statistical-sector:kempen-transport-region"
     assert module.series_key(mol) == "statbel:population-statistical-sector:mol"
+    assert module.series_key(belgium) == "statbel:population-statistical-sector:belgium"
 
 
 def test_population_operator_resolves_the_persisted_scope_boundary(tmp_path: Path) -> None:
@@ -90,6 +96,48 @@ def test_population_operator_resolves_the_persisted_scope_boundary(tmp_path: Pat
     assert module.resolve_boundary_path(args, scope) == boundary
 
 
+def test_population_operator_resolves_checksum_verified_belgium_boundary(tmp_path: Path) -> None:
+    module = load_script("provision_mol_population_history.py")
+    scope = module.GEOGRAPHIC_SCOPES["belgium"]
+    scope_dir = tmp_path / "belgium-north-sea"
+    scope_dir.mkdir(parents=True)
+    boundary = scope_dir / "belgium_land_boundary.geojson"
+    boundary.write_text(
+        json.dumps(
+            {
+                "type": "FeatureCollection",
+                "features": [
+                    {
+                        "type": "Feature",
+                        "geometry": {
+                            "type": "Polygon",
+                            "coordinates": [[[2.5, 49.5], [6.4, 49.5], [6.4, 51.5], [2.5, 51.5], [2.5, 49.5]]],
+                        },
+                        "properties": {},
+                    }
+                ],
+            }
+        ),
+        encoding="utf-8",
+    )
+    (scope_dir / "manifest.json").write_text(
+        json.dumps(
+            {
+                "scope": "belgium-and-belgian-north-sea",
+                "artifacts": {
+                    "belgium_land_boundary": {
+                        "sha256": module.sha256_path(boundary),
+                    }
+                },
+            }
+        ),
+        encoding="utf-8",
+    )
+    args = argparse.Namespace(boundary_path=None, scope_output_root=tmp_path)
+
+    assert module.resolve_boundary_path(args, scope) == boundary
+
+
 def test_regional_coordinator_builds_explicit_population_and_landuse_commands(tmp_path: Path) -> None:
     module = load_script("provision_regional_timeseries.py")
     scope = module.GEOGRAPHIC_SCOPES["kempen-transport-region"]

backend/tests/test_sprint227_statbel_population_preflight.py

@@ -205,6 +205,42 @@ def test_preflight_reconciles_spatial_and_unlocated_population(tmp_path: Path) -
     assert len(manifest["schemas"]["geometry_schema_sha256"]) == 64
 
 
+def test_preflight_supports_the_complete_national_scope() -> None:
+    result = PREFLIGHT.validate_statbel_release(
+        year=2025,
+        layout="new",
+        population_content=population_archive(),
+        population_url=POPULATION_URL,
+        geometry_content=geometry_archive(),
+        geometry_url=GEOMETRY_URL,
+        scope=PREFLIGHT.GEOGRAPHIC_SCOPES["belgium"],
+    )
+
+    accounting = result.manifest["scope_accounting"]
+    assert accounting["scope_key"] == "belgium"
+    assert accounting["member_count"] == 2
+    assert accounting["member_nis_codes"] == ["13008", "13025"]
+    assert accounting["spatial_population_total"] == 240
+    assert accounting["unlocated_population_total"] == 3
+    assert accounting["accounted_population_total"] == 243
+
+
+def test_national_preflight_rejects_an_unscoped_baseline(tmp_path: Path) -> None:
+    with pytest.raises(PREFLIGHT.StatbelPreflightError) as exc_info:
+        PREFLIGHT.validate_statbel_release(
+            year=2025,
+            layout="new",
+            population_content=population_archive(),
+            population_url=POPULATION_URL,
+            geometry_content=geometry_archive(),
+            geometry_url=GEOMETRY_URL,
+            scope=PREFLIGHT.GEOGRAPHIC_SCOPES["belgium"],
+            baseline_snapshot=baseline_snapshot(tmp_path / "baseline.geojson"),
+        )
+
+    assert exc_info.value.code == "STATBEL_BASELINE_SCOPE_MISMATCH"
+
+
 def test_preflight_reports_bounded_topology_repairs(tmp_path: Path) -> None:
     result = validate(tmp_path, geometry_content=geometry_archive(repairable_invalid=True))
 

backend/tests/test_sprint240_official_flemish_themes.py

@@ -125,7 +125,16 @@ def test_product_registries_expose_honest_forest_agriculture_nature_and_soil() -
     assert raster["agricultural_land_use_2025"]["included_source_values"] == [13, 14]
     assert "geen juridische bosgrens" in raster["forest_land_use_2025"]["limitation_message"].lower()
     assert "geen alz-perceelaangifte" in raster["agricultural_land_use_2025"]["limitation_message"].lower()
-    assert set(vector) == {"bwk_natura2000_2025", "dov_soil_types"}
+    assert {
+        "bwk_natura2000_2025",
+        "dov_soil_types",
+        "spw_picc_buildings",
+        "spw_picc_roads",
+        "spw_picc_waterways",
+        "spw_picc_water_surfaces",
+        "urbis_buildings",
+        "urbis_cadastral_parcels",
+    } == set(vector)
     assert vector["bwk_natura2000_2025"]["authority_level"] == "authoritative"
     assert vector["dov_soil_types"]["authority_level"] == "authoritative_historical_baseline"
     assert "1949-1971" in vector["dov_soil_types"]["observation_label"]
@@ -392,7 +401,7 @@ def test_official_vector_routes_and_frontend_use_canonical_backend_path(monkeypa
 
     assert products_response.status_code == 200
     assert set(products_response.json()) == {"data"}
-    assert products_response.json()["data"]["total"] == 2
+    assert products_response.json()["data"]["total"] == 8
     assert acquire_response.status_code == 200
     assert set(acquire_response.json()) == {"data"}
     assert acquire_response.json()["data"]["job_type"] == "vector.official.acquire"