From 1e527bf810596f57a66b3473a9717332ab59ec16 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 18 Jul 2026 01:45:00 +0200 Subject: [PATCH] Add Belgium and North Sea coverage foundation --- CHANGELOG.md | 13 + backend/README.md | 24 + backend/app/api/routes/external.py | 19 +- backend/app/schemas/__init__.py | 14 + backend/app/schemas/coverage.py | 78 ++ .../app/services/coverage_registry_service.py | 513 ++++++++++ backend/tests/test_rc4_national_coverage.py | 212 ++++ .../tests/test_rc4_national_scope_operator.py | 161 +++ .../tests/test_sprint31_unraid_template.py | 5 +- deploy/unraid/Dockerfile.all-in-one | 1 + docs/API_CONTRACTS.md | 43 + docs/CODEX_EXECUTION_LOG.md | 26 + docs/DATA_SOURCES.md | 36 + docs/DATA_SPECIFICATION.md | 25 + docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md | 5 + docs/STORAGE_ARCHITECTURE.md | 20 + docs/TODO.md | 6 + frontend/README.md | 13 + frontend/src/App.tsx | 26 +- frontend/src/components/map/MapWorkspace.tsx | 104 +- frontend/src/config/primaryFocus.ts | 2 + frontend/src/hooks/useCoverageResolver.ts | 63 ++ frontend/src/hooks/useProjectWorkspace.ts | 18 +- frontend/src/services/api/external.ts | 14 + frontend/src/styles/app.css | 64 ++ frontend/src/types.ts | 51 + scripts/README.md | 27 + scripts/provision_belgium_north_sea_scope.py | 950 ++++++++++++++++++ scripts/run_readiness_check.sh | 1 + 29 files changed, 2524 insertions(+), 10 deletions(-) create mode 100644 backend/app/schemas/coverage.py create mode 100644 backend/app/services/coverage_registry_service.py create mode 100644 backend/tests/test_rc4_national_coverage.py create mode 100644 backend/tests/test_rc4_national_scope_operator.py create mode 100644 frontend/src/hooks/useCoverageResolver.ts create mode 100644 scripts/provision_belgium_north_sea_scope.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 816a0194..0b966e31 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,19 @@ - Completed live Detection Lab acceptance with an intentionally empty raster choice, disabled start action and zero browser console errors. No stale running Job or AnalysisRun remained after reconciliation. +- Added a separate national coverage registry with normalized themes, legal + land/sea zones and only four honest states: `operational`, `partial`, + `not_configured` and `unsupported`. +- Added canonical coverage catalog/resolve endpoints. Drawn bboxes are + resolved against persisted Areas, cross-zone output stays split and + `operational` requires a matching ready Dataset. +- Added the explicit Belgium/North Sea operator with fixed official NGI/RBINS + allowlists, strict TLS, size limits, safe archive extraction, complete WFS + pagination, checksums and API-only persistence. +- Added national-workspace preference and a compact selection coverage surface + to the Map UI without removing Mol/Kempen regression workspaces. +- Local RC-4 validation passed 986 backend tests, frontend typecheck/build, + the full readiness gate, one Alembic head and complete offline migration SQL. ## Sprint 240 Operational forest, agriculture, nature and soil themes (2026-07-17) diff --git a/backend/README.md b/backend/README.md index 13ee511e..aba34b14 100644 --- a/backend/README.md +++ b/backend/README.md @@ -987,6 +987,30 @@ it does not create a derived dataset. ## Geographic scope provisioning +The release-candidate national foundation is provisioned explicitly: + +```bash +docker exec geointel python /app/scripts/provision_belgium_north_sea_scope.py +``` + +Use `--fetch-only` to validate official NGI AdminVector, RBINS marine +reporting units and the Belgian Marine Spatial Plan 2026-2034 without changing +application persistence. The normal command creates or reuses +`Belgium and North Sea Workbench`, persists Belgium, all three regions, +territorial sea, EEZ and continental shelf as Areas, and uploads six +checksum-bound reference Datasets through `DatasetService`. + +The operator has a fixed URL/layer allowlist, verified TLS, archive and +response-size limits, safe ZIP extraction, complete WFS pagination and +immutable artifact checksums. It does not run at startup and does not write +directly to `vector_features`. + +`GET /api/v1/external/coverage/catalog` exposes audited national source +contracts. `POST /api/v1/external/coverage/resolve` intersects a drawn bbox +with persisted legal/administrative Areas and reports a split zone/theme +matrix. `operational` requires a matching `ready` Dataset; integration without +materialized data is only `partial`. + The explicit operator command below provisions the official 28-municipality Vlaamse vervoerregio Kempen boundary foundation: diff --git a/backend/app/api/routes/external.py b/backend/app/api/routes/external.py index 8f9d9ff6..959230ba 100644 --- a/backend/app/api/routes/external.py +++ b/backend/app/api/routes/external.py @@ -7,7 +7,8 @@ from app.core.errors import AppError from app.db.session import get_db from app.models import Area, Project from app.providers.registry import fetch_provider_data, get_provider, import_provider_dataset, list_provider_capabilities -from app.schemas import ExternalFetchRequest, ExternalFetchResponse, ProviderImportRequest +from app.schemas import CoverageResolveRequest, ExternalFetchRequest, ExternalFetchResponse, ProviderImportRequest +from app.services.coverage_registry_service import CoverageRegistryService from app.utils.response import envelope router = APIRouter(prefix="/external", tags=["external"]) @@ -46,6 +47,22 @@ def list_external_providers() -> dict: }) +@router.get("/coverage/catalog") +def get_coverage_catalog() -> dict: + return envelope(CoverageRegistryService.catalog().model_dump()) + + +@router.post("/coverage/resolve") +def resolve_project_coverage(payload: CoverageResolveRequest, db: Session = Depends(get_db)) -> dict: + result = CoverageRegistryService.resolve( + db, + project_id=payload.project_id, + bbox=payload.bbox, + themes=payload.themes, + ) + return envelope(result.model_dump()) + + @router.get("/providers/capabilities") def get_external_provider_capabilities() -> dict: return envelope({ diff --git a/backend/app/schemas/__init__.py b/backend/app/schemas/__init__.py index 68ba5a20..8c81d7a2 100644 --- a/backend/app/schemas/__init__.py +++ b/backend/app/schemas/__init__.py @@ -1,6 +1,14 @@ from __future__ import annotations from .common import ApiErrorEnvelope, ApiErrorItem, Envelope, PaginationEnvelope +from .coverage import ( + CoverageBBox, + CoverageCatalogResponse, + CoverageResolutionItem, + CoverageResolveRequest, + CoverageResolveResponse, + CoverageSourceContract, +) from .project import ProjectCreate, ProjectList, ProjectRead, ProjectUpdate from .area import AreaCreate, AreaList, AreaRead, AreaUpdate from .analysis import ChangeDetectionRequest, ChangeDetectionSummary @@ -145,6 +153,12 @@ __all__ = [ "ApiErrorEnvelope", "ApiErrorItem", "PaginationEnvelope", + "CoverageBBox", + "CoverageCatalogResponse", + "CoverageResolutionItem", + "CoverageResolveRequest", + "CoverageResolveResponse", + "CoverageSourceContract", "ProjectCreate", "ProjectRead", "ProjectUpdate", diff --git a/backend/app/schemas/coverage.py b/backend/app/schemas/coverage.py new file mode 100644 index 00000000..9091f740 --- /dev/null +++ b/backend/app/schemas/coverage.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from typing import Literal +from uuid import UUID + +from pydantic import BaseModel, Field, model_validator + + +CoverageStatus = Literal["operational", "partial", "not_configured", "unsupported"] +CoverageAuthority = Literal["authoritative", "official_context", "contextual"] +CoverageAcquisitionMode = Literal[ + "operator_archive", + "operator_wfs", + "bounded_api", + "bounded_raster", + "catalog_only", +] + + +class CoverageBBox(BaseModel): + minx: float = Field(ge=-180, le=180) + miny: float = Field(ge=-90, le=90) + maxx: float = Field(ge=-180, le=180) + maxy: float = Field(ge=-90, le=90) + + @model_validator(mode="after") + def validate_extent(self) -> "CoverageBBox": + if self.maxx <= self.minx or self.maxy <= self.miny: + raise ValueError("bbox max values must be greater than min values") + return self + + +class CoverageSourceContract(BaseModel): + source_name: str + display_name: str + authority_level: CoverageAuthority + coverage_zones: list[str] + themes: list[str] + native_layers: list[str] + supported_geometry_types: list[str] + acquisition_mode: CoverageAcquisitionMode + integration_status: CoverageStatus + source_url: str + attribution: str + license_note: str + limitation_message: str + + +class CoverageCatalogResponse(BaseModel): + themes: list[str] + zones: list[str] + statuses: list[CoverageStatus] + sources: list[CoverageSourceContract] + + +class CoverageResolveRequest(BaseModel): + project_id: UUID + bbox: CoverageBBox + themes: list[str] = Field(default_factory=list, max_length=32) + + +class CoverageResolutionItem(BaseModel): + zone: str + theme: str + status: CoverageStatus + source_names: list[str] + materialized_dataset_ids: list[UUID] + limitation_message: str + + +class CoverageResolveResponse(BaseModel): + project_id: UUID + bbox: CoverageBBox + requested_themes: list[str] + intersected_zones: list[str] + outside_supported_scope: bool + items: list[CoverageResolutionItem] + warnings: list[str] diff --git a/backend/app/services/coverage_registry_service.py b/backend/app/services/coverage_registry_service.py new file mode 100644 index 00000000..76753aa0 --- /dev/null +++ b/backend/app/services/coverage_registry_service.py @@ -0,0 +1,513 @@ +from __future__ import annotations + +from collections.abc import Iterable +from dataclasses import dataclass +from typing import Any +from uuid import UUID + +from geoalchemy2.shape import to_shape +from shapely.geometry import box +from shapely.ops import unary_union +from sqlalchemy.orm import Session + +from app.core.errors import AppError +from app.models import Area, Dataset, Project +from app.schemas.coverage import ( + CoverageBBox, + CoverageCatalogResponse, + CoverageResolutionItem, + CoverageResolveResponse, + CoverageSourceContract, +) + + +THEMES = ( + "admin", + "buildings", + "roads", + "surface_water", + "land_cover_use", + "nature", + "population", + "parcels", + "soil", + "elevation", + "orthophoto", + "flood_climate", + "maritime_planning", + "marine_environment", + "bathymetry", +) + +ZONES = ( + "belgium", + "flanders", + "wallonia", + "brussels", + "belgian_north_sea", + "territorial_sea", + "exclusive_economic_zone", + "continental_shelf", +) + +STATUS_ORDER = ("unsupported", "not_configured", "partial", "operational") +STATUS_RANK = {status: index for index, status in enumerate(STATUS_ORDER)} + +SCOPE_AREA_NAMES = { + "belgium": "Belgium land", + "flanders": "Flanders", + "wallonia": "Wallonia", + "brussels": "Brussels-Capital Region", + "belgian_north_sea": "Belgian part of the North Sea", + "territorial_sea": "Belgian territorial sea (0-12 nautical miles)", + "exclusive_economic_zone": "Belgian exclusive economic zone beyond territorial sea", + "continental_shelf": "Belgian continental shelf beyond territorial sea", +} + +DETAIL_ZONES = ( + "flanders", + "wallonia", + "brussels", + "territorial_sea", + "exclusive_economic_zone", + "continental_shelf", +) + + +@dataclass(frozen=True) +class _SourceDefinition: + contract: CoverageSourceContract + materialized_layer_names: tuple[str, ...] = () + materialized_source_names: tuple[str, ...] = () + + +def _contract( + *, + source_name: str, + display_name: str, + authority_level: str, + coverage_zones: tuple[str, ...], + themes: tuple[str, ...], + native_layers: tuple[str, ...], + geometry_types: tuple[str, ...], + acquisition_mode: str, + integration_status: str, + source_url: str, + attribution: str, + license_note: str, + limitation_message: str, + materialized_layer_names: tuple[str, ...] = (), + materialized_source_names: tuple[str, ...] = (), +) -> _SourceDefinition: + return _SourceDefinition( + contract=CoverageSourceContract( + source_name=source_name, + display_name=display_name, + authority_level=authority_level, + coverage_zones=list(coverage_zones), + themes=list(themes), + native_layers=list(native_layers), + supported_geometry_types=list(geometry_types), + acquisition_mode=acquisition_mode, + integration_status=integration_status, + source_url=source_url, + attribution=attribution, + license_note=license_note, + limitation_message=limitation_message, + ), + materialized_layer_names=materialized_layer_names, + materialized_source_names=materialized_source_names or (source_name,), + ) + + +SOURCE_DEFINITIONS = ( + _contract( + source_name="ngi_adminvector", + display_name="NGI AdminVector", + authority_level="authoritative", + coverage_zones=("belgium", "flanders", "wallonia", "brussels", "belgian_north_sea"), + themes=("admin",), + native_layers=( + "belgianterritory", + "belgianmaritimezone", + "region", + "province", + "municipality", + ), + geometry_types=("Polygon", "MultiPolygon"), + acquisition_mode="operator_archive", + integration_status="operational", + source_url="https://www.geo.be/catalog/details/fb1e2993-2020-428c-9188-eb5f75e284b9", + attribution="National Geographic Institute (NGI), AdminVector", + license_note="CC BY 4.0", + limitation_message="Administrative reference geometry; it does not provide thematic land content.", + materialized_layer_names=( + "belgium_land_boundary", + "belgium_regions", + "belgium_provinces", + "belgium_municipalities", + ), + ), + _contract( + source_name="statbel", + display_name="Statbel statistical sectors and population", + authority_level="authoritative", + coverage_zones=("belgium", "flanders", "wallonia", "brussels"), + themes=("admin", "population"), + native_layers=("statistical_sectors", "population_statistics"), + geometry_types=("Polygon", "MultiPolygon", "Tabular"), + acquisition_mode="catalog_only", + integration_status="not_configured", + source_url="https://statbel.fgov.be/en/open-data", + attribution="Statbel", + license_note="Consult the license of the selected Statbel release.", + limitation_message="The catalog is audited, but no national bounded acquisition adapter is configured yet.", + ), + _contract( + source_name="digitaal_vlaanderen", + display_name="Flemish authoritative services", + authority_level="authoritative", + coverage_zones=("flanders",), + themes=( + "buildings", + "roads", + "surface_water", + "land_cover_use", + "nature", + "parcels", + "soil", + "elevation", + "orthophoto", + "flood_climate", + ), + native_layers=("GRB", "BWK", "DHMV", "OMWRGBMRVL", "OGRK", "Mercator"), + geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"), + acquisition_mode="bounded_api", + integration_status="operational", + source_url="https://www.vlaanderen.be/datavindplaats", + attribution="Digitaal Vlaanderen and the authoritative Flemish source owners", + license_note="Consult the license and attribution stored with each acquired dataset.", + limitation_message="Operational only for bounded products implemented by GeoIntel and materialized in the project.", + materialized_source_names=( + "grb", + "digitaal_vlaanderen_buildings_addresses_register", + "digitaal_vlaanderen_dhmv", + "digitaal_vlaanderen_orthophoto", + "vmm_flood_hazard", + "department_omgeving_thematic_raster", + "inbo_bwk_natura2000", + "dov_soil_map", + "agentschap_landbouw_zeevisserij_agricultural_parcels", + ), + ), + _contract( + source_name="spw_geoportail", + display_name="SPW Geoportail Wallonie", + authority_level="authoritative", + coverage_zones=("wallonia",), + themes=( + "buildings", + "roads", + "surface_water", + "land_cover_use", + "nature", + "soil", + "elevation", + "orthophoto", + "flood_climate", + "bathymetry", + ), + native_layers=("PICC", "orthophotos", "MNT", "hydrography", "land_cover"), + geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"), + acquisition_mode="catalog_only", + integration_status="not_configured", + source_url="https://geoportail.wallonie.be/catalogue", + attribution="Service public de Wallonie", + license_note="Consult the license of each Geoportail Wallonie product.", + limitation_message="Official sources are identified, but bounded acquisition and metric adapters are not implemented.", + ), + _contract( + source_name="urbis", + display_name="UrbIS Brussels", + authority_level="authoritative", + coverage_zones=("brussels",), + themes=("buildings", "roads", "surface_water", "land_cover_use", "parcels", "orthophoto"), + native_layers=("parcels", "buildings", "roads", "hydrography", "orthophoto"), + geometry_types=("Point", "LineString", "Polygon", "MultiPolygon", "Raster"), + acquisition_mode="catalog_only", + integration_status="not_configured", + source_url="https://datastore.brussels", + attribution="Brussels UrbIS", + license_note="Consult the license of the selected UrbIS dataset.", + limitation_message="Official sources are identified, but bounded acquisition and metric adapters are not implemented.", + ), + _contract( + source_name="rbins_marine_reporting_units", + display_name="RBINS marine reporting units", + authority_level="authoritative", + coverage_zones=( + "belgian_north_sea", + "territorial_sea", + "exclusive_economic_zone", + "continental_shelf", + ), + themes=("admin", "marine_environment"), + native_layers=("marine_reporting_units_2024",), + geometry_types=("Polygon", "MultiPolygon"), + acquisition_mode="operator_wfs", + integration_status="operational", + source_url=( + "https://metadata.naturalsciences.be/geonetwork/srv/api/records/" + "29f40b0d-2a3e-49a8-870a-e9b4acd4d1e3" + ), + attribution="Royal Belgian Institute of Natural Sciences (RBINS), BMDC", + license_note="Reuse conditions are retained from the source metadata with every persisted artifact.", + limitation_message="The EEZ and continental shelf can share geometry while retaining different legal semantics.", + materialized_layer_names=("marine_legal_scopes",), + ), + _contract( + source_name="rbins_msp_2026", + display_name="Belgian Marine Spatial Plan 2026-2034", + authority_level="authoritative", + coverage_zones=( + "belgian_north_sea", + "territorial_sea", + "exclusive_economic_zone", + "continental_shelf", + ), + themes=("maritime_planning", "marine_environment"), + native_layers=("imsp26",), + geometry_types=("Point", "LineString", "Polygon", "MultiPolygon"), + acquisition_mode="operator_wfs", + integration_status="operational", + source_url="https://www.health.belgium.be/en/themes/environment/marine-environment/marine-spatial-plan", + attribution="Belgian federal Marine Environment service and RBINS", + license_note="Official source metadata and attribution are retained with the imported snapshot.", + limitation_message="The dataset represents the legally current 2026-2034 plan, not live maritime activity.", + materialized_layer_names=("marine_spatial_plan_2026",), + ), + _contract( + source_name="mdk_bathymetry", + display_name="MDK Belgian North Sea depth model", + authority_level="authoritative", + coverage_zones=( + "belgian_north_sea", + "territorial_sea", + "exclusive_economic_zone", + "continental_shelf", + ), + themes=("bathymetry",), + native_layers=("depth_model_20m_lat",), + geometry_types=("Raster",), + acquisition_mode="catalog_only", + integration_status="not_configured", + source_url="https://www.vlaanderen.be/datavindplaats", + attribution="Agentschap Maritieme Dienstverlening en Kust (MDK)", + license_note="Consult the official product license before acquisition.", + limitation_message="Strict-TLS acquisition and vertical datum evidence are not yet sufficient; no depths are synthesized.", + ), +) + +FLANDERS_THEME_DATASETS: dict[str, dict[str, tuple[str, ...]]] = { + "buildings": { + "grb": ("buildings",), + "digitaal_vlaanderen_buildings_addresses_register": (), + }, + "roads": {"grb": ("roads",)}, + "surface_water": {"grb": ("water",)}, + "land_cover_use": { + "department_omgeving_thematic_raster": (), + "agentschap_landbouw_zeevisserij_agricultural_parcels": (), + }, + "nature": {"inbo_bwk_natura2000": ()}, + "parcels": { + "grb": ("parcels",), + "agentschap_landbouw_zeevisserij_agricultural_parcels": (), + }, + "soil": {"dov_soil_map": ()}, + "elevation": {"digitaal_vlaanderen_dhmv": ()}, + "orthophoto": {"digitaal_vlaanderen_orthophoto": ()}, + "flood_climate": {"vmm_flood_hazard": ()}, +} + + +class CoverageRegistryService: + @staticmethod + def catalog() -> CoverageCatalogResponse: + return CoverageCatalogResponse( + themes=list(THEMES), + zones=list(ZONES), + statuses=list(STATUS_ORDER), + sources=[definition.contract for definition in SOURCE_DEFINITIONS], + ) + + @staticmethod + def normalize_themes(themes: Iterable[str]) -> list[str]: + requested = list(dict.fromkeys(str(theme).strip().lower() for theme in themes if str(theme).strip())) + invalid = sorted(set(requested) - set(THEMES)) + if invalid: + raise AppError( + code="COVERAGE_THEME_UNSUPPORTED", + message="One or more coverage themes are unsupported", + status_code=422, + details={"unsupported_themes": invalid, "supported_themes": list(THEMES)}, + ) + return requested or list(THEMES) + + @staticmethod + def _geometry(value: Any): + if value is None: + return None + return value if hasattr(value, "__geo_interface__") else to_shape(value) + + @staticmethod + def _intersected_zones(areas: list[Area], selection) -> tuple[list[str], bool]: + geometries: dict[str, Any] = {} + by_name = {area.name: area for area in areas} + for zone, area_name in SCOPE_AREA_NAMES.items(): + area = by_name.get(area_name) + geometry = CoverageRegistryService._geometry(area.geometry) if area else None + if geometry is not None and not geometry.is_empty: + geometries[zone] = geometry + + detail_intersections = [ + zone for zone in DETAIL_ZONES if zone in geometries and geometries[zone].intersects(selection) + ] + zones = detail_intersections + if not any(zone in zones for zone in ("flanders", "wallonia", "brussels")): + if "belgium" in geometries and geometries["belgium"].intersects(selection): + zones = ["belgium", *zones] + if not any(zone in zones for zone in ("territorial_sea", "exclusive_economic_zone", "continental_shelf")): + if "belgian_north_sea" in geometries and geometries["belgian_north_sea"].intersects(selection): + zones = [*zones, "belgian_north_sea"] + + intersected_geometries = [geometries[zone].intersection(selection) for zone in zones if zone in geometries] + covered = unary_union(intersected_geometries) if intersected_geometries else None + outside = covered is None or covered.is_empty or not covered.covers(selection) + return zones, outside + + @staticmethod + def _matching_datasets( + datasets: list[Dataset], + definition: _SourceDefinition, + theme: str, + zone: str, + ) -> list[Dataset]: + matches: list[Dataset] = [] + for dataset in datasets: + if dataset.status != "ready" or dataset.source_name not in definition.materialized_source_names: + continue + layer_names = definition.materialized_layer_names + if definition.contract.source_name == "digitaal_vlaanderen": + theme_sources = FLANDERS_THEME_DATASETS.get(theme, {}) + if dataset.source_name not in theme_sources: + continue + layer_names = theme_sources[dataset.source_name] + metadata = dataset.source_metadata if isinstance(dataset.source_metadata, dict) else {} + coverage_zones = metadata.get("coverage_zones") or metadata.get("coverage_zone") or [] + if isinstance(coverage_zones, str): + coverage_zones = [coverage_zones] + layer_matches = not layer_names or dataset.reference_layer_name in layer_names + zone_matches = not coverage_zones or zone in coverage_zones or "belgium" in coverage_zones + if layer_matches and zone_matches: + matches.append(dataset) + return matches + + @staticmethod + def _resolve_item( + *, + zone: str, + theme: str, + datasets: list[Dataset], + ) -> CoverageResolutionItem: + definitions = [ + definition + for definition in SOURCE_DEFINITIONS + if zone in definition.contract.coverage_zones and theme in definition.contract.themes + ] + if not definitions: + return CoverageResolutionItem( + zone=zone, + theme=theme, + status="unsupported", + source_names=[], + materialized_dataset_ids=[], + limitation_message="No audited source contract supports this theme in the selected zone.", + ) + + materialized: list[Dataset] = [] + source_statuses: list[str] = [] + limitations: list[str] = [] + for definition in definitions: + matches = CoverageRegistryService._matching_datasets( + datasets, + definition, + theme, + zone, + ) + materialized.extend(matches) + if matches: + source_statuses.append("operational") + elif definition.contract.integration_status == "operational": + source_statuses.append("partial") + else: + source_statuses.append(definition.contract.integration_status) + limitations.append(definition.contract.limitation_message) + + best_status = max(source_statuses, key=STATUS_RANK.__getitem__) + return CoverageResolutionItem( + zone=zone, + theme=theme, + status=best_status, + source_names=[definition.contract.source_name for definition in definitions], + materialized_dataset_ids=list(dict.fromkeys(dataset.id for dataset in materialized)), + limitation_message=" ".join(dict.fromkeys(limitations)), + ) + + @staticmethod + def resolve( + db: Session, + project_id: UUID, + bbox: CoverageBBox, + themes: Iterable[str], + ) -> CoverageResolveResponse: + if not db.get(Project, project_id): + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + + requested_themes = CoverageRegistryService.normalize_themes(themes) + selection = box(bbox.minx, bbox.miny, bbox.maxx, bbox.maxy) + areas = db.query(Area).filter(Area.project_id == project_id).all() + datasets = db.query(Dataset).filter(Dataset.project_id == project_id).all() + zones, outside_supported_scope = CoverageRegistryService._intersected_zones(areas, selection) + if not zones: + return CoverageResolveResponse( + project_id=project_id, + bbox=bbox, + requested_themes=requested_themes, + intersected_zones=[], + outside_supported_scope=True, + items=[], + warnings=["The selection does not intersect a persisted Belgium or Belgian North Sea scope."], + ) + + items = [ + CoverageRegistryService._resolve_item(zone=zone, theme=theme, datasets=datasets) + for zone in zones + for theme in requested_themes + ] + warnings = [] + if outside_supported_scope: + warnings.append("Part of the selection lies outside the persisted Belgium and Belgian North Sea scopes.") + if len(zones) > 1: + warnings.append( + "The selection crosses coverage zones; results remain split and only semantically compatible metrics may be merged." + ) + return CoverageResolveResponse( + project_id=project_id, + bbox=bbox, + requested_themes=requested_themes, + intersected_zones=zones, + outside_supported_scope=outside_supported_scope, + items=items, + warnings=warnings, + ) diff --git a/backend/tests/test_rc4_national_coverage.py b/backend/tests/test_rc4_national_coverage.py new file mode 100644 index 00000000..f635c4f3 --- /dev/null +++ b/backend/tests/test_rc4_national_coverage.py @@ -0,0 +1,212 @@ +from __future__ import annotations + +from types import SimpleNamespace +from pathlib import Path +from uuid import uuid4 + +from fastapi.testclient import TestClient +from shapely.geometry import box + +from app.core.errors import AppError +from app.main import app +from app.models import Area, Dataset, Project +from app.schemas.coverage import CoverageBBox +from app.services.coverage_registry_service import CoverageRegistryService, THEMES, ZONES + + +class FakeQuery: + def __init__(self, rows): + self.rows = rows + + def filter(self, *_args, **_kwargs): + return self + + def all(self): + return list(self.rows) + + +class FakeSession: + def __init__(self, *, project, areas, datasets): + self.project = project + self.areas = areas + self.datasets = datasets + + def get(self, model, object_id): + if model is Project and str(self.project.id) == str(object_id): + return self.project + return None + + def query(self, model): + if model is Area: + return FakeQuery(self.areas) + if model is Dataset: + return FakeQuery(self.datasets) + raise AssertionError(f"Unexpected query model: {model}") + + +def scope_area(name: str, geometry): + return SimpleNamespace(name=name, geometry=geometry) + + +def test_coverage_catalog_uses_normalized_contracts_and_does_not_change_provider_registry() -> None: + catalog = CoverageRegistryService.catalog() + + assert set(catalog.themes) == set(THEMES) + assert set(catalog.zones) == set(ZONES) + assert catalog.statuses == ["unsupported", "not_configured", "partial", "operational"] + assert {source.source_name for source in catalog.sources} >= { + "ngi_adminvector", + "statbel", + "digitaal_vlaanderen", + "spw_geoportail", + "urbis", + "rbins_marine_reporting_units", + "rbins_msp_2026", + "mdk_bathymetry", + } + assert next(source for source in catalog.sources if source.source_name == "ngi_adminvector").license_note == "CC BY 4.0" + assert next(source for source in catalog.sources if source.source_name == "mdk_bathymetry").integration_status == "not_configured" + + response = TestClient(app).get("/api/v1/external/coverage/catalog") + assert response.status_code == 200 + assert response.json()["data"]["themes"] == list(THEMES) + + +def test_coverage_resolver_only_reports_operational_for_materialized_ready_dataset() -> None: + project_id = uuid4() + project = SimpleNamespace(id=project_id) + areas = [ + scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)), + scope_area("Flanders", box(2.5, 50.7, 5.9, 51.5)), + ] + bbox = CoverageBBox(minx=4.9, miny=50.9, maxx=5.0, maxy=51.0) + + without_materialized = CoverageRegistryService.resolve( + FakeSession(project=project, areas=areas, datasets=[]), + project_id, + bbox, + ["admin"], + ) + assert without_materialized.intersected_zones == ["flanders"] + assert without_materialized.items[0].status == "partial" + assert without_materialized.items[0].materialized_dataset_ids == [] + + dataset_id = uuid4() + materialized = SimpleNamespace( + id=dataset_id, + status="ready", + source_name="ngi_adminvector", + reference_layer_name="belgium_regions", + source_metadata={"coverage_zones": ["belgium", "flanders", "wallonia", "brussels"]}, + ) + with_materialized = CoverageRegistryService.resolve( + FakeSession(project=project, areas=areas, datasets=[materialized]), + project_id, + bbox, + ["admin"], + ) + admin_item = next(item for item in with_materialized.items if item.zone == "flanders") + assert admin_item.status == "operational" + assert admin_item.materialized_dataset_ids == [dataset_id] + + +def test_mixed_land_and_north_sea_selection_remains_split() -> None: + project_id = uuid4() + project = SimpleNamespace(id=project_id) + areas = [ + scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5)), + scope_area("Flanders", box(2.5, 50.7, 5.9, 51.5)), + scope_area("Belgian part of the North Sea", box(2.2, 51.1, 3.4, 51.9)), + scope_area("Belgian territorial sea (0-12 nautical miles)", box(2.7, 51.1, 3.4, 51.5)), + scope_area("Belgian exclusive economic zone beyond territorial sea", box(2.2, 51.5, 3.1, 51.9)), + scope_area("Belgian continental shelf beyond territorial sea", box(2.2, 51.5, 3.1, 51.9)), + ] + + result = CoverageRegistryService.resolve( + FakeSession(project=project, areas=areas, datasets=[]), + project_id, + CoverageBBox(minx=2.65, miny=51.05, maxx=2.85, maxy=51.2), + ["admin", "bathymetry"], + ) + + assert result.intersected_zones == ["flanders", "territorial_sea"] + assert len(result.items) == 4 + assert any("crosses coverage zones" in warning for warning in result.warnings) + bathymetry = next(item for item in result.items if item.zone == "territorial_sea" and item.theme == "bathymetry") + assert bathymetry.status == "not_configured" + assert bathymetry.materialized_dataset_ids == [] + + +def test_flemish_materialization_is_theme_specific() -> None: + project_id = uuid4() + project = SimpleNamespace(id=project_id) + orthophoto_id = uuid4() + orthophoto = SimpleNamespace( + id=orthophoto_id, + status="ready", + source_name="digitaal_vlaanderen_orthophoto", + reference_layer_name="orthophoto", + source_metadata={"coverage_zones": ["flanders"]}, + ) + result = CoverageRegistryService.resolve( + FakeSession( + project=project, + areas=[scope_area("Flanders", box(2.5, 50.7, 5.9, 51.5))], + datasets=[orthophoto], + ), + project_id, + CoverageBBox(minx=4.9, miny=50.9, maxx=5.0, maxy=51.0), + ["orthophoto", "roads"], + ) + + assert next(item for item in result.items if item.theme == "orthophoto").status == "operational" + assert next(item for item in result.items if item.theme == "roads").status == "partial" + + +def test_outside_scope_and_unknown_theme_are_explicit() -> None: + project_id = uuid4() + project = SimpleNamespace(id=project_id) + db = FakeSession( + project=project, + areas=[scope_area("Belgium land", box(2.5, 49.5, 6.4, 51.5))], + datasets=[], + ) + result = CoverageRegistryService.resolve( + db, + project_id, + CoverageBBox(minx=7.0, miny=52.0, maxx=7.1, maxy=52.1), + ["admin"], + ) + assert result.intersected_zones == [] + assert result.outside_supported_scope is True + assert result.items == [] + + try: + CoverageRegistryService.resolve( + db, + project_id, + CoverageBBox(minx=4.0, miny=50.0, maxx=4.1, maxy=50.1), + ["invented_metric"], + ) + except AppError as exc: + assert exc.code == "COVERAGE_THEME_UNSUPPORTED" + assert exc.status_code == 422 + assert exc.details["unsupported_themes"] == ["invented_metric"] + else: + raise AssertionError("Unknown coverage theme was accepted") + + +def test_frontend_prefers_materialized_national_workspace_and_resolves_drawn_bbox() -> None: + root = Path(__file__).parents[2] + focus = (root / "frontend" / "src" / "config" / "primaryFocus.ts").read_text(encoding="utf-8") + workspace_hook = (root / "frontend" / "src" / "hooks" / "useProjectWorkspace.ts").read_text(encoding="utf-8") + coverage_hook = (root / "frontend" / "src" / "hooks" / "useCoverageResolver.ts").read_text(encoding="utf-8") + map_workspace = (root / "frontend" / "src" / "components" / "map" / "MapWorkspace.tsx").read_text(encoding="utf-8") + + assert "Belgium and North Sea Workbench" in focus + assert "nationalProject" in workspace_hook + assert "data.areas.length > 0" in workspace_hook + assert "dataset.status === 'ready'" in workspace_hook + assert "externalApi.resolveCoverage" in coverage_hook + assert "coverage.outside_supported_scope" in map_workspace + assert "coverageStatusLabel" in map_workspace diff --git a/backend/tests/test_rc4_national_scope_operator.py b/backend/tests/test_rc4_national_scope_operator.py new file mode 100644 index 00000000..f0dbe99e --- /dev/null +++ b/backend/tests/test_rc4_national_scope_operator.py @@ -0,0 +1,161 @@ +from __future__ import annotations + +import json +import sys +import zipfile +from pathlib import Path + +import pytest +from shapely.geometry import box, mapping, shape + + +ROOT = Path(__file__).parents[2] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +import provision_belgium_north_sea_scope as operator # noqa: E402 + + +def marine_feature(identifier: str, geometry): + return { + "type": "Feature", + "id": identifier, + "geometry": mapping(geometry), + "properties": {"MarineReportingUnitId": identifier}, + } + + +def test_marine_legal_scopes_are_derived_from_official_reporting_units() -> None: + reporting_units = { + "type": "FeatureCollection", + "features": [ + marine_feature("ANS-BE-MS-1", box(0, 0, 10, 10)), + marine_feature("ANS-BE-AA-CW", box(0, 0, 2, 2)), + marine_feature("ANS-BE-AA-TEW", box(0, 2, 2, 4)), + marine_feature("ANS-BE-AA-OFFSHORE", box(2, 0, 10, 10)), + ], + } + + payload = operator.derive_marine_scope_payload(reporting_units) + by_zone = { + feature["properties"]["coverage_zone"]: feature + for feature in payload["features"] + } + + assert set(by_zone) == { + "belgian_north_sea", + "territorial_sea", + "exclusive_economic_zone", + "continental_shelf", + } + assert shape(by_zone["territorial_sea"]["geometry"]).area == pytest.approx(8.0) + assert shape(by_zone["exclusive_economic_zone"]["geometry"]).equals( + shape(by_zone["continental_shelf"]["geometry"]) + ) + assert ( + by_zone["exclusive_economic_zone"]["properties"]["legal_domain"] + != by_zone["continental_shelf"]["properties"]["legal_domain"] + ) + assert by_zone["territorial_sea"]["properties"]["derived_from_reporting_unit_ids"] == [ + "ANS-BE-AA-CW", + "ANS-BE-AA-TEW", + ] + + +def test_marine_scope_derivation_fails_when_a_required_unit_is_missing() -> None: + with pytest.raises(RuntimeError, match="ANS-BE-AA-CW"): + operator.derive_marine_scope_payload( + { + "type": "FeatureCollection", + "features": [ + marine_feature("ANS-BE-MS-1", box(0, 0, 10, 10)), + marine_feature("ANS-BE-AA-TEW", box(0, 2, 2, 4)), + marine_feature("ANS-BE-AA-OFFSHORE", box(2, 0, 10, 10)), + ], + } + ) + + +def test_archive_extraction_accepts_one_safe_geopackage_and_rejects_traversal(tmp_path: Path) -> None: + archive = tmp_path / "adminvector.zip" + with zipfile.ZipFile(archive, "w") as handle: + handle.writestr("release/adminvector.gpkg", b"sqlite-bytes") + + result = operator.extract_single_geopackage(archive, tmp_path / "output") + assert result.read_bytes() == b"sqlite-bytes" + + unsafe = tmp_path / "unsafe.zip" + with zipfile.ZipFile(unsafe, "w") as handle: + handle.writestr("../adminvector.gpkg", b"unsafe") + with pytest.raises(RuntimeError, match="unsafe"): + operator.extract_single_geopackage(unsafe, tmp_path / "unsafe-output") + + +class FakeResponse: + def __init__(self, payload): + self.payload = payload + self.content = json.dumps(payload).encode("utf-8") + + def raise_for_status(self): + return None + + def json(self): + return self.payload + + +class FakeSession: + def __init__(self, pages): + self.pages = list(pages) + self.calls = [] + + def get(self, url, params, timeout): + self.calls.append({"url": url, "params": params, "timeout": timeout}) + return FakeResponse(self.pages.pop(0)) + + +def test_wfs_fetch_is_allowlisted_paginated_and_complete() -> None: + pages = [ + { + "type": "FeatureCollection", + "numberMatched": 2, + "features": [{"type": "Feature", "id": "unit.1", "geometry": None, "properties": {}}], + }, + { + "type": "FeatureCollection", + "numberMatched": 2, + "features": [{"type": "Feature", "id": "unit.2", "geometry": None, "properties": {}}], + }, + ] + session = FakeSession(pages) + payload = operator.fetch_wfs_layer( + session, + service_url=operator.RBINS_MRU_WFS_URL, + layer_name=operator.RBINS_MRU_LAYER, + timeout=30, + page_size=1, + ) + + assert [feature["id"] for feature in payload["features"]] == ["unit.1", "unit.2"] + assert [call["params"]["startIndex"] for call in session.calls] == [0, 1] + assert all(call["params"]["srsName"] == "EPSG:4326" for call in session.calls) + + with pytest.raises(RuntimeError, match="allowlist"): + operator.fetch_wfs_layer( + FakeSession([]), + service_url=operator.RBINS_MSP_WFS_URL, + layer_name="untrusted:layer", + timeout=30, + ) + + +def test_operator_is_packaged_and_guarded_by_readiness() -> None: + dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8") + readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") + source = (ROOT / "scripts" / "provision_belgium_north_sea_scope.py").read_text(encoding="utf-8") + + assert "COPY scripts/provision_belgium_north_sea_scope.py /app/scripts/" in dockerfile + assert "py_compile scripts/provision_belgium_north_sea_scope.py" in readiness + assert "/datasets/upload" in source + assert "from app.models" not in source + assert "INSERT INTO vector_features" not in source diff --git a/backend/tests/test_sprint31_unraid_template.py b/backend/tests/test_sprint31_unraid_template.py index cc05d58f..02d9365b 100644 --- a/backend/tests/test_sprint31_unraid_template.py +++ b/backend/tests/test_sprint31_unraid_template.py @@ -113,7 +113,10 @@ def test_tower_deploy_uses_single_container_unraid_compose() -> None: for script in (powershell, bash): assert "docker compose -f docker-compose.unraid.yml config" in script assert "--build-arg GEOINTEL_INSTALL_AI=" in script - assert "-f deploy/unraid/Dockerfile.all-in-one -t geointel-all-in-one:latest ." in script + assert '--build-arg GEOINTEL_BUILD_SHA="$GEOINTEL_BUILD_SHA"' in script + assert '--build-arg GEOINTEL_BUILD_TIME="$GEOINTEL_BUILD_TIME"' in script + assert "-f deploy/unraid/Dockerfile.all-in-one" in script + assert "-t geointel-all-in-one:latest" in script assert "docker compose -f docker-compose.unraid.yml build geointel" not in script assert "bash deploy/unraid/run-dockerman-container.sh" in script assert "LIVE_SMOKE_CONTAINER=geointel bash scripts/live_migration_smoke.sh" in script diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index ff9a64dc..a381a685 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -106,6 +106,7 @@ COPY scripts/provision_buildings_addresses_register.py /app/scripts/provision_bu COPY scripts/provision_regional_timeseries.py /app/scripts/provision_regional_timeseries.py COPY scripts/geographic_scopes.py /app/scripts/geographic_scopes.py COPY scripts/provision_geographic_scope.py /app/scripts/provision_geographic_scope.py +COPY scripts/provision_belgium_north_sea_scope.py /app/scripts/provision_belgium_north_sea_scope.py COPY scripts/provision_regional_grb_buildings.py /app/scripts/provision_regional_grb_buildings.py COPY scripts/provision_regional_grb_context.py /app/scripts/provision_regional_grb_context.py COPY scripts/audit_source_freshness.py /app/scripts/audit_source_freshness.py diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index 1881f845..c8863d43 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -935,6 +935,49 @@ Read simplified job status payload. ## Provider registry +The legacy Sprint 7 provider registry remains the exact +`grb|osm|manual|fixture` import abstraction. National and maritime source +authority is exposed separately so adding official source families cannot +silently change that import contract. + +### GET `/api/v1/external/coverage/catalog` + +Returns the normalized Belgium and Belgian North Sea theme vocabulary, legal +coverage zones, allowed availability states and audited source contracts. +Every source includes authority, native layers, geometry types, acquisition +mode, source URL, attribution, licence and limitation text. + +Allowed result states are exactly: + +- `operational`: a matching `ready` Dataset is materialized in the project; +- `partial`: the governed integration exists but matching project data is + absent or incomplete; +- `not_configured`: an audited source has no operational adapter; +- `unsupported`: no audited contract supports the theme in that zone. + +### POST `/api/v1/external/coverage/resolve` + +Resolves a user-drawn EPSG:4326 bbox against persisted national and maritime +Areas. Cross-region and land/sea selections remain split by zone. + +```json +{ + "project_id": "uuid", + "bbox": { + "minx": 2.65, + "miny": 51.05, + "maxx": 2.85, + "maxy": 51.20 + }, + "themes": ["buildings", "surface_water", "bathymetry"] +} +``` + +The response returns `intersected_zones`, `outside_supported_scope`, one item +per zone/theme combination, matching source names and IDs of actually +materialized Datasets. An empty `themes` list requests the complete normalized +vocabulary. Unknown themes fail with `COVERAGE_THEME_UNSUPPORTED`. + ### GET `/api/v1/external/providers` Returns all configured provider capability descriptors. diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 5664f676..56342ebf 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -1,5 +1,31 @@ ## Autonomous RC program for Belgium and the Belgian North Sea (2026-07-17) +### RC-4 national and maritime coverage foundation + +- Added an independent coverage registry so the exact Sprint 7 + `grb|osm|manual|fixture` provider contract remains unchanged. +- Added `GET /api/v1/external/coverage/catalog` and + `POST /api/v1/external/coverage/resolve` with canonical envelopes, normalized + themes and explicit project-materialization checks. +- Added `provision_belgium_north_sea_scope.py`. It accepts only the official + NGI AdminVector archive, RBINS marine reporting units and the exact 15-layer + `imsp26` WFS allowlist. It verifies archive/response limits, safe extraction, + layer counts, pagination, geometries and SHA256 evidence. +- Verified live source contracts from the Codex host: all 15 MSP layers were + reachable and returned 101 features in total; the current RBINS reporting + units yielded BPNS, territorial sea, EEZ and continental-shelf geometries. +- The territorial sea is derived only by unioning official 0-1 nm and 1-12 nm + units. EEZ and continental shelf retain the same official offshore geometry + as separate records with different legal-domain provenance. +- Frontend startup now prefers a materialized + `Belgium and North Sea Workbench`; drawn bboxes receive a compact, + zone-split coverage matrix. Existing Mol/Kempen workspaces remain available. +- Local validation passed 986 backend tests, frontend typecheck/build, the + full readiness gate, Alembic head `202607160001`, complete offline migration + SQL and live-smoke shell syntax. +- Tower fetch-only, persistence, live coverage API and browser acceptance are + the remaining RC-4 exit evidence. + - Froze the RC geography as all Belgian land plus the separately labelled territorial sea, EEZ and continental shelf. - Retained Mol and the Kempen as validated regression areas instead of the diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md index a242e536..e0d6800c 100644 --- a/docs/DATA_SOURCES.md +++ b/docs/DATA_SOURCES.md @@ -719,6 +719,42 @@ the canonical vector persistence path for Mol and Kempen. Additional Flemish vector themes still require governed regional operators; a catalogue record alone never makes them operational. +## Belgium and Belgian North Sea scope foundation + +The release-candidate workbench uses two distinct registries: + +- the fixed Sprint 7 import-provider registry for `grb`, `osm`, `manual` and + `fixture`; +- the national coverage registry for audited source authority and actual + project materialization by theme and zone. + +The explicit operator +`scripts/provision_belgium_north_sea_scope.py` downloads the official NGI +AdminVector EPSG:4326 GeoPackage archive, verifies size and archive safety, +requires the territory, maritime zone, region, province and municipality +layers, and retains archive/GeoPackage/artifact checksums. NGI output is +licensed CC BY 4.0 and persists as reference Datasets through +`DatasetService`. + +The same operator obtains the RBINS/BMDC +`od_nature:marine_reporting_units_2024` WFS layer. The territorial sea is the +union of official 0-1 nm coastal waters and 1-12 nm territorial waters. The +official offshore-beyond-12-nm geometry is persisted twice only to preserve +the distinct legal semantics of the EEZ water column and continental-shelf +seabed/subsoil. This is explicit provenance-preserving derivation, not +invented geometry. + +The legally current Belgian Marine Spatial Plan 2026-2034 is fetched from the +fixed `imsp26` WFS allowlist and retained as one versioned reference Dataset. +No live vessel activity, depth, water volume or other unsupported metric is +inferred from these plan zones. + +The coverage resolver reports a theme `operational` only when a matching +`ready` Dataset exists in the selected project. Implemented acquisition +without project materialization is `partial`; Walloon, Brussels, Statbel and +MDK families remain `not_configured` until their own bounded adapters and +source-specific metric contracts are implemented. + ## Bathymetry, inland profiles and maritime scope The official VHA Digital Atlas profile-point layer is the first operational diff --git a/docs/DATA_SPECIFICATION.md b/docs/DATA_SPECIFICATION.md index 0e0ab669..17448ed3 100644 --- a/docs/DATA_SPECIFICATION.md +++ b/docs/DATA_SPECIFICATION.md @@ -369,6 +369,31 @@ Ondersteund: - GPKG - WFS-resultaten +### National and maritime coverage contracts + +Belgium-wide and maritime reference Datasets use the same Dataset and +VectorFeature entities as every other vector source. No parallel national +schema is introduced. + +Required source metadata includes: + +```yaml +authority_level: authoritative|official_context|contextual +coverage_zones: + - belgium|flanders|wallonia|brussels + - belgian_north_sea|territorial_sea|exclusive_economic_zone|continental_shelf +source_url: https://... +attribution: text +license_note: text +reference_layer_name: normalized_layer_key +``` + +EEZ and continental-shelf records may have equal horizontal geometry while +their `legal_domain` differs. They must remain distinct records. A selected +bbox crossing regions or the coast produces zone-specific coverage items; +only metrics with compatible source meaning, unit, time and method may be +merged. + Metadata: - CRS diff --git a/docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md b/docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md index c7235c55..4f04fc78 100644 --- a/docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md +++ b/docs/RC_ROADMAP_BELGIUM_NORTH_SEA.md @@ -291,6 +291,11 @@ errors. ## RC-4 - Belgium and North Sea coverage foundation +**State: in progress.** The national coverage registry, canonical +catalog/resolve API, explicit NGI/RBINS operator, frontend selection matrix and +focused safety tests are implemented. Local readiness is green. Tower +fetch-only, persistence, live API and browser acceptance remain the phase exit. + ### Work 1. Persist NGI authoritative Belgium land and administrative scopes. diff --git a/docs/STORAGE_ARCHITECTURE.md b/docs/STORAGE_ARCHITECTURE.md index 5b645cc6..a7e73375 100644 --- a/docs/STORAGE_ARCHITECTURE.md +++ b/docs/STORAGE_ARCHITECTURE.md @@ -43,8 +43,28 @@ storage/ osm/ sentinel/ dhmv/ + operator-data/ + geographic-scopes/ + belgium-north-sea/ + adminvector_4326.zip + adminvector/ + adminvector_4326.gpkg + belgium_land_boundary.geojson + belgium_regions.geojson + belgium_provinces.geojson + belgium_municipalities.geojson + ngi_maritime_zone.geojson + marine_legal_scopes.geojson + marine_spatial_plan_2026.geojson + manifest.json ``` +The Belgium/North Sea directory is written only by the explicit +`provision_belgium_north_sea_scope.py` operator. The manifest binds every +artifact to SHA256, source URL, edition/validity and feature count. Persistence +still runs through the ordinary Dataset upload API; operator artifacts are +evidence and replay inputs, not an alternate database. + ## Upload policy When a file is uploaded: diff --git a/docs/TODO.md b/docs/TODO.md index 6f9b07c2..aa214d86 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -14,6 +14,12 @@ maritieme zones. - [x] RC-2: liveness/readiness/capabilities fail-closed en waarheidsgetrouw maken. - [x] RC-2: verweesde jobs en analysis runs na een procesherstart verzoenen. - [x] RC-3: expliciete rasterkeuze en temporeel compatibele detectie-QA afdwingen. +- [x] RC-4: aparte nationale bron- en dekkingscontracten implementeren zonder + het vaste Sprint 7-providerregister te wijzigen. +- [x] RC-4: expliciete NGI/RBINS-operator en selectiegebonden coverage-API + implementeren en lokaal valideren. +- [ ] RC-4: Tower fetch-only, canonieke persistence en live browseracceptatie + bewijzen. - [ ] RC-4: nationale basisdekking, Wallonie, Brussel en Belgische Noordzee via beheerde providers en golden areas operationaliseren. - [ ] RC-5: secrets/configuratie/uploadlimieten/immutable deploy en rollback diff --git a/frontend/README.md b/frontend/README.md index 3f8715da..7c23f64a 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -713,3 +713,16 @@ user selects a municipality or draws a bounded rectangle. Forest and agriculture use thematic raster analysis; nature value and soil use the persisted-vector GeoJSON pattern. The browser calls only the GeoIntel API and never contacts WCS, WFS or OGC providers directly. + +## Belgium and Belgian North Sea coverage + +When `Belgium and North Sea Workbench` exists with ready reference data, it is +preferred over the legacy Mol/Kempen workspaces at startup. Mol and Kempen +remain regression workspaces and can still be selected normally. + +Drawing a rectangle on the Map invokes the GeoIntel coverage resolver after a +short debounce. The map displays every intersected land or legal sea zone and +the active theme as `Beschikbaar`, `Gedeeltelijk`, `Niet gekoppeld` or `Niet +ondersteund`. A coastal or cross-region selection remains visibly split. The +browser never calls NGI, SPW, UrbIS, RBINS or MDK directly and never promotes +an audited catalog entry to operational data without a matching ready Dataset. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 341267b2..c238b77d 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -19,6 +19,7 @@ import { ProviderPanel } from './components/providers/ProviderPanel' import { SegmentationLab } from './components/segmentation/SegmentationLab' import { useChangeDetectionWorkflow } from './hooks/useChangeDetectionWorkflow' import { useDemoWorkflow } from './hooks/useDemoWorkflow' +import { useCoverageResolver } from './hooks/useCoverageResolver' import { useDetectionWorkflow } from './hooks/useDetectionWorkflow' import { useDatasetWorkflow } from './hooks/useDatasetWorkflow' import { useExportWorkflow } from './hooks/useExportWorkflow' @@ -39,6 +40,8 @@ import { getDatasetDisplayName } from './lib/datasetDisplay' import { FLANDERS_WORKSPACE_LABEL, FLANDERS_WORKSPACE_PROJECT_NAME, + NATIONAL_WORKSPACE_LABEL, + NATIONAL_WORKSPACE_PROJECT_NAME, REGIONAL_WORKSPACE_LABEL, REGIONAL_WORKSPACE_PROJECT_NAME, } from './config/primaryFocus' @@ -475,6 +478,14 @@ function App(): JSX.Element { selectedDataset, isVectorDatasetType, }) + const { + coverage: mapCoverage, + loadingCoverage: mapCoverageLoading, + coverageError: mapCoverageError, + } = useCoverageResolver({ + projectId: selectedProjectId, + bbox: mapSelectionBbox, + }) const { selectionDatasetSaving, selectionDatasetError, @@ -668,11 +679,13 @@ function App(): JSX.Element { || Boolean(changeDetectionResult) || detectionItems.length > 0 || segmentationItems.length > 0 - const projectContextLabel = selectedProject?.name === REGIONAL_WORKSPACE_PROJECT_NAME - ? REGIONAL_WORKSPACE_LABEL - : selectedProject?.name === FLANDERS_WORKSPACE_PROJECT_NAME - ? FLANDERS_WORKSPACE_LABEL - : selectedProject?.name ?? 'Geen werkruimte' + const projectContextLabel = selectedProject?.name === NATIONAL_WORKSPACE_PROJECT_NAME + ? NATIONAL_WORKSPACE_LABEL + : selectedProject?.name === REGIONAL_WORKSPACE_PROJECT_NAME + ? REGIONAL_WORKSPACE_LABEL + : selectedProject?.name === FLANDERS_WORKSPACE_PROJECT_NAME + ? FLANDERS_WORKSPACE_LABEL + : selectedProject?.name ?? 'Geen werkruimte' const areaContextLabel = selectedArea?.name ?? (areas.length > 0 ? 'Kies een gebied' : 'Geen gebied') const bathymetryContextActive = Boolean( activeWorkspace === 'map' @@ -927,6 +940,9 @@ function App(): JSX.Element { mapSelectionResult={mapSelectionResult} mapSelectionLoading={mapSelectionLoading} mapSelectionError={mapSelectionError} + coverage={mapCoverage} + coverageLoading={mapCoverageLoading} + coverageError={mapCoverageError} selectionExporting={selectionExporting} selectionExportError={selectionExportError} latestSelectionExportPath={latestSelectionExport?.path ?? null} diff --git a/frontend/src/components/map/MapWorkspace.tsx b/frontend/src/components/map/MapWorkspace.tsx index 10541908..c2ddc2d7 100644 --- a/frontend/src/components/map/MapWorkspace.tsx +++ b/frontend/src/components/map/MapWorkspace.tsx @@ -1,6 +1,6 @@ import { useEffect, useMemo, useState } from 'react' import GeoMap from '../GeoMap' -import type { AreaRead, DatasetCreateResponse, DetectionQaResult, MapResultExportRequest, MapViewportState, OrthophotoAcquisitionResult, OrthophotoProductRead, ProjectRead, QaComparisonResult, VectorSelectionBBox, VectorSelectionMetric, VectorSelectionResponse } from '../../types' +import type { AreaRead, CoverageResolveResponse, CoverageStatus, DatasetCreateResponse, DetectionQaResult, MapResultExportRequest, MapViewportState, OrthophotoAcquisitionResult, OrthophotoProductRead, ProjectRead, QaComparisonResult, VectorSelectionBBox, VectorSelectionMetric, VectorSelectionResponse } from '../../types' import { useMapThemeSelectionInsights, type MapThemeAcquisition, type MapThemeQuery } from '../../hooks/useMapThemeSelectionInsights' import { useOfficialMapProducts } from '../../hooks/useOfficialMapProducts' import { useTemporalComparison } from '../../hooks/useTemporalComparison' @@ -176,6 +176,49 @@ const DATA_THEMES: DataTheme[] = [ }, ] +const COVERAGE_THEME_BY_MAP_THEME: Record = { + buildings: 'buildings', + space_occupation: 'land_cover_use', + open_space: 'land_cover_use', + population: 'population', + forest: 'land_cover_use', + nature_value: 'nature', + agriculture: 'land_cover_use', + soil: 'soil', + water: 'surface_water', + bathymetry: 'bathymetry', + flood_hazard: 'flood_climate', + elevation: 'elevation', + accessibility: 'roads', + services: 'population', + roads: 'roads', + parcels: 'parcels', +} + +function coverageStatusLabel(status: CoverageStatus): string { + const labels: Record = { + operational: 'Beschikbaar', + partial: 'Gedeeltelijk', + not_configured: 'Niet gekoppeld', + unsupported: 'Niet ondersteund', + } + return labels[status] +} + +function coverageZoneLabel(zone: string): string { + const labels: Record = { + belgium: 'Belgie', + flanders: 'Vlaanderen', + wallonia: 'Wallonie', + brussels: 'Brussel', + belgian_north_sea: 'Belgische Noordzee', + territorial_sea: 'Territoriale zee', + exclusive_economic_zone: 'EEZ', + continental_shelf: 'Continentaal plat', + } + return labels[zone] ?? zone +} + const DATA_THEME_MAP_STYLES: Record = { buildings: { fill: '#d45f3d', line: '#9f3e24' }, space_occupation: { fill: '#be3e33', line: '#8f2c24' }, @@ -516,6 +559,9 @@ interface MapWorkspaceProps { mapSelectionResult: VectorSelectionResponse | null mapSelectionLoading: boolean mapSelectionError: string | null + coverage: CoverageResolveResponse | null + coverageLoading: boolean + coverageError: string | null selectionExporting: boolean selectionExportError: string | null latestSelectionExportPath: string | null @@ -602,6 +648,9 @@ export function MapWorkspace({ mapSelectionResult, mapSelectionLoading, mapSelectionError, + coverage, + coverageLoading, + coverageError, selectionExporting, selectionExportError, latestSelectionExportPath, @@ -819,6 +868,15 @@ export function MapWorkspace({ [availableMapDatasets, regionalScopeSelected, selectedMapAreaId, themeDatasetMap], ) const activeTheme = DATA_THEMES.find((theme) => theme.id === activeThemeId) ?? DATA_THEMES[0] + const activeCoverageTheme = COVERAGE_THEME_BY_MAP_THEME[activeTheme.id] + const activeCoverageItems = coverage?.items.filter((item) => item.theme === activeCoverageTheme) ?? [] + const coverageCounts = useMemo( + () => coverage?.items.reduce>( + (counts, item) => ({ ...counts, [item.status]: counts[item.status] + 1 }), + { operational: 0, partial: 0, not_configured: 0, unsupported: 0 }, + ) ?? { operational: 0, partial: 0, not_configured: 0, unsupported: 0 }, + [coverage], + ) const onDemandProductMap = useMemo( () => { const result = new Map() @@ -2527,6 +2585,50 @@ export function MapWorkspace({ ) : null} + {mapSelectionBbox ? ( +
+
+
+

Dekking van deze selectie

+

{activeTheme.label}

+
+ {coverageLoading ? controleren : null} +
+ {coverageError ?

{coverageError}

: null} + {coverage ? ( + <> +
+ {coverage.intersected_zones.map((zone) => ( + {coverageZoneLabel(zone)} + ))} + {coverage.outside_supported_scope ? deels buiten scope : null} +
+
+ {activeCoverageItems.map((item) => ( +
+ {coverageZoneLabel(item.zone)} + {coverageStatusLabel(item.status)} + {item.source_names.join(', ') || 'Geen broncontract'} +
+ ))} + {activeCoverageItems.length === 0 ? ( +

Deze selectie raakt geen bewaarde Belgische land- of zeezone.

+ ) : null} +
+
+ {coverageCounts.operational} beschikbaar + {coverageCounts.partial} gedeeltelijk + {coverageCounts.not_configured} niet gekoppeld + {coverageCounts.unsupported} niet ondersteund +
+ {coverage.warnings.map((warning) =>

{warning}

)} + + ) : !coverageLoading && !coverageError ? ( +

De dekkingsmatrix wordt bepaald zodra de selectie volledig is.

+ ) : null} +
+ ) : null} +
diff --git a/frontend/src/config/primaryFocus.ts b/frontend/src/config/primaryFocus.ts index e9a3d3dc..db30c245 100644 --- a/frontend/src/config/primaryFocus.ts +++ b/frontend/src/config/primaryFocus.ts @@ -2,6 +2,8 @@ import type { DatasetCreateResponse, ProjectRead } from '../types' export const PRIMARY_FOCUS_LABEL = 'Mol' export const PRIMARY_FOCUS_REGION = 'Mol, Kempen' +export const NATIONAL_WORKSPACE_PROJECT_NAME = 'Belgium and North Sea Workbench' +export const NATIONAL_WORKSPACE_LABEL = 'Belgie en Belgische Noordzee' export const REGIONAL_WORKSPACE_PROJECT_NAME = 'Kempen Regional Workbench' export const REGIONAL_WORKSPACE_LABEL = 'Kempen (28 gemeenten)' export const FLANDERS_WORKSPACE_PROJECT_NAME = 'Flanders Regional Workbench' diff --git a/frontend/src/hooks/useCoverageResolver.ts b/frontend/src/hooks/useCoverageResolver.ts new file mode 100644 index 00000000..fa02e083 --- /dev/null +++ b/frontend/src/hooks/useCoverageResolver.ts @@ -0,0 +1,63 @@ +import { useEffect, useState } from 'react' +import { externalApi } from '../services/api' +import type { CoverageResolveResponse, VectorSelectionBBox } from '../types' + +interface CoverageResolverOptions { + projectId: string | null + bbox: VectorSelectionBBox | null +} + +export function useCoverageResolver({ projectId, bbox }: CoverageResolverOptions) { + const [coverage, setCoverage] = useState(null) + const [loadingCoverage, setLoadingCoverage] = useState(false) + const [coverageError, setCoverageError] = useState(null) + + useEffect(() => { + if (!projectId || !bbox) { + setCoverage(null) + setCoverageError(null) + setLoadingCoverage(false) + return + } + let cancelled = false + const timer = window.setTimeout(() => { + setLoadingCoverage(true) + setCoverageError(null) + externalApi.resolveCoverage({ + projectId, + bbox: { + minx: bbox.min_x, + miny: bbox.min_y, + maxx: bbox.max_x, + maxy: bbox.max_y, + }, + }) + .then((result) => { + if (!cancelled) { + setCoverage(result) + } + }) + .catch((error) => { + if (!cancelled) { + setCoverage(null) + setCoverageError(error instanceof Error ? error.message : 'Dekking kon niet worden bepaald') + } + }) + .finally(() => { + if (!cancelled) { + setLoadingCoverage(false) + } + }) + }, 250) + return () => { + cancelled = true + window.clearTimeout(timer) + } + }, [projectId, bbox?.min_x, bbox?.min_y, bbox?.max_x, bbox?.max_y]) + + return { + coverage, + loadingCoverage, + coverageError, + } +} diff --git a/frontend/src/hooks/useProjectWorkspace.ts b/frontend/src/hooks/useProjectWorkspace.ts index 030185cd..13a61456 100644 --- a/frontend/src/hooks/useProjectWorkspace.ts +++ b/frontend/src/hooks/useProjectWorkspace.ts @@ -3,6 +3,7 @@ import { PRIMARY_FOCUS_AREA_GEOJSON, PRIMARY_FOCUS_AREA_NAME, PRIMARY_FOCUS_REGION, + NATIONAL_WORKSPACE_PROJECT_NAME, REGIONAL_WORKSPACE_PROJECT_NAME, isPrimaryFocusMunicipalityBoundaryDataset, isPrimaryFocusMunicipalityProject, @@ -81,6 +82,17 @@ export function useProjectWorkspace() { if (selectedProjectId && items.some((project) => project.id === selectedProjectId)) { return selectedProjectId } + const nationalProject = items.find((project) => project.name === NATIONAL_WORKSPACE_PROJECT_NAME) + if (nationalProject) { + try { + const data = await fetchProjectData(nationalProject.id) + if (data.areas.length > 0 && data.datasets.some((dataset) => dataset.status === 'ready')) { + return nationalProject.id + } + } catch { + // Continue with regional and municipality fallbacks while national data is unavailable. + } + } const regionalProject = items.find((project) => project.name === REGIONAL_WORKSPACE_PROJECT_NAME) if (regionalProject) { try { @@ -145,13 +157,15 @@ export function useProjectWorkspace() { setLoadingProjects(true) setErrorMessage(null) try { - const [response, canonicalResponse] = await Promise.all([ + const [response, nationalResponse, canonicalResponse] = await Promise.all([ projectsApi.list(), + projectsApi.list({ name: NATIONAL_WORKSPACE_PROJECT_NAME, limit: 1 }), projectsApi.list({ name: REGIONAL_WORKSPACE_PROJECT_NAME, limit: 1 }), ]) + const regionalItems = [...canonicalResponse.items, ...response.items] const items = Array.from( new Map( - [...canonicalResponse.items, ...response.items].map((project) => [project.id, project]), + [...nationalResponse.items, ...regionalItems].map((project) => [project.id, project]), ).values(), ) setProjects(items) diff --git a/frontend/src/services/api/external.ts b/frontend/src/services/api/external.ts index fd0ddfb7..7b13d4f1 100644 --- a/frontend/src/services/api/external.ts +++ b/frontend/src/services/api/external.ts @@ -1,5 +1,7 @@ import { apiGet, apiPost } from './client' import type { + CoverageCatalogResponse, + CoverageResolveResponse, ProviderCapability, ProviderCapabilitiesResponse, ProviderImportResponse, @@ -13,6 +15,18 @@ const normalize = (layers: string[] = []) => layers.filter((value) => value.trim export const externalApi = { listSystemCapabilities: (): Promise => apiGet('/api/v1/system/capabilities'), + getCoverageCatalog: (): Promise => + apiGet('/api/v1/external/coverage/catalog'), + resolveCoverage: (payload: { + projectId: string + bbox: { minx: number; miny: number; maxx: number; maxy: number } + themes?: string[] + }): Promise => + apiPost('/api/v1/external/coverage/resolve', { + project_id: payload.projectId, + bbox: payload.bbox, + themes: payload.themes ?? [], + }), listProviders: (): Promise => apiGet('/api/v1/external/providers'), listProviderCapabilities: (): Promise => diff --git a/frontend/src/styles/app.css b/frontend/src/styles/app.css index 190d6414..fa650fda 100644 --- a/frontend/src/styles/app.css +++ b/frontend/src/styles/app.css @@ -7347,3 +7347,67 @@ section { .workspace-persistent-map[hidden] { display: none; } +.coverage-resolution-surface { + padding: 18px 20px; + border-block: 1px solid var(--line); + background: color-mix(in srgb, var(--surface-raised) 94%, var(--accent-soft)); +} + +.coverage-zone-row, +.coverage-summary-row { + display: flex; + flex-wrap: wrap; + gap: 8px; + margin-top: 10px; +} + +.coverage-zone-row span, +.coverage-summary-row span { + padding: 4px 8px; + border: 1px solid var(--line); + border-radius: 4px; + background: var(--surface-raised); + color: var(--muted); + font-size: 0.78rem; +} + +.coverage-zone-row .coverage-zone-warning { + border-color: var(--warning); + color: var(--warning); +} + +.coverage-active-theme-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(170px, 1fr)); + gap: 8px; + margin-top: 12px; +} + +.coverage-status-item { + display: grid; + gap: 3px; + min-height: 76px; + padding: 10px 12px; + border: 1px solid var(--line); + border-left: 3px solid var(--muted); + border-radius: 6px; + background: var(--surface-raised); +} + +.coverage-status-item span, +.coverage-status-item small { + color: var(--muted); +} + +.coverage-status-operational { + border-left-color: var(--accent); +} + +.coverage-status-partial { + border-left-color: var(--warning); +} + +.coverage-status-not_configured, +.coverage-status-unsupported { + border-left-color: var(--muted); +} diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 71237b31..675463dd 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -957,6 +957,57 @@ export interface ProviderCapability { not_configured_reason: string | null } +export type CoverageStatus = 'operational' | 'partial' | 'not_configured' | 'unsupported' + +export interface CoverageBBox { + minx: number + miny: number + maxx: number + maxy: number +} + +export interface CoverageSourceContract { + source_name: string + display_name: string + authority_level: 'authoritative' | 'official_context' | 'contextual' + coverage_zones: string[] + themes: string[] + native_layers: string[] + supported_geometry_types: string[] + acquisition_mode: 'operator_archive' | 'operator_wfs' | 'bounded_api' | 'bounded_raster' | 'catalog_only' + integration_status: CoverageStatus + source_url: string + attribution: string + license_note: string + limitation_message: string +} + +export interface CoverageCatalogResponse { + themes: string[] + zones: string[] + statuses: CoverageStatus[] + sources: CoverageSourceContract[] +} + +export interface CoverageResolutionItem { + zone: string + theme: string + status: CoverageStatus + source_names: string[] + materialized_dataset_ids: string[] + limitation_message: string +} + +export interface CoverageResolveResponse { + project_id: string + bbox: CoverageBBox + requested_themes: string[] + intersected_zones: string[] + outside_supported_scope: boolean + items: CoverageResolutionItem[] + warnings: string[] +} + export interface SystemCapabilitiesResponse { status: string service: string diff --git a/scripts/README.md b/scripts/README.md index c5248b8f..c9f114f0 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1319,6 +1319,33 @@ audits artifacts without uploading; `--force` explicitly refreshes the source partitions. Historical identities are declared unstable and support hectare comparison only, not object lineage. +## Belgium and Belgian North Sea foundation + +Prepare and validate the complete national/maritime source snapshot without +changing application persistence: + +```bash +docker exec -it geointel python3 /app/scripts/provision_belgium_north_sea_scope.py \ + --fetch-only +``` + +Persist the national workbench through the canonical API: + +```bash +docker exec -it geointel python3 /app/scripts/provision_belgium_north_sea_scope.py +``` + +Use `--force` only for an explicit source refresh. The operator creates or +reuses `Belgium and North Sea Workbench`, eight legal/administrative Areas and +six reference Datasets. It never runs on container startup, never disables +TLS verification and never writes directly to PostGIS. Output and immutable +checksums are retained below +`/app/storage/operator-data/geographic-scopes/belgium-north-sea`. + +The command imports the common administrative baseline and marine legal/use +zones. It does not make detailed Walloon, Brussels, population, bathymetry or +other audited source families operational. + ## Official Kempen operational scope GeoIntel defines its regional `Kempen` workspace as the official Vlaamse diff --git a/scripts/provision_belgium_north_sea_scope.py b/scripts/provision_belgium_north_sea_scope.py new file mode 100644 index 00000000..8647a520 --- /dev/null +++ b/scripts/provision_belgium_north_sea_scope.py @@ -0,0 +1,950 @@ +"""Provision Belgium and the Belgian North Sea through the canonical API. + +This explicit operator downloads a bounded, allowlisted set of authoritative +sources. It never runs at application startup and never writes directly to +PostGIS or vector_features. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import sys +import zipfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +import requests +from requests.adapters import HTTPAdapter +from shapely.geometry import GeometryCollection, MultiPolygon, Polygon, mapping, shape +from shapely.ops import unary_union +from shapely.validation import make_valid +from urllib3.util.retry import Retry + + +PROJECT_NAME = "Belgium and North Sea Workbench" +PROJECT_REGION = "Belgium and Belgian North Sea" +DEFAULT_API_URL = "http://127.0.0.1:8000" +DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-data/geographic-scopes/belgium-north-sea") +NGI_ARCHIVE_URL = ( + "https://ac.ngi.be/remoteclient-open/ngi-standard-open/Vectordata/" + "TerritorialDivisions/TerritorialDivisions-AdminVector/" + "fb1e2993-2020-428c-9188-eb5f75e284b9_geopackage+sqlite3_4326.zip" +) +NGI_CATALOG_URL = "https://www.geo.be/catalog/details/fb1e2993-2020-428c-9188-eb5f75e284b9" +NGI_ATTRIBUTION = "National Geographic Institute (NGI), AdminVector" +NGI_LICENSE = "CC BY 4.0" +RBINS_MRU_WFS_URL = "https://spatial.naturalsciences.be/geoserver/od_nature/ows" +RBINS_MRU_METADATA_URL = ( + "https://metadata.naturalsciences.be/geonetwork/srv/api/records/" + "29f40b0d-2a3e-49a8-870a-e9b4acd4d1e3" +) +RBINS_MRU_LAYER = "od_nature:marine_reporting_units_2024" +RBINS_MSP_WFS_URL = "https://spatial.naturalsciences.be/geoserver/ows" +RBINS_MSP_SOURCE_URL = ( + "https://www.health.belgium.be/en/themes/environment/marine-environment/marine-spatial-plan" +) +MSP_VALID_FROM = "2026-03-20T00:00:00Z" +MAX_ARCHIVE_BYTES = 160 * 1024 * 1024 +MAX_EXTRACTED_BYTES = 1024 * 1024 * 1024 +MAX_WFS_RESPONSE_BYTES = 64 * 1024 * 1024 +WFS_PAGE_SIZE = 5000 +GEOJSON_CRS = {"type": "name", "properties": {"name": "EPSG:4326"}} + +ADMIN_LAYERS = { + "belgianterritory": (1, 1), + "belgianmaritimezone": (1, 1), + "region": (3, 3), + "province": (11, 11), + "municipality": (560, 590), +} + +MSP_LAYERS = ( + "imsp26:bmsp_aquaculture_zone", + "imsp26:bmsp_coastal_protection_experiment_zone", + "imsp26:bmsp_commercial_industrial_zone", + "imsp26:bmsp_cultural_heritage", + "imsp26:bmsp_dredging_zone", + "imsp26:bmsp_energy_cables_pipelines_zone", + "imsp26:bmsp_fisheries_zone", + "imsp26:bmsp_conservation_zone", + "imsp26:bmsp_measuring_poles", + "imsp26:bmsp_military_zone", + "imsp26:bmsp_port_expansion_zone", + "imsp26:bmsp_radar_towers", + "imsp26:bmsp_research_recreation_zone", + "imsp26:bmsp_extraction_zone", + "imsp26:bmsp_shipping_ports_zone", +) + +MARINE_REPORTING_IDS = { + "belgian_north_sea": "ANS-BE-MS-1", + "territorial_1_12": "ANS-BE-AA-TEW", + "coastal_0_1": "ANS-BE-AA-CW", + "offshore": "ANS-BE-AA-OFFSHORE", +} + +AREA_NAMES = { + "belgium": "Belgium land", + "flanders": "Flanders", + "wallonia": "Wallonia", + "brussels": "Brussels-Capital Region", + "belgian_north_sea": "Belgian part of the North Sea", + "territorial_sea": "Belgian territorial sea (0-12 nautical miles)", + "exclusive_economic_zone": "Belgian exclusive economic zone beyond territorial sea", + "continental_shelf": "Belgian continental shelf beyond territorial sea", +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Provision the Belgium and Belgian North Sea scope.") + parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL)) + parser.add_argument( + "--output-root", + type=Path, + default=Path(os.environ.get("GEOINTEL_NATIONAL_SCOPE_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)), + ) + parser.add_argument("--request-timeout", type=int, default=300) + parser.add_argument("--import-timeout", type=int, default=3600) + parser.add_argument("--force", action="store_true") + parser.add_argument("--fetch-only", action="store_true") + return parser.parse_args() + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_json_atomic(path: Path, payload: dict[str, Any], *, pretty: bool = False) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(f"{path.suffix}.partial") + temporary.write_text( + json.dumps( + payload, + ensure_ascii=False, + indent=2 if pretty else None, + separators=None if pretty else (",", ":"), + sort_keys=pretty, + ), + encoding="utf-8", + ) + temporary.replace(path) + + +def build_session() -> requests.Session: + retry = Retry( + total=5, + connect=5, + read=5, + status=5, + backoff_factor=1.0, + status_forcelist=(429, 500, 502, 503, 504), + allowed_methods=frozenset({"GET"}), + raise_on_status=True, + ) + session = requests.Session() + adapter = HTTPAdapter(max_retries=retry) + session.mount("https://", adapter) + session.mount("http://", adapter) + session.headers.update({"User-Agent": "GeoIntel-Belgium-North-Sea-Operator/1.0"}) + return session + + +def download_limited(session: requests.Session, url: str, target: Path, timeout: int, max_bytes: int) -> Path: + if url != NGI_ARCHIVE_URL: + raise RuntimeError("Archive URL is not in the fixed operator allowlist") + temporary = target.with_suffix(f"{target.suffix}.partial") + target.parent.mkdir(parents=True, exist_ok=True) + size = 0 + with session.get(url, stream=True, timeout=timeout) as response: + response.raise_for_status() + content_length = int(response.headers.get("content-length") or 0) + if content_length > max_bytes: + raise RuntimeError(f"NGI archive declares {content_length} bytes, above the {max_bytes} byte limit") + with temporary.open("wb") as handle: + for chunk in response.iter_content(chunk_size=1024 * 1024): + if not chunk: + continue + size += len(chunk) + if size > max_bytes: + raise RuntimeError(f"NGI archive exceeded the {max_bytes} byte limit") + handle.write(chunk) + if size == 0: + raise RuntimeError("NGI archive download was empty") + temporary.replace(target) + return target + + +def extract_single_geopackage(archive_path: Path, output_dir: Path) -> Path: + extraction_root = output_dir / "adminvector" + temporary_root = output_dir / "adminvector.partial" + if temporary_root.exists(): + shutil.rmtree(temporary_root) + temporary_root.mkdir(parents=True) + with zipfile.ZipFile(archive_path) as archive: + members = [member for member in archive.infolist() if not member.is_dir()] + gpkg_members = [member for member in members if Path(member.filename).suffix.lower() == ".gpkg"] + if len(gpkg_members) != 1: + raise RuntimeError(f"Expected one GeoPackage in NGI archive, received {len(gpkg_members)}") + member = gpkg_members[0] + member_path = Path(member.filename) + if member_path.is_absolute() or ".." in member_path.parts: + raise RuntimeError("NGI archive contains an unsafe GeoPackage path") + if member.file_size <= 0 or member.file_size > MAX_EXTRACTED_BYTES: + raise RuntimeError("NGI GeoPackage size is empty or above the extraction limit") + target = temporary_root / "adminvector_4326.gpkg" + with archive.open(member) as source, target.open("wb") as destination: + shutil.copyfileobj(source, destination, length=1024 * 1024) + if extraction_root.exists(): + shutil.rmtree(extraction_root) + temporary_root.replace(extraction_root) + return extraction_root / "adminvector_4326.gpkg" + + +def _polygonal(geometry): + if geometry is None or geometry.is_empty: + return None + if not geometry.is_valid: + geometry = make_valid(geometry) + if isinstance(geometry, (Polygon, MultiPolygon)): + return geometry + if isinstance(geometry, GeometryCollection): + parts = [part for part in geometry.geoms if isinstance(part, (Polygon, MultiPolygon))] + merged = unary_union(parts) if parts else None + if isinstance(merged, (Polygon, MultiPolygon)) and not merged.is_empty: + return merged + return None + + +def read_adminvector_layers(gpkg_path: Path) -> dict[str, dict[str, Any]]: + try: + import geopandas + import pandas + import pyogrio + except ImportError as exc: + raise RuntimeError("GeoPandas and Pyogrio are required for the national scope operator") from exc + + available_layers = {str(row[0]) for row in pyogrio.list_layers(gpkg_path)} + missing = sorted(set(ADMIN_LAYERS) - available_layers) + if missing: + raise RuntimeError(f"NGI AdminVector is missing expected layers: {', '.join(missing)}") + + payloads: dict[str, dict[str, Any]] = {} + for layer_name, (minimum, maximum) in ADMIN_LAYERS.items(): + frame = geopandas.read_file(gpkg_path, layer=layer_name) + if frame.crs is None: + raise RuntimeError(f"NGI layer {layer_name} has no CRS") + frame = frame.to_crs(4326) + if not minimum <= len(frame) <= maximum: + raise RuntimeError( + f"NGI layer {layer_name} has {len(frame)} records; expected between {minimum} and {maximum}" + ) + payload = json.loads(frame.to_json(drop_id=False)) + source_max_modification = None + if "modifdate" in frame.columns: + parsed = pandas.to_datetime(frame["modifdate"], errors="coerce", utc=True).dropna() + if not parsed.empty: + source_max_modification = parsed.max().isoformat() + payload.update( + { + "name": f"NGI AdminVector {layer_name}", + "crs": GEOJSON_CRS, + "source_url": NGI_CATALOG_URL, + "attribution": NGI_ATTRIBUTION, + "license": NGI_LICENSE, + "source_max_modification": source_max_modification, + } + ) + for feature in payload["features"]: + properties = feature.setdefault("properties", {}) + properties.update( + { + "source_name": "ngi_adminvector", + "source_layer": layer_name, + "source_feature_id": str(feature.get("id") or properties.get("tgid") or ""), + "authority_level": "authoritative", + "attribution": NGI_ATTRIBUTION, + "source_url": NGI_CATALOG_URL, + } + ) + payloads[layer_name] = payload + return payloads + + +def _response_json_limited(response: requests.Response, max_bytes: int) -> dict[str, Any]: + response.raise_for_status() + content = response.content + if len(content) > max_bytes: + raise RuntimeError(f"WFS response exceeded the {max_bytes} byte limit") + try: + payload = response.json() + except ValueError as exc: + raise RuntimeError("WFS returned non-JSON content") from exc + if payload.get("type") != "FeatureCollection" or not isinstance(payload.get("features"), list): + raise RuntimeError("WFS response is not a GeoJSON FeatureCollection") + return payload + + +def fetch_wfs_layer( + session: requests.Session, + *, + service_url: str, + layer_name: str, + timeout: int, + page_size: int = WFS_PAGE_SIZE, +) -> dict[str, Any]: + if (service_url, layer_name) not in { + (RBINS_MRU_WFS_URL, RBINS_MRU_LAYER), + *((RBINS_MSP_WFS_URL, layer) for layer in MSP_LAYERS), + }: + raise RuntimeError(f"WFS layer is not in the fixed operator allowlist: {layer_name}") + features: list[dict[str, Any]] = [] + source_ids: set[str] = set() + start_index = 0 + matched: int | None = None + while True: + response = session.get( + service_url, + params={ + "service": "WFS", + "version": "2.0.0", + "request": "GetFeature", + "typeNames": layer_name, + "outputFormat": "application/json", + "srsName": "EPSG:4326", + "count": page_size, + "startIndex": start_index, + }, + timeout=timeout, + ) + payload = _response_json_limited(response, MAX_WFS_RESPONSE_BYTES) + page = payload["features"] + if matched is None and str(payload.get("numberMatched", "")).isdigit(): + matched = int(payload["numberMatched"]) + for feature in page: + source_id = str(feature.get("id") or "") + if source_id and source_id in source_ids: + raise RuntimeError(f"WFS layer {layer_name} returned duplicate feature id {source_id}") + if source_id: + source_ids.add(source_id) + features.append(feature) + if not page or len(page) < page_size or (matched is not None and len(features) >= matched): + break + start_index += len(page) + if matched is not None and len(features) != matched: + raise RuntimeError(f"WFS layer {layer_name} returned {len(features)} of {matched} matched features") + return { + "type": "FeatureCollection", + "name": layer_name, + "crs": GEOJSON_CRS, + "features": features, + } + + +def _reporting_id(feature: dict[str, Any]) -> str: + properties = feature.get("properties") or {} + return str( + properties.get("MarineReportingUnitId") + or properties.get("marineReportingUnitId") + or properties.get("localId") + or "" + ) + + +def derive_marine_scope_payload(reporting_units: dict[str, Any]) -> dict[str, Any]: + by_id = {_reporting_id(feature): feature for feature in reporting_units.get("features") or []} + missing = [identifier for identifier in MARINE_REPORTING_IDS.values() if identifier not in by_id] + if missing: + raise RuntimeError(f"Marine reporting units are missing required identifiers: {', '.join(missing)}") + + def geometry(identifier: str): + result = _polygonal(shape(by_id[identifier].get("geometry"))) + if result is None: + raise RuntimeError(f"Marine reporting unit {identifier} has no valid polygon geometry") + return result + + bpns = geometry(MARINE_REPORTING_IDS["belgian_north_sea"]) + territorial = _polygonal( + unary_union( + ( + geometry(MARINE_REPORTING_IDS["territorial_1_12"]), + geometry(MARINE_REPORTING_IDS["coastal_0_1"]), + ) + ) + ) + offshore = geometry(MARINE_REPORTING_IDS["offshore"]) + if territorial is None or not bpns.covers(territorial.representative_point()) or not bpns.covers(offshore.representative_point()): + raise RuntimeError("Derived marine legal scopes do not align with the official BPNS reporting unit") + + definitions = ( + ( + "belgian_north_sea", + AREA_NAMES["belgian_north_sea"], + bpns, + "marine_water_and_seabed_scope", + [MARINE_REPORTING_IDS["belgian_north_sea"]], + ), + ( + "territorial_sea", + AREA_NAMES["territorial_sea"], + territorial, + "territorial_water_column_and_seabed", + [MARINE_REPORTING_IDS["coastal_0_1"], MARINE_REPORTING_IDS["territorial_1_12"]], + ), + ( + "exclusive_economic_zone", + AREA_NAMES["exclusive_economic_zone"], + offshore, + "water_column_rights_beyond_territorial_sea", + [MARINE_REPORTING_IDS["offshore"]], + ), + ( + "continental_shelf", + AREA_NAMES["continental_shelf"], + offshore, + "seabed_and_subsoil_rights_beyond_territorial_sea", + [MARINE_REPORTING_IDS["offshore"]], + ), + ) + features = [] + for zone, name, legal_geometry, legal_domain, source_ids in definitions: + features.append( + { + "type": "Feature", + "id": f"belgian-marine-scope:{zone}", + "geometry": mapping(legal_geometry), + "properties": { + "name": name, + "coverage_zone": zone, + "legal_domain": legal_domain, + "derived_from_reporting_unit_ids": source_ids, + "source_name": "rbins_marine_reporting_units", + "source_layer": RBINS_MRU_LAYER, + "authority_level": "authoritative", + "attribution": "Royal Belgian Institute of Natural Sciences (RBINS), BMDC", + "source_url": RBINS_MRU_METADATA_URL, + "derivation": ( + "Official reporting-unit geometry reused with explicit legal semantics" + if len(source_ids) == 1 + else "Union of official 0-1 nm coastal waters and 1-12 nm territorial waters" + ), + }, + } + ) + return { + "type": "FeatureCollection", + "name": "Belgian marine legal scopes", + "crs": GEOJSON_CRS, + "features": features, + "source_url": RBINS_MRU_METADATA_URL, + } + + +def build_msp_payload(layer_payloads: dict[str, dict[str, Any]]) -> dict[str, Any]: + if set(layer_payloads) != set(MSP_LAYERS): + missing = sorted(set(MSP_LAYERS) - set(layer_payloads)) + raise RuntimeError(f"Marine Spatial Plan payload is incomplete: {', '.join(missing)}") + features: list[dict[str, Any]] = [] + for layer_name in MSP_LAYERS: + short_name = layer_name.split(":", 1)[1] + for index, source_feature in enumerate(layer_payloads[layer_name]["features"]): + feature = dict(source_feature) + properties = dict(feature.get("properties") or {}) + source_id = str(feature.get("id") or f"{short_name}.{index}") + properties.update( + { + "source_name": "rbins_msp_2026", + "source_layer": layer_name, + "source_feature_id": source_id, + "authority_level": "authoritative", + "valid_from": MSP_VALID_FROM, + "valid_to": "2034-12-31T23:59:59Z", + "attribution": "Belgian federal Marine Environment service and RBINS", + "source_url": RBINS_MSP_SOURCE_URL, + } + ) + feature["id"] = f"{short_name}:{source_id}" + feature["properties"] = properties + features.append(feature) + return { + "type": "FeatureCollection", + "name": "Belgian Marine Spatial Plan 2026-2034", + "crs": GEOJSON_CRS, + "features": features, + "source_url": RBINS_MSP_SOURCE_URL, + "valid_from": MSP_VALID_FROM, + "valid_to": "2034-12-31T23:59:59Z", + } + + +def _max_admin_modification(payloads: dict[str, dict[str, Any]], fallback: str) -> str: + values = [ + str(payload["source_max_modification"]) + for payload in payloads.values() + if payload.get("source_max_modification") + ] + value = max(values) if values else fallback + return f"{value[:10]}T00:00:00Z" + + +def prepare_artifacts(args: argparse.Namespace) -> tuple[dict[str, Path], dict[str, Any]]: + output_root = args.output_root + output_root.mkdir(parents=True, exist_ok=True) + archive_path = output_root / "adminvector_4326.zip" + generated_at = utc_now() + with build_session() as session: + if args.force or not archive_path.is_file(): + download_limited(session, NGI_ARCHIVE_URL, archive_path, args.request_timeout, MAX_ARCHIVE_BYTES) + gpkg_path = extract_single_geopackage(archive_path, output_root) + admin_payloads = read_adminvector_layers(gpkg_path) + reporting_units = fetch_wfs_layer( + session, + service_url=RBINS_MRU_WFS_URL, + layer_name=RBINS_MRU_LAYER, + timeout=args.request_timeout, + ) + marine_scopes = derive_marine_scope_payload(reporting_units) + msp_payloads = { + layer: fetch_wfs_layer( + session, + service_url=RBINS_MSP_WFS_URL, + layer_name=layer, + timeout=args.request_timeout, + ) + for layer in MSP_LAYERS + } + msp = build_msp_payload(msp_payloads) + + artifact_payloads = { + "belgium_land_boundary": admin_payloads["belgianterritory"], + "belgium_regions": admin_payloads["region"], + "belgium_provinces": admin_payloads["province"], + "belgium_municipalities": admin_payloads["municipality"], + "ngi_maritime_zone": admin_payloads["belgianmaritimezone"], + "marine_legal_scopes": marine_scopes, + "marine_spatial_plan_2026": msp, + } + artifacts: dict[str, Path] = {} + for key, payload in artifact_payloads.items(): + path = output_root / f"{key}.geojson" + write_json_atomic(path, payload) + artifacts[key] = path + + admin_observed_at = _max_admin_modification(admin_payloads, generated_at) + mru_versions = [ + str((feature.get("properties") or {}).get("beginLife") or "") + for feature in reporting_units["features"] + if (feature.get("properties") or {}).get("beginLife") + ] + marine_observed_at = f"{max(mru_versions)[:10]}T00:00:00Z" if mru_versions else generated_at + manifest = { + "schema_version": 1, + "status": "complete", + "generated_at": generated_at, + "project_name": PROJECT_NAME, + "scope": "belgium-and-belgian-north-sea", + "ngi_archive_url": NGI_ARCHIVE_URL, + "ngi_archive_sha256": sha256_file(archive_path), + "ngi_geopackage_sha256": sha256_file(gpkg_path), + "admin_observed_at": admin_observed_at, + "marine_reporting_units_url": RBINS_MRU_WFS_URL, + "marine_reporting_units_feature_count": len(reporting_units["features"]), + "marine_observed_at": marine_observed_at, + "msp_wfs_url": RBINS_MSP_WFS_URL, + "msp_layer_count": len(MSP_LAYERS), + "msp_feature_count": len(msp["features"]), + "msp_valid_from": MSP_VALID_FROM, + "artifacts": { + key: { + "filename": path.name, + "sha256": sha256_file(path), + "feature_count": len(artifact_payloads[key]["features"]), + } + for key, path in artifacts.items() + }, + } + write_json_atomic(output_root / "manifest.json", manifest, pretty=True) + return artifacts, manifest + + +def response_data(response: requests.Response) -> Any: + try: + payload = response.json() + except ValueError as exc: + raise RuntimeError(f"GeoIntel API returned non-JSON ({response.status_code}): {response.text[:500]}") from exc + if not response.ok: + raise RuntimeError(f"GeoIntel API failed ({response.status_code}): {json.dumps(payload, ensure_ascii=False)[:1000]}") + if not isinstance(payload, dict) or "data" not in payload: + raise RuntimeError("GeoIntel API response does not use the canonical data envelope") + return payload["data"] + + +def list_paginated(session: requests.Session, url: str, timeout: int) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + offset = 0 + while True: + page = response_data(session.get(url, params={"limit": 200, "offset": offset}, timeout=timeout)) + page_items = list(page.get("items") or []) + items.extend(page_items) + total = int(page.get("total") or len(items)) + if not page_items or len(items) >= total: + break + offset += len(page_items) + return items + + +def _feature_geometry(payload: dict[str, Any], *, feature_id: str | None = None) -> dict[str, Any]: + features = payload["features"] + feature = next((item for item in features if str(item.get("id")) == feature_id), None) if feature_id else features[0] + if not feature or not feature.get("geometry"): + raise RuntimeError(f"Expected geometry is missing for {feature_id or payload.get('name')}") + return feature["geometry"] + + +def _canonical_region_name(feature: dict[str, Any]) -> str: + properties = feature.get("properties") or {} + nis_code = str(properties.get("niscode") or "").lstrip("0") + if nis_code == "2000": + return "flanders" + if nis_code == "3000": + return "wallonia" + if nis_code == "4000": + return "brussels" + names = " ".join(str(value) for key, value in properties.items() if "name" in key.lower() or "naam" in key.lower()) + normalized = names.casefold() + if "vlaams" in normalized or "flam" in normalized: + return "flanders" + if "wallon" in normalized: + return "wallonia" + if "brux" in normalized or "brussel" in normalized: + return "brussels" + raise RuntimeError(f"Cannot map NGI region feature {feature.get('id')} to a canonical region") + + +def _upload_dataset( + session: requests.Session, + *, + base_url: str, + project_id: str, + area_id: str, + path: Path, + source_name: str, + reference_layer_name: str, + coverage_zones: list[str], + observed_at: str, + valid_from: str, + valid_to: str | None, + source_version: str, + artifact_sha256: str, + source_url: str, + attribution: str, + license_note: str, + timeout: int, +) -> dict[str, Any]: + source_metadata = { + "provider": source_name, + "authority_level": "authoritative", + "coverage_zones": coverage_zones, + "reference_layer_name": reference_layer_name, + "source_url": source_url, + "attribution": attribution, + "license_note": license_note, + } + provenance = { + "operator_tool": "provision_belgium_north_sea_scope.py", + "operator_explicit_fetch": True, + "artifact_sha256": artifact_sha256, + "source_url": source_url, + "direct_vector_feature_write": False, + } + data = { + "dataset_type": "vector", + "source": "operator_official_import", + "dataset_role": "reference", + "source_name": source_name, + "reference_layer_name": reference_layer_name, + "source_metadata_json": json.dumps(source_metadata, ensure_ascii=False), + "provenance_metadata_json": json.dumps(provenance, ensure_ascii=False), + "area_id": area_id, + "temporal_series_key": f"{source_name}:{reference_layer_name}", + "observed_at": observed_at, + "valid_from": valid_from, + "temporal_granularity": "period" if valid_to else "snapshot", + "source_version": source_version, + } + if valid_to: + data["valid_to"] = valid_to + with path.open("rb") as handle: + response = session.post( + f"{base_url}/api/v1/projects/{project_id}/datasets/upload", + data=data, + files={"file": (path.name, handle, "application/geo+json")}, + timeout=timeout, + ) + return response_data(response) + + +def provision(args: argparse.Namespace, artifacts: dict[str, Path], manifest: dict[str, Any]) -> dict[str, Any]: + base_url = args.base_url.rstrip("/") + payloads = {key: json.loads(path.read_text(encoding="utf-8")) for key, path in artifacts.items()} + with requests.Session() as session: + projects = list_paginated(session, f"{base_url}/api/v1/projects", args.import_timeout) + project = next((item for item in projects if item.get("name") == PROJECT_NAME), None) + if project is None: + project = response_data( + session.post( + f"{base_url}/api/v1/projects", + json={ + "name": PROJECT_NAME, + "description": ( + "Authoritative national and maritime scope for Belgium, its territorial sea, " + "exclusive economic zone and continental shelf." + ), + "region": PROJECT_REGION, + }, + timeout=args.import_timeout, + ) + ) + project_id = str(project["id"]) + existing_areas = list_paginated( + session, + f"{base_url}/api/v1/projects/{project_id}/areas", + args.import_timeout, + ) + areas_by_name = {str(area.get("name")): area for area in existing_areas} + + def ensure_area(name: str, geometry: dict[str, Any]) -> dict[str, Any]: + if name in areas_by_name: + existing = areas_by_name[name] + existing_geometry = existing.get("geometry") + if not existing_geometry or not shape(existing_geometry).equals(shape(geometry)): + raise RuntimeError( + f"Persisted Area {name!r} differs from the current authoritative geometry; " + "provision a fresh versioned national project instead of silently mutating it" + ) + return existing + area = response_data( + session.post( + f"{base_url}/api/v1/projects/{project_id}/areas", + json={"name": name, "crs": "EPSG:4326", "geometry": geometry}, + timeout=args.import_timeout, + ) + ) + areas_by_name[name] = area + return area + + land_area = ensure_area(AREA_NAMES["belgium"], _feature_geometry(payloads["belgium_land_boundary"])) + region_features = {_canonical_region_name(feature): feature for feature in payloads["belgium_regions"]["features"]} + for zone in ("flanders", "wallonia", "brussels"): + ensure_area(AREA_NAMES[zone], region_features[zone]["geometry"]) + marine_features = { + str((feature.get("properties") or {}).get("coverage_zone")): feature + for feature in payloads["marine_legal_scopes"]["features"] + } + for zone in ("belgian_north_sea", "territorial_sea", "exclusive_economic_zone", "continental_shelf"): + ensure_area(AREA_NAMES[zone], marine_features[zone]["geometry"]) + + datasets = list_paginated( + session, + f"{base_url}/api/v1/projects/{project_id}/datasets", + args.import_timeout, + ) + specs = ( + ( + "belgium_land_boundary", + "ngi_adminvector", + "belgium_land_boundary", + ["belgium"], + str(land_area["id"]), + manifest["admin_observed_at"], + manifest["admin_observed_at"], + None, + manifest["admin_observed_at"][:10], + NGI_CATALOG_URL, + NGI_ATTRIBUTION, + NGI_LICENSE, + ), + ( + "belgium_regions", + "ngi_adminvector", + "belgium_regions", + ["belgium", "flanders", "wallonia", "brussels"], + str(land_area["id"]), + manifest["admin_observed_at"], + manifest["admin_observed_at"], + None, + manifest["admin_observed_at"][:10], + NGI_CATALOG_URL, + NGI_ATTRIBUTION, + NGI_LICENSE, + ), + ( + "belgium_provinces", + "ngi_adminvector", + "belgium_provinces", + ["belgium", "flanders", "wallonia"], + str(land_area["id"]), + manifest["admin_observed_at"], + manifest["admin_observed_at"], + None, + manifest["admin_observed_at"][:10], + NGI_CATALOG_URL, + NGI_ATTRIBUTION, + NGI_LICENSE, + ), + ( + "belgium_municipalities", + "ngi_adminvector", + "belgium_municipalities", + ["belgium", "flanders", "wallonia", "brussels"], + str(land_area["id"]), + manifest["admin_observed_at"], + manifest["admin_observed_at"], + None, + manifest["admin_observed_at"][:10], + NGI_CATALOG_URL, + NGI_ATTRIBUTION, + NGI_LICENSE, + ), + ( + "marine_legal_scopes", + "rbins_marine_reporting_units", + "marine_legal_scopes", + [ + "belgian_north_sea", + "territorial_sea", + "exclusive_economic_zone", + "continental_shelf", + ], + str(areas_by_name[AREA_NAMES["belgian_north_sea"]]["id"]), + manifest["marine_observed_at"], + manifest["marine_observed_at"], + None, + manifest["marine_observed_at"][:10], + RBINS_MRU_METADATA_URL, + "Royal Belgian Institute of Natural Sciences (RBINS), BMDC", + "See source metadata", + ), + ( + "marine_spatial_plan_2026", + "rbins_msp_2026", + "marine_spatial_plan_2026", + [ + "belgian_north_sea", + "territorial_sea", + "exclusive_economic_zone", + "continental_shelf", + ], + str(areas_by_name[AREA_NAMES["belgian_north_sea"]]["id"]), + MSP_VALID_FROM, + MSP_VALID_FROM, + "2034-12-31T23:59:59Z", + "2026-2034", + RBINS_MSP_SOURCE_URL, + "Belgian federal Marine Environment service and RBINS", + "See source metadata", + ), + ) + persisted = [] + for ( + artifact_key, + source_name, + layer_name, + coverage_zones, + area_id, + observed_at, + valid_from, + valid_to, + source_version, + source_url, + attribution, + license_note, + ) in specs: + checksum = manifest["artifacts"][artifact_key]["sha256"] + existing = next( + ( + dataset + for dataset in datasets + if dataset.get("source_name") == source_name + and dataset.get("reference_layer_name") == layer_name + and dataset.get("source_version") == source_version + ), + None, + ) + if existing: + persisted_checksum = (existing.get("provenance_metadata") or {}).get("artifact_sha256") + if persisted_checksum != checksum: + raise RuntimeError( + f"Existing immutable dataset {layer_name} has checksum {persisted_checksum}, expected {checksum}" + ) + persisted.append(existing) + continue + created = _upload_dataset( + session, + base_url=base_url, + project_id=project_id, + area_id=area_id, + path=artifacts[artifact_key], + source_name=source_name, + reference_layer_name=layer_name, + coverage_zones=coverage_zones, + observed_at=observed_at, + valid_from=valid_from, + valid_to=valid_to, + source_version=source_version, + artifact_sha256=checksum, + source_url=source_url, + attribution=attribution, + license_note=license_note, + timeout=args.import_timeout, + ) + persisted.append(created) + + return { + "project_id": project_id, + "project_name": PROJECT_NAME, + "area_count": len(AREA_NAMES), + "dataset_ids": [str(dataset["id"]) for dataset in persisted], + } + + +def main() -> int: + args = parse_args() + try: + artifacts, manifest = prepare_artifacts(args) + workspace = None if args.fetch_only else provision(args, artifacts, manifest) + except (OSError, RuntimeError, ValueError, KeyError, requests.RequestException, zipfile.BadZipFile) as exc: + print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr) + return 1 + print( + json.dumps( + { + "status": "ok", + "mode": "fetch_only" if args.fetch_only else "provisioned", + "scope": manifest["scope"], + "manifest_path": str(args.output_root / "manifest.json"), + "artifact_count": len(artifacts), + "msp_feature_count": manifest["msp_feature_count"], + "workspace": workspace, + }, + ensure_ascii=False, + indent=2, + ) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 10f22ffc..50f1fcc1 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -78,6 +78,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_regional_soil_map.py ${PYTHON_BIN} -m py_compile scripts/provision_regional_timeseries.py ${PYTHON_BIN} -m py_compile scripts/geographic_scopes.py ${PYTHON_BIN} -m py_compile scripts/provision_geographic_scope.py +${PYTHON_BIN} -m py_compile scripts/provision_belgium_north_sea_scope.py ${PYTHON_BIN} -m py_compile scripts/provision_regional_grb_buildings.py ${PYTHON_BIN} -m py_compile scripts/provision_regional_grb_context.py ${PYTHON_BIN} -m py_compile scripts/audit_source_freshness.py