Add Flemish bathymetry partition workflow
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-17 16:15:29 +02:00
parent 8ef42a23e5
commit 132c4feed8
34 changed files with 1862 additions and 27 deletions
+4
View File
@@ -43,6 +43,10 @@ BATHYMETRY_PROFILES_PAGE_SIZE=1000
BATHYMETRY_PROFILES_MAX_FEATURES=50000 BATHYMETRY_PROFILES_MAX_FEATURES=50000
BATHYMETRY_PROFILES_TIMEOUT_SECONDS=120 BATHYMETRY_PROFILES_TIMEOUT_SECONDS=120
BATHYMETRY_PROFILES_MAX_RESPONSE_MB=32 BATHYMETRY_PROFILES_MAX_RESPONSE_MB=32
MDK_BATHYMETRY_PROBE_ENABLED=true
MDK_BATHYMETRY_WCS_URL=https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs
MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS=20
MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB=4
THEMATIC_RASTER_ENABLED=true THEMATIC_RASTER_ENABLED=true
THEMATIC_RASTER_WCS_URL=https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs THEMATIC_RASTER_WCS_URL=https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs
THEMATIC_RASTER_MIN_SIDE_M=100 THEMATIC_RASTER_MIN_SIDE_M=100
+11
View File
@@ -7,6 +7,17 @@
# Changelog # Changelog
## Sprint 236 Flemish bathymetry partitions and safe North Sea probe (2026-07-17)
- Added dynamic provisioning of the complete official VRBG Flemish
municipality inventory, validated at 285 current Areas.
- Added an atomic, resumable VHA profile coordinator and server-side manifest
finalization that prevents partial partitions from appearing regional.
- Added a strict-TLS, size-bounded MDK WCS GetCapabilities readiness probe
without coverage download or insecure fallback.
- Exposed truthful MDK readiness and bathymetry partition-finalization API
contracts and packaged all operator scripts in the all-in-one image.
## Sprint 235 Governed bathymetry profiles and Belgian scale architecture (2026-07-17) ## Sprint 235 Governed bathymetry profiles and Belgian scale architecture (2026-07-17)
- Added bounded official VHA cross-section acquisition with exact Area - Added bounded official VHA cross-section acquisition with exact Area
+26
View File
@@ -1728,3 +1728,29 @@ The Dataset exposes profile count and nullable structured depth/width metrics.
It does not claim a continuous bed model, current depth or volume. Use It does not claim a continuous bed model, current depth or volume. Use
`scripts/provision_mol_bathymetry_profiles.py` for the canonical Mol operator `scripts/provision_mol_bathymetry_profiles.py` for the canonical Mol operator
flow. flow.
Provision the complete current Flemish land scope and then run the resumable
VHA municipality coordinator:
```bash
docker exec geointel python /app/scripts/provision_flanders_geographic_scope.py
docker exec geointel python /app/scripts/provision_flanders_bathymetry_profiles.py
```
The first command discovers all current VRBG RefGem municipalities, validates
a 270..300 safety range and persists their exact Areas. The second writes its
manifest after every partition. Repeating it reuses completed source
identities; `--force` deliberately refreshes them. A partial `--members` or
`--max-partitions` run never marks regional coverage complete.
Inspect the MDK North Sea WCS without downloading coverage:
```bash
docker exec geointel python /app/scripts/probe_mdk_bathymetry.py
```
Exit code `0` means verified capabilities; `2` means a truthful blocked
readiness state such as TLS or endpoint failure. Runtime controls are
`MDK_BATHYMETRY_PROBE_ENABLED`, `MDK_BATHYMETRY_WCS_URL`,
`MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS` and
`MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB`. TLS verification cannot be disabled.
+18
View File
@@ -27,6 +27,7 @@ from app.schemas import (
FloodHazardAcquireRequest, FloodHazardAcquireRequest,
FloodHazardPartitionSelectionRequest, FloodHazardPartitionSelectionRequest,
FloodHazardSelectionRequest, FloodHazardSelectionRequest,
BathymetryPartitionFinalizeRequest,
BathymetryProfileAcquireRequest, BathymetryProfileAcquireRequest,
ThematicRasterAcquireRequest, ThematicRasterAcquireRequest,
ThematicRasterSelectionRequest, ThematicRasterSelectionRequest,
@@ -54,6 +55,7 @@ from app.services.terrain_analysis_service import TerrainAnalysisService
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
from app.utils.response import envelope from app.utils.response import envelope
@@ -223,6 +225,13 @@ def list_bathymetry_sources(project_id: UUID, db: Session = Depends(get_db)):
return envelope({"items": items, "total": len(items)}) return envelope({"items": items, "total": len(items)})
@router.get("/datasets/bathymetry/sources/mdk_bcp_bathymetry/readiness", response_model=dict)
def probe_mdk_bathymetry_readiness(project_id: UUID, db: Session = Depends(get_db)):
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
return envelope(MdkBathymetryProbeService.probe())
@router.post("/datasets/bathymetry/profiles/acquire", response_model=dict) @router.post("/datasets/bathymetry/profiles/acquire", response_model=dict)
def acquire_bounded_bathymetry_profiles( def acquire_bounded_bathymetry_profiles(
project_id: UUID, project_id: UUID,
@@ -239,6 +248,15 @@ def acquire_bounded_bathymetry_profiles(
return envelope(job) return envelope(job)
@router.post("/datasets/bathymetry/profiles/partitions/finalize", response_model=dict)
def finalize_bathymetry_profile_partitions(
project_id: UUID,
payload: BathymetryPartitionFinalizeRequest,
db: Session = Depends(get_db),
):
return envelope(BathymetryProfileAcquisitionService.finalize_partitions(db, project_id, payload))
@router.post("/datasets/thematic-raster/acquire", response_model=dict) @router.post("/datasets/thematic-raster/acquire", response_model=dict)
def acquire_bounded_thematic_raster( def acquire_bounded_thematic_raster(
project_id: UUID, project_id: UUID,
+17
View File
@@ -123,6 +123,23 @@ class Settings(BaseSettings):
le=256, le=256,
validation_alias="BATHYMETRY_PROFILES_MAX_RESPONSE_MB", validation_alias="BATHYMETRY_PROFILES_MAX_RESPONSE_MB",
) )
mdk_bathymetry_probe_enabled: bool = Field(default=True, validation_alias="MDK_BATHYMETRY_PROBE_ENABLED")
mdk_bathymetry_wcs_url: str = Field(
default="https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs",
validation_alias="MDK_BATHYMETRY_WCS_URL",
)
mdk_bathymetry_probe_timeout_seconds: int = Field(
default=20,
ge=1,
le=120,
validation_alias="MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS",
)
mdk_bathymetry_probe_max_response_mb: int = Field(
default=4,
ge=1,
le=16,
validation_alias="MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB",
)
thematic_raster_enabled: bool = Field(default=True, validation_alias="THEMATIC_RASTER_ENABLED") thematic_raster_enabled: bool = Field(default=True, validation_alias="THEMATIC_RASTER_ENABLED")
thematic_raster_wcs_url: str = Field( thematic_raster_wcs_url: str = Field(
default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs", default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs",
+6
View File
@@ -66,8 +66,11 @@ from .flood_hazard import (
FloodHazardSelectionSummary, FloodHazardSelectionSummary,
) )
from .bathymetry import ( from .bathymetry import (
BathymetryPartitionFinalizeRequest,
BathymetryPartitionFinalizationResult,
BathymetryProfileAcquireRequest, BathymetryProfileAcquireRequest,
BathymetryProfileAcquisitionResult, BathymetryProfileAcquisitionResult,
BathymetrySourceProbeRead,
BathymetrySourceRead, BathymetrySourceRead,
) )
from .thematic_raster import ( from .thematic_raster import (
@@ -209,6 +212,9 @@ __all__ = [
"FloodHazardSelectionSummary", "FloodHazardSelectionSummary",
"BathymetryProfileAcquireRequest", "BathymetryProfileAcquireRequest",
"BathymetryProfileAcquisitionResult", "BathymetryProfileAcquisitionResult",
"BathymetryPartitionFinalizeRequest",
"BathymetryPartitionFinalizationResult",
"BathymetrySourceProbeRead",
"BathymetrySourceRead", "BathymetrySourceRead",
"ThematicRasterAcquireRequest", "ThematicRasterAcquireRequest",
"ThematicRasterAcquisitionResult", "ThematicRasterAcquisitionResult",
+61 -2
View File
@@ -1,9 +1,10 @@
from __future__ import annotations from __future__ import annotations
from datetime import datetime
from typing import Literal from typing import Literal
from uuid import UUID from uuid import UUID
from pydantic import BaseModel, Field from pydantic import BaseModel, Field, field_validator
from .operations import VectorSelectionBBox from .operations import VectorSelectionBBox
@@ -25,7 +26,7 @@ class BathymetrySourceRead(BaseModel):
vertical_reference: str vertical_reference: str
horizontal_crs: str horizontal_crs: str
native_resolution: str | None = None native_resolution: str | None = None
integration_status: Literal["operational", "available_not_integrated", "catalog_only"] integration_status: Literal["operational", "probe_only", "available_not_integrated", "catalog_only"]
acquisition_supported: bool acquisition_supported: bool
configured: bool configured: bool
service_url: str | None = None service_url: str | None = None
@@ -50,3 +51,61 @@ class BathymetryProfileAcquisitionResult(BaseModel):
measurement_date_max: str | None = None measurement_date_max: str | None = None
attribution: str attribution: str
limitation_message: str limitation_message: str
class BathymetryPartitionFinalizeRequest(BaseModel):
partition_scope_key: str = Field(min_length=1, max_length=120, pattern=r"^[a-z0-9][a-z0-9_-]*$")
expected_area_ids: list[UUID] = Field(min_length=1, max_length=500)
dataset_ids: list[UUID] = Field(default_factory=list, max_length=500)
no_profile_area_ids: list[UUID] = Field(default_factory=list, max_length=500)
manifest_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
observed_at: datetime
@field_validator("expected_area_ids", "dataset_ids", "no_profile_area_ids")
@classmethod
def require_unique_ids(cls, value: list[UUID]) -> list[UUID]:
if len(value) != len(set(value)):
raise ValueError("Partition identifiers must be unique")
return value
class BathymetryPartitionFinalizationResult(BaseModel):
partition_scope_key: str
regional_partitions_complete: bool
partition_count: int = Field(ge=1)
data_partition_count: int = Field(ge=0)
no_profile_partition_count: int = Field(ge=0)
profile_count: int = Field(ge=0)
document_count: int = Field(ge=0)
structured_depth_count: int = Field(ge=0)
measurement_date_min: str | None = None
measurement_date_max: str | None = None
dataset_ids: list[UUID]
manifest_sha256: str
observed_at: datetime
limitation_message: str
class BathymetrySourceProbeRead(BaseModel):
source_key: str
status: Literal[
"disabled",
"invalid_configuration",
"tls_error",
"endpoint_unavailable",
"invalid_capabilities",
"reachable",
]
configured_url: str
capabilities_url: str | None = None
tls_verified: bool
capabilities_reachable: bool
acquisition_supported: bool = False
wcs_version: str | None = None
coverage_identifiers: list[str] = Field(default_factory=list)
advertised_formats: list[str] = Field(default_factory=list)
advertised_crs: list[str] = Field(default_factory=list)
response_sha256: str | None = None
checked_at: datetime
message: str
limitation_message: str
@@ -15,8 +15,10 @@ from shapely.geometry import Point, box, mapping
from app.core.config import Settings, get_settings from app.core.config import Settings, get_settings
from app.core.errors import AppError from app.core.errors import AppError
from app.models import Area, Dataset, Project from app.models import Area, Dataset, DatasetVersion, Project
from app.schemas.bathymetry import ( from app.schemas.bathymetry import (
BathymetryPartitionFinalizeRequest,
BathymetryPartitionFinalizationResult,
BathymetryProfileAcquireRequest, BathymetryProfileAcquireRequest,
BathymetryProfileAcquisitionResult, BathymetryProfileAcquisitionResult,
BathymetrySourceRead, BathymetrySourceRead,
@@ -70,16 +72,16 @@ class BathymetryProfileAcquisitionService:
"vertical_reference": "LAT", "vertical_reference": "LAT",
"horizontal_crs": "bronafhankelijk; expliciet per WCS-respons", "horizontal_crs": "bronafhankelijk; expliciet per WCS-respons",
"native_resolution": "20 x 20 m", "native_resolution": "20 x 20 m",
"integration_status": "available_not_integrated", "integration_status": "probe_only",
"acquisition_supported": False, "acquisition_supported": False,
"configured": False, "configured": False,
"service_url": "https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/WCS_Public", "service_url": "https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs",
"catalog_url": "https://www.vlaanderen.be/datavindplaats/catalogus/dieptemodel-van-de-zeebodem-belgisch-continentaal-plat-noordzee", "catalog_url": "https://www.vlaanderen.be/datavindplaats/catalogus/dieptemodel-van-de-zeebodem-belgisch-continentaal-plat-noordzee",
"attribution": "Agentschap Maritieme Dienstverlening en Kust", "attribution": "Agentschap Maritieme Dienstverlening en Kust",
"license_note": "Zie de officiële datasetmetadata en gebruiksvoorwaarden.", "license_note": "Zie de officiële datasetmetadata en gebruiksvoorwaarden.",
"limitation_message": ( "limitation_message": (
"Niet operationeel in GeoIntel totdat WCS, maritieme begrenzing, tegels, LAT-semantiek en " "Alleen een read-only GetCapabilities-probe is beschikbaar. Rasteracquisitie blijft uit totdat "
"servercertificaten in een live integratietest zijn gevalideerd." "WCS, maritieme begrenzing, tegels, LAT-semantiek en servercertificaten live zijn gevalideerd."
), ),
}, },
{ {
@@ -404,6 +406,8 @@ class BathymetryProfileAcquisitionService:
raw_features: list[dict[str, Any]], raw_features: list[dict[str, Any]],
scope_geometry, scope_geometry,
watercourse_names: dict[int, dict[str, str | None]], watercourse_names: dict[int, dict[str, str | None]],
*,
partition_properties: dict[str, Any] | None = None,
) -> tuple[dict[str, Any], dict[str, Any]]: ) -> tuple[dict[str, Any], dict[str, Any]]:
normalized: list[dict[str, Any]] = [] normalized: list[dict[str, Any]] = []
dates: list[str] = [] dates: list[str] = []
@@ -465,6 +469,7 @@ class BathymetryProfileAcquisitionService:
"provider": BathymetryProfileAcquisitionService.PROVIDER, "provider": BathymetryProfileAcquisitionService.PROVIDER,
"measurement_semantics": "historical_cross_section_profile_point", "measurement_semantics": "historical_cross_section_profile_point",
"vertical_reference": "document-specific", "vertical_reference": "document-specific",
**(partition_properties or {}),
}, },
"geometry": mapping(point), "geometry": mapping(point),
} }
@@ -487,6 +492,17 @@ class BathymetryProfileAcquisitionService:
}, },
) )
@staticmethod
def _municipality_name(area: Area | None) -> str | None:
if area is None:
return None
normalized = str(area.name or "").strip()
prefix = "Gemeente "
if not normalized.casefold().startswith(prefix.casefold()):
return None
municipality = normalized[len(prefix) :].split(" - ", 1)[0].strip()
return municipality or None
@staticmethod @staticmethod
def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None: def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None:
return ( return (
@@ -541,6 +557,8 @@ class BathymetryProfileAcquisitionService:
scope_geometry = BathymetryProfileAcquisitionService._scope_geometry( scope_geometry = BathymetryProfileAcquisitionService._scope_geometry(
db, project_id, payload.area_id, bbox_values db, project_id, payload.area_id, bbox_values
) )
area = db.get(Area, payload.area_id) if payload.area_id else None
municipality = BathymetryProfileAcquisitionService._municipality_name(area)
exact_bbox = tuple(float(value) for value in scope_geometry.bounds) exact_bbox = tuple(float(value) for value in scope_geometry.bounds)
request_identity = { request_identity = {
"provider": BathymetryProfileAcquisitionService.PROVIDER, "provider": BathymetryProfileAcquisitionService.PROVIDER,
@@ -568,7 +586,16 @@ class BathymetryProfileAcquisitionService:
vhag_codes, resolved_settings, opener vhag_codes, resolved_settings, opener
) )
feature_collection, summary = BathymetryProfileAcquisitionService._normalize_features( feature_collection, summary = BathymetryProfileAcquisitionService._normalize_features(
raw_features, scope_geometry, names raw_features,
scope_geometry,
names,
partition_properties={
"partition_area_id": str(area.id),
"partition_area_name": area.name,
**({"municipality": municipality} if municipality else {}),
}
if area
else None,
) )
if summary["profile_count"] == 0: if summary["profile_count"] == 0:
raise AppError( raise AppError(
@@ -585,6 +612,11 @@ class BathymetryProfileAcquisitionService:
"theme": "bathymetry", "theme": "bathymetry",
"layer_name": "bathymetry_profiles", "layer_name": "bathymetry_profiles",
"coverage_scope": "municipality" if payload.area_id else "bounded_selection", "coverage_scope": "municipality" if payload.area_id else "bounded_selection",
"partition_area_id": str(area.id) if area else None,
"partition_area_name": area.name if area else None,
"municipality": municipality,
"partitioned_source_audit": False,
"regional_partitions_complete": False,
"bbox_epsg4326": list(exact_bbox), "bbox_epsg4326": list(exact_bbox),
**summary, **summary,
"selection_aggregation": { "selection_aggregation": {
@@ -669,3 +701,160 @@ class BathymetryProfileAcquisitionService:
) )
persisted = db.get(Dataset, dataset.id) persisted = db.get(Dataset, dataset.id)
return BathymetryProfileAcquisitionService._result(persisted, reused=False) return BathymetryProfileAcquisitionService._result(persisted, reused=False)
@staticmethod
def finalize_partitions(
db,
project_id: UUID,
payload: BathymetryPartitionFinalizeRequest,
) -> dict[str, Any]:
if not db.get(Project, project_id):
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
expected_area_ids = set(payload.expected_area_ids)
dataset_ids = set(payload.dataset_ids)
no_profile_area_ids = set(payload.no_profile_area_ids)
areas: dict[UUID, Area] = {}
for area_id in expected_area_ids:
area = db.get(Area, area_id)
if area is None or area.project_id != project_id:
raise AppError(
code="BATHYMETRY_PARTITION_AREA_INVALID",
message="Every expected partition Area must belong to the project",
details={"area_id": str(area_id)},
status_code=400,
)
if BathymetryProfileAcquisitionService._municipality_name(area) is None:
raise AppError(
code="BATHYMETRY_PARTITION_AREA_INVALID",
message="Bathymetry partitions must use persisted municipality Areas",
details={"area_id": str(area_id), "area_name": area.name},
status_code=400,
)
areas[area_id] = area
if not no_profile_area_ids.issubset(expected_area_ids):
raise AppError(
code="BATHYMETRY_PARTITION_MANIFEST_INVALID",
message="No-profile partitions must be part of the expected Area set",
status_code=400,
)
datasets: list[Dataset] = []
data_area_ids: set[UUID] = set()
for dataset_id in payload.dataset_ids:
dataset = db.get(Dataset, dataset_id)
if (
dataset is None
or dataset.project_id != project_id
or dataset.source_name != BathymetryProfileAcquisitionService.PROVIDER
or dataset.status != "ready"
or dataset.area_id not in expected_area_ids
):
raise AppError(
code="BATHYMETRY_PARTITION_DATASET_INVALID",
message="Every partition Dataset must be a ready VHA profile Dataset scoped to an expected Area",
details={"dataset_id": str(dataset_id)},
status_code=400,
)
if dataset.area_id in data_area_ids:
raise AppError(
code="BATHYMETRY_PARTITION_DATASET_DUPLICATE",
message="A complete manifest may reference only one profile Dataset per Area",
details={"area_id": str(dataset.area_id)},
status_code=400,
)
data_area_ids.add(dataset.area_id)
datasets.append(dataset)
accounted_area_ids = data_area_ids.union(no_profile_area_ids)
if accounted_area_ids != expected_area_ids:
raise AppError(
code="BATHYMETRY_PARTITION_MANIFEST_INCOMPLETE",
message="Every expected Area must have one ready Dataset or an explicit no-profile result",
details={
"missing_area_ids": sorted(str(value) for value in expected_area_ids - accounted_area_ids),
"unexpected_area_ids": sorted(str(value) for value in accounted_area_ids - expected_area_ids),
},
status_code=400,
)
profile_count = 0
document_count = 0
structured_depth_count = 0
dates_min: list[str] = []
dates_max: list[str] = []
shared_metadata = {
"partition_scope_key": payload.partition_scope_key,
"partition_count": len(expected_area_ids),
"data_partition_count": len(datasets),
"no_profile_partition_count": len(no_profile_area_ids),
"partition_manifest_sha256": payload.manifest_sha256,
"partition_manifest_observed_at": payload.observed_at.isoformat(),
"partitioned_source_audit": True,
"regional_partitions_complete": True,
}
shared_provenance = {
"partition_manifest_sha256": payload.manifest_sha256,
"partition_manifest_observed_at": payload.observed_at.isoformat(),
"partition_scope_key": payload.partition_scope_key,
"regional_partitions_complete": True,
"no_profile_area_ids": sorted(str(value) for value in no_profile_area_ids),
}
for dataset in datasets:
source_metadata = dict(dataset.source_metadata or {})
provenance_metadata = dict(dataset.provenance_metadata or {})
profile_count += int(source_metadata.get("profile_count") or 0)
document_count += int(source_metadata.get("document_count") or 0)
structured_depth_count += int(source_metadata.get("structured_depth_count") or 0)
if source_metadata.get("measurement_date_min"):
dates_min.append(str(source_metadata["measurement_date_min"]))
if source_metadata.get("measurement_date_max"):
dates_max.append(str(source_metadata["measurement_date_max"]))
area = areas[dataset.area_id]
source_metadata.update(
{
**shared_metadata,
"coverage_scope": payload.partition_scope_key,
"partition_area_id": str(area.id),
"partition_area_name": area.name,
"municipality": BathymetryProfileAcquisitionService._municipality_name(area),
}
)
provenance_metadata.update(shared_provenance)
dataset.source_metadata = source_metadata
dataset.provenance_metadata = provenance_metadata
if dataset_ids:
versions = (
db.query(DatasetVersion)
.filter(DatasetVersion.dataset_id.in_(dataset_ids))
.all()
)
for version in versions:
version.source_metadata = dict(
next(dataset.source_metadata for dataset in datasets if dataset.id == version.dataset_id)
)
version.provenance_metadata = dict(
next(dataset.provenance_metadata for dataset in datasets if dataset.id == version.dataset_id)
)
db.commit()
return BathymetryPartitionFinalizationResult(
partition_scope_key=payload.partition_scope_key,
regional_partitions_complete=True,
partition_count=len(expected_area_ids),
data_partition_count=len(datasets),
no_profile_partition_count=len(no_profile_area_ids),
profile_count=profile_count,
document_count=document_count,
structured_depth_count=structured_depth_count,
measurement_date_min=min(dates_min) if dates_min else None,
measurement_date_max=max(dates_max) if dates_max else None,
dataset_ids=payload.dataset_ids,
manifest_sha256=payload.manifest_sha256,
observed_at=payload.observed_at,
limitation_message=BathymetryProfileAcquisitionService.LIMITATION,
).model_dump(mode="json")
@@ -0,0 +1,253 @@
from __future__ import annotations
from datetime import UTC, datetime
import hashlib
import ssl
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
from urllib.request import Request, urlopen
from xml.etree import ElementTree
from app.core.config import Settings, get_settings
from app.schemas.bathymetry import BathymetrySourceProbeRead
class MdkBathymetryProbeService:
SOURCE_KEY = "mdk_bcp_bathymetry"
LIMITATION = (
"Deze probe leest alleen WCS GetCapabilities met strikte TLS-controle. "
"GeoIntel downloadt of activeert geen Noordzee-raster totdat endpoint, coverage-id, CRS, LAT, "
"nodata, resolutie, begrenzing en responslimieten live zijn gevalideerd."
)
@staticmethod
def _capabilities_url(configured_url: str) -> str:
parsed = urlsplit(configured_url.strip())
if parsed.scheme.lower() != "https" or not parsed.hostname:
raise ValueError("The MDK WCS probe requires an absolute HTTPS URL")
parameters = dict(parse_qsl(parsed.query, keep_blank_values=True))
parameters.update({"service": "WCS", "request": "GetCapabilities", "version": "1.0.0"})
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(parameters), ""))
@staticmethod
def _read_capabilities(
capabilities_url: str,
settings: Settings,
opener: Callable[..., Any] | None,
) -> tuple[bytes, str]:
request = Request(
capabilities_url,
headers={
"Accept": "application/xml,text/xml;q=0.9,*/*;q=0.1",
"User-Agent": "GeoIntel/1.0 MDK-bathymetry-readiness-probe",
},
)
with (opener or urlopen)(request, timeout=settings.mdk_bathymetry_probe_timeout_seconds) as response:
limit = settings.mdk_bathymetry_probe_max_response_mb * 1024 * 1024
content = response.read(limit + 1)
if len(content) > limit:
raise ValueError("MDK WCS capabilities response exceeded the configured size limit")
content_type = str(response.headers.get("Content-Type") or "") if hasattr(response, "headers") else ""
return content, content_type
@staticmethod
def _local_name(tag: str) -> str:
return tag.rsplit("}", 1)[-1].casefold()
@staticmethod
def _parse_capabilities(content: bytes) -> dict[str, Any]:
root = ElementTree.fromstring(content)
root_name = MdkBathymetryProbeService._local_name(root.tag)
if "capabilities" not in root_name:
raise ValueError("MDK endpoint did not return a WCS capabilities document")
coverage_identifiers: set[str] = set()
advertised_formats: set[str] = set()
advertised_crs: set[str] = set()
for element in root.iter():
local_name = MdkBathymetryProbeService._local_name(element.tag)
text = (element.text or "").strip()
if local_name in {"coverageofferingbrief", "coverageoffering"}:
for child in element:
if MdkBathymetryProbeService._local_name(child.tag) in {"name", "identifier"}:
identifier = (child.text or "").strip()
if identifier:
coverage_identifiers.add(identifier)
break
if local_name in {"format", "formats"} and text:
advertised_formats.add(text)
if local_name in {"requestresponsecrss", "requestcrss", "responsecrss", "nativecrss", "crs"} and text:
advertised_crs.add(text)
for attribute_value in element.attrib.values():
normalized = str(attribute_value).strip()
if "EPSG" in normalized.upper() or "CRS:" in normalized.upper():
advertised_crs.add(normalized)
return {
"wcs_version": str(root.attrib.get("version") or "") or None,
"coverage_identifiers": sorted(coverage_identifiers),
"advertised_formats": sorted(advertised_formats),
"advertised_crs": sorted(advertised_crs),
}
@staticmethod
def _result(
*,
settings: Settings,
status: str,
checked_at: datetime,
message: str,
capabilities_url: str | None = None,
tls_verified: bool = False,
capabilities_reachable: bool = False,
response_sha256: str | None = None,
parsed: dict[str, Any] | None = None,
) -> dict[str, Any]:
parsed = parsed or {}
return BathymetrySourceProbeRead(
source_key=MdkBathymetryProbeService.SOURCE_KEY,
status=status,
configured_url=settings.mdk_bathymetry_wcs_url,
capabilities_url=capabilities_url,
tls_verified=tls_verified,
capabilities_reachable=capabilities_reachable,
acquisition_supported=False,
wcs_version=parsed.get("wcs_version"),
coverage_identifiers=parsed.get("coverage_identifiers") or [],
advertised_formats=parsed.get("advertised_formats") or [],
advertised_crs=parsed.get("advertised_crs") or [],
response_sha256=response_sha256,
checked_at=checked_at,
message=message,
limitation_message=MdkBathymetryProbeService.LIMITATION,
).model_dump(mode="json")
@staticmethod
def probe(
*,
settings: Settings | None = None,
opener: Callable[..., Any] | None = None,
checked_at: datetime | None = None,
) -> dict[str, Any]:
resolved_settings = settings or get_settings()
now = checked_at or datetime.now(UTC)
if not resolved_settings.mdk_bathymetry_probe_enabled:
return MdkBathymetryProbeService._result(
settings=resolved_settings,
status="disabled",
checked_at=now,
message="MDK bathymetry readiness probing is disabled.",
)
try:
capabilities_url = MdkBathymetryProbeService._capabilities_url(
resolved_settings.mdk_bathymetry_wcs_url
)
except ValueError as exc:
return MdkBathymetryProbeService._result(
settings=resolved_settings,
status="invalid_configuration",
checked_at=now,
message=str(exc),
)
try:
content, content_type = MdkBathymetryProbeService._read_capabilities(
capabilities_url, resolved_settings, opener
)
except HTTPError as exc:
return MdkBathymetryProbeService._result(
settings=resolved_settings,
status="endpoint_unavailable",
checked_at=now,
capabilities_url=capabilities_url,
tls_verified=True,
message=f"MDK WCS GetCapabilities returned HTTP {exc.code}.",
)
except ssl.SSLCertVerificationError:
return MdkBathymetryProbeService._result(
settings=resolved_settings,
status="tls_error",
checked_at=now,
capabilities_url=capabilities_url,
message="MDK WCS TLS certificate validation failed; insecure fallback is prohibited.",
)
except URLError as exc:
reason = exc.reason
is_tls_error = isinstance(reason, (ssl.SSLError, ssl.CertificateError)) or "certificate" in str(
reason
).casefold()
return MdkBathymetryProbeService._result(
settings=resolved_settings,
status="tls_error" if is_tls_error else "endpoint_unavailable",
checked_at=now,
capabilities_url=capabilities_url,
message=(
"MDK WCS TLS certificate validation failed; insecure fallback is prohibited."
if is_tls_error
else "MDK WCS GetCapabilities could not be reached."
),
)
except (TimeoutError, OSError) as exc:
return MdkBathymetryProbeService._result(
settings=resolved_settings,
status="endpoint_unavailable",
checked_at=now,
capabilities_url=capabilities_url,
message=f"MDK WCS GetCapabilities could not be reached ({type(exc).__name__}).",
)
except ValueError as exc:
return MdkBathymetryProbeService._result(
settings=resolved_settings,
status="invalid_capabilities",
checked_at=now,
capabilities_url=capabilities_url,
tls_verified=True,
capabilities_reachable=True,
message=str(exc),
)
response_sha256 = hashlib.sha256(content).hexdigest()
try:
parsed = MdkBathymetryProbeService._parse_capabilities(content)
except (ElementTree.ParseError, ValueError) as exc:
detail = " ".join(str(exc).split())
if content_type:
detail = f"{detail} Content-Type: {content_type}."
return MdkBathymetryProbeService._result(
settings=resolved_settings,
status="invalid_capabilities",
checked_at=now,
capabilities_url=capabilities_url,
tls_verified=True,
capabilities_reachable=True,
response_sha256=response_sha256,
message=detail,
)
if not parsed["coverage_identifiers"]:
return MdkBathymetryProbeService._result(
settings=resolved_settings,
status="invalid_capabilities",
checked_at=now,
capabilities_url=capabilities_url,
tls_verified=True,
capabilities_reachable=True,
response_sha256=response_sha256,
parsed=parsed,
message="MDK WCS capabilities are reachable but advertise no coverage identifier.",
)
return MdkBathymetryProbeService._result(
settings=resolved_settings,
status="reachable",
checked_at=now,
capabilities_url=capabilities_url,
tls_verified=True,
capabilities_reachable=True,
response_sha256=response_sha256,
parsed=parsed,
message=(
"MDK WCS capabilities are reachable with verified TLS. "
"Raster acquisition remains disabled pending bounded coverage validation."
),
)
@@ -282,8 +282,11 @@ def test_bathymetry_acquisition_persists_reference_dataset_through_dataset_servi
assert captured["reference_layer_name"] == "bathymetry_profiles" assert captured["reference_layer_name"] == "bathymetry_profiles"
assert captured["source_metadata"]["theme"] == "bathymetry" assert captured["source_metadata"]["theme"] == "bathymetry"
assert captured["source_metadata"]["volume_supported"] is False assert captured["source_metadata"]["volume_supported"] is False
assert captured["provenance_metadata"]["water_volume_available"] is False assert captured["source_metadata"]["municipality"] == "Mol"
assert captured["source_metadata"]["regional_partitions_complete"] is False
payload = json.loads(captured["content"]) payload = json.loads(captured["content"])
assert payload["features"][0]["properties"]["municipality"] == "Mol"
assert captured["provenance_metadata"]["water_volume_available"] is False
assert len(payload["features"]) == 2 assert len(payload["features"]) == 2
@@ -0,0 +1,295 @@
from __future__ import annotations
from datetime import UTC, datetime
import json
from pathlib import Path
import ssl
import sys
from types import SimpleNamespace
from urllib.error import URLError
from uuid import uuid4
from fastapi.testclient import TestClient
import pytest
from app.core.config import Settings
from app.core.errors import AppError
from app.db.session import get_db
from app.main import app
from app.models import Area, Dataset, DatasetVersion, Project
from app.schemas.bathymetry import BathymetryPartitionFinalizeRequest
from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService
ROOT = Path(__file__).resolve().parents[2]
SCRIPTS = ROOT / "scripts"
if str(SCRIPTS) not in sys.path:
sys.path.insert(0, str(SCRIPTS))
import provision_flanders_geographic_scope as flanders_scope # noqa: E402
class BinaryResponse:
def __init__(self, content: bytes, *, content_type: str = "application/xml"):
self.content = content
self.headers = {"Content-Type": content_type}
def __enter__(self):
return self
def __exit__(self, *_args):
return False
def read(self, size=-1):
return self.content if size < 0 else self.content[:size]
class JsonResponse:
def __init__(self, payload, *, url="https://geo.api.vlaanderen.be/VRBG/items"):
self.payload = payload
self.url = url
def raise_for_status(self):
return None
def json(self):
return self.payload
class SourceSession:
def __init__(self, payload):
self.payload = payload
def get(self, *_args, **_kwargs):
return JsonResponse(self.payload)
class VersionQuery:
def __init__(self, versions):
self.versions = versions
def filter(self, *_args):
return self
def all(self):
return self.versions
class FinalizeSession:
def __init__(self, rows, versions=None):
self.rows = rows
self.versions = versions or []
self.commit_count = 0
def get(self, model, row_id):
return self.rows.get((model, row_id))
def query(self, model):
assert model is DatasetVersion
return VersionQuery(self.versions)
def commit(self):
self.commit_count += 1
def capabilities_xml() -> bytes:
return b"""<?xml version="1.0"?>
<WCS_Capabilities version="1.0.0" xmlns="http://www.opengis.net/wcs">
<ContentMetadata>
<CoverageOfferingBrief>
<name>EL.GridCoverage</name>
<label>Belgian Continental Shelf bathymetry</label>
<lonLatEnvelope srsName="urn:ogc:def:crs:OGC:1.3:CRS84" />
</CoverageOfferingBrief>
</ContentMetadata>
<Capability><Request><GetCoverage><resultFormat><formats>GeoTIFF</formats></resultFormat></GetCoverage></Request></Capability>
</WCS_Capabilities>"""
def test_mdk_probe_parses_capabilities_without_enabling_acquisition() -> None:
seen = {}
def opener(request, timeout):
seen["url"] = request.full_url
seen["timeout"] = timeout
return BinaryResponse(capabilities_xml())
result = MdkBathymetryProbeService.probe(
settings=Settings(_env_file=None),
opener=opener,
checked_at=datetime(2026, 7, 17, tzinfo=UTC),
)
assert result["status"] == "reachable"
assert result["tls_verified"] is True
assert result["capabilities_reachable"] is True
assert result["acquisition_supported"] is False
assert result["coverage_identifiers"] == ["EL.GridCoverage"]
assert result["advertised_formats"] == ["GeoTIFF"]
assert result["response_sha256"]
assert "request=GetCapabilities" in seen["url"]
assert seen["timeout"] == 20
def test_mdk_probe_reports_tls_failure_and_never_uses_insecure_fallback() -> None:
calls = 0
def opener(_request, timeout):
nonlocal calls
assert timeout == 20
calls += 1
raise URLError(ssl.SSLCertVerificationError("hostname mismatch"))
result = MdkBathymetryProbeService.probe(
settings=Settings(_env_file=None),
opener=opener,
)
assert calls == 1
assert result["status"] == "tls_error"
assert result["tls_verified"] is False
assert result["capabilities_reachable"] is False
assert "insecure fallback is prohibited" in result["message"]
def test_mdk_readiness_api_uses_canonical_envelope(monkeypatch) -> None:
project_id = uuid4()
db = SimpleNamespace(get=lambda model, row_id: Project(id=project_id, name="Mol") if model is Project else None)
monkeypatch.setattr(
MdkBathymetryProbeService,
"probe",
lambda: {
"source_key": "mdk_bcp_bathymetry",
"status": "tls_error",
"acquisition_supported": False,
},
)
app.dependency_overrides[get_db] = lambda: db
try:
response = TestClient(app).get(
f"/api/v1/projects/{project_id}/datasets/bathymetry/sources/mdk_bcp_bathymetry/readiness"
)
finally:
app.dependency_overrides.clear()
assert response.status_code == 200
assert set(response.json()) == {"data"}
assert response.json()["data"]["status"] == "tls_error"
assert response.json()["data"]["acquisition_supported"] is False
def test_partition_finalization_requires_complete_area_accounting_and_updates_versions() -> None:
project_id = uuid4()
area_ids = [uuid4(), uuid4()]
dataset_id = uuid4()
project = Project(id=project_id, name="Flanders")
areas = [
Area(id=area_ids[0], project_id=project_id, name="Gemeente Mol - officiële grens"),
Area(id=area_ids[1], project_id=project_id, name="Gemeente Geel - officiële grens"),
]
dataset = Dataset(
id=dataset_id,
project_id=project_id,
area_id=area_ids[0],
name="vha.geojson",
dataset_type="vector",
source="VHA",
source_name=BathymetryProfileAcquisitionService.PROVIDER,
source_metadata={
"profile_count": 3,
"document_count": 2,
"structured_depth_count": 1,
"measurement_date_min": "1990-01-01",
"measurement_date_max": "2020-01-01",
},
provenance_metadata={},
status="ready",
)
version = DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1)
rows = {
(Project, project_id): project,
(Area, area_ids[0]): areas[0],
(Area, area_ids[1]): areas[1],
(Dataset, dataset_id): dataset,
}
db = FinalizeSession(rows, [version])
payload = BathymetryPartitionFinalizeRequest(
partition_scope_key="flanders",
expected_area_ids=area_ids,
dataset_ids=[dataset_id],
no_profile_area_ids=[area_ids[1]],
manifest_sha256="a" * 64,
observed_at=datetime(2026, 7, 17, tzinfo=UTC),
)
result = BathymetryProfileAcquisitionService.finalize_partitions(db, project_id, payload)
assert result["regional_partitions_complete"] is True
assert result["partition_count"] == 2
assert result["data_partition_count"] == 1
assert result["no_profile_partition_count"] == 1
assert result["profile_count"] == 3
assert dataset.source_metadata["coverage_scope"] == "flanders"
assert dataset.source_metadata["municipality"] == "Mol"
assert dataset.source_metadata["partitioned_source_audit"] is True
assert version.source_metadata == dataset.source_metadata
assert version.provenance_metadata == dataset.provenance_metadata
assert db.commit_count == 1
incomplete = payload.model_copy(update={"no_profile_area_ids": []})
with pytest.raises(AppError) as exc_info:
BathymetryProfileAcquisitionService.finalize_partitions(
FinalizeSession(rows, [version]),
project_id,
incomplete,
)
assert exc_info.value.code == "BATHYMETRY_PARTITION_MANIFEST_INCOMPLETE"
def test_flanders_scope_discovery_uses_complete_unique_vrbg_inventory() -> None:
features = [
{
"type": "Feature",
"id": f"Refgem.{index:05d}",
"properties": {"NISCODE": f"{index:05d}", "NAAM": f"Gemeente {index:03d}"},
"geometry": {
"type": "Polygon",
"coordinates": [[[4.0, 50.5], [4.1, 50.5], [4.1, 50.6], [4.0, 50.5]]],
},
}
for index in range(1, 286)
]
scope, selected, source_url = flanders_scope.discover_flanders_scope(
SourceSession({"features": list(reversed(features))}),
timeout=30,
min_municipalities=270,
max_municipalities=300,
)
assert scope.key == "flanders"
assert scope.project_name == "Flanders Regional Workbench"
assert len(scope.members) == 285
assert len(set(scope.nis_codes)) == 285
assert [item["properties"]["NISCODE"] for item in selected] == sorted(scope.nis_codes)
assert source_url.startswith("https://geo.api.vlaanderen.be/")
def test_expansion_scripts_are_packaged_and_readiness_checked() -> None:
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
for script in (
"provision_flanders_geographic_scope.py",
"provision_flanders_bathymetry_profiles.py",
"probe_mdk_bathymetry.py",
):
assert f"COPY scripts/{script}" in dockerfile
assert f"py_compile scripts/{script}" in readiness
sources = {
item["key"]: item
for item in BathymetryProfileAcquisitionService.list_sources()
}
assert sources["mdk_bcp_bathymetry"]["integration_status"] == "probe_only"
assert sources["mdk_bcp_bathymetry"]["acquisition_supported"] is False
assert "EL_wcs" in sources["mdk_bcp_bathymetry"]["service_url"]
+3
View File
@@ -89,6 +89,9 @@ COPY scripts/provision_regional_historical_landuse.py /app/scripts/provision_reg
COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_official_landuse_timeseries.py COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_official_landuse_timeseries.py
COPY scripts/provision_waterinfo_station_history.py /app/scripts/provision_waterinfo_station_history.py COPY scripts/provision_waterinfo_station_history.py /app/scripts/provision_waterinfo_station_history.py
COPY scripts/provision_mol_bathymetry_profiles.py /app/scripts/provision_mol_bathymetry_profiles.py COPY scripts/provision_mol_bathymetry_profiles.py /app/scripts/provision_mol_bathymetry_profiles.py
COPY scripts/provision_flanders_geographic_scope.py /app/scripts/provision_flanders_geographic_scope.py
COPY scripts/provision_flanders_bathymetry_profiles.py /app/scripts/provision_flanders_bathymetry_profiles.py
COPY scripts/probe_mdk_bathymetry.py /app/scripts/probe_mdk_bathymetry.py
COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natura2000.py COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natura2000.py
COPY scripts/provision_regional_bwk_natura2000.py /app/scripts/provision_regional_bwk_natura2000.py COPY scripts/provision_regional_bwk_natura2000.py /app/scripts/provision_regional_bwk_natura2000.py
COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py
@@ -51,6 +51,11 @@
<Config Name="VMM Flood Hazard WCS URL" Target="FLOOD_HAZARD_WCS_URL" Default="https://geoservice.waterinfo.be/OGRK/wcs" Mode="" Description="Official VMM OGRK WCS endpoint for governed flood-depth scenarios." Type="Variable" Display="advanced" Required="true" Mask="false">https://geoservice.waterinfo.be/OGRK/wcs</Config> <Config Name="VMM Flood Hazard WCS URL" Target="FLOOD_HAZARD_WCS_URL" Default="https://geoservice.waterinfo.be/OGRK/wcs" Mode="" Description="Official VMM OGRK WCS endpoint for governed flood-depth scenarios." Type="Variable" Display="advanced" Required="true" Mask="false">https://geoservice.waterinfo.be/OGRK/wcs</Config>
<Config Name="Flood Hazard Analysis Resolution (m)" Target="FLOOD_HAZARD_RESOLUTION_M" Default="5.0" Mode="" Description="Stored analysis grid resolution; official source values are converted from centimetres to metres." Type="Variable" Display="advanced" Required="true" Mask="false">5.0</Config> <Config Name="Flood Hazard Analysis Resolution (m)" Target="FLOOD_HAZARD_RESOLUTION_M" Default="5.0" Mode="" Description="Stored analysis grid resolution; official source values are converted from centimetres to metres." Type="Variable" Display="advanced" Required="true" Mask="false">5.0</Config>
<Config Name="Flood Hazard Maximum Cells" Target="FLOOD_HAZARD_MAX_PIXELS" Default="12000000" Mode="" Description="Maximum raster cells per flood-hazard acquisition or selection analysis." Type="Variable" Display="advanced" Required="true" Mask="false">12000000</Config> <Config Name="Flood Hazard Maximum Cells" Target="FLOOD_HAZARD_MAX_PIXELS" Default="12000000" Mode="" Description="Maximum raster cells per flood-hazard acquisition or selection analysis." Type="Variable" Display="advanced" Required="true" Mask="false">12000000</Config>
<Config Name="VHA Bathymetry Profiles" Target="BATHYMETRY_PROFILES_ENABLED" Default="true" Mode="" Description="Allow explicit bounded acquisition of official VHA cross-section profile points." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
<Config Name="VHA Profile Maximum Features" Target="BATHYMETRY_PROFILES_MAX_FEATURES" Default="50000" Mode="" Description="Safety limit per municipality or bounded VHA profile request." Type="Variable" Display="advanced" Required="true" Mask="false">50000</Config>
<Config Name="MDK Bathymetry Readiness Probe" Target="MDK_BATHYMETRY_PROBE_ENABLED" Default="true" Mode="" Description="Enable strict-TLS read-only GetCapabilities checks. This never downloads or activates a North Sea raster." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
<Config Name="MDK Bathymetry WCS URL" Target="MDK_BATHYMETRY_WCS_URL" Default="https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs" Mode="" Description="Official metadata WCS endpoint. TLS verification is mandatory and cannot be bypassed." Type="Variable" Display="advanced" Required="true" Mask="false">https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs</Config>
<Config Name="MDK Probe Timeout Seconds" Target="MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS" Default="20" Mode="" Description="Maximum wait for one read-only MDK GetCapabilities request." Type="Variable" Display="advanced" Required="true" Mask="false">20</Config>
<Config Name="Official Thematic Raster Acquisition" Target="THEMATIC_RASTER_ENABLED" Default="true" Mode="" Description="Allow bounded official Departement Omgeving rasters for space, population, accessibility and services." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config> <Config Name="Official Thematic Raster Acquisition" Target="THEMATIC_RASTER_ENABLED" Default="true" Mode="" Description="Allow bounded official Departement Omgeving rasters for space, population, accessibility and services." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
<Config Name="Thematic Raster WCS URL" Target="THEMATIC_RASTER_WCS_URL" Default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs" Mode="" Description="Official public MercatorNet WCS endpoint. Product identifiers remain server allowlisted." Type="Variable" Display="advanced" Required="true" Mask="false">https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs</Config> <Config Name="Thematic Raster WCS URL" Target="THEMATIC_RASTER_WCS_URL" Default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs" Mode="" Description="Official public MercatorNet WCS endpoint. Product identifiers remain server allowlisted." Type="Variable" Display="advanced" Required="true" Mask="false">https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs</Config>
<Config Name="Thematic Raster Maximum Side (m)" Target="THEMATIC_RASTER_MAX_SIDE_M" Default="60000" Mode="" Description="Maximum side length for one allowlisted thematic raster scope; supports the complete Kempen work area while WCS transfers remain tiled." Type="Variable" Display="advanced" Required="true" Mask="false">60000</Config> <Config Name="Thematic Raster Maximum Side (m)" Target="THEMATIC_RASTER_MAX_SIDE_M" Default="60000" Mode="" Description="Maximum side length for one allowlisted thematic raster scope; supports the complete Kempen work area while WCS transfers remain tiled." Type="Variable" Display="advanced" Required="true" Mask="false">60000</Config>
+11
View File
@@ -59,6 +59,17 @@ FLOOD_HAZARD_MAX_SIDE_M=20000
FLOOD_HAZARD_MAX_PIXELS=12000000 FLOOD_HAZARD_MAX_PIXELS=12000000
FLOOD_HAZARD_TIMEOUT_SECONDS=300 FLOOD_HAZARD_TIMEOUT_SECONDS=300
FLOOD_HAZARD_MAX_RESPONSE_MB=160 FLOOD_HAZARD_MAX_RESPONSE_MB=160
BATHYMETRY_PROFILES_ENABLED=true
BATHYMETRY_PROFILES_LAYER_URL=https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0
BATHYMETRY_WATERCOURSE_LAYER_URL=https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1
BATHYMETRY_PROFILES_PAGE_SIZE=1000
BATHYMETRY_PROFILES_MAX_FEATURES=50000
BATHYMETRY_PROFILES_TIMEOUT_SECONDS=120
BATHYMETRY_PROFILES_MAX_RESPONSE_MB=32
MDK_BATHYMETRY_PROBE_ENABLED=true
MDK_BATHYMETRY_WCS_URL=https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs
MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS=20
MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB=4
# Allowlisted Departement Omgeving policy rasters. Regional requests are # Allowlisted Departement Omgeving policy rasters. Regional requests are
# transferred as fixed 10 km WCS tiles before exact Area clipping. # transferred as fixed 10 km WCS tiles before exact Area clipping.
+22
View File
@@ -51,6 +51,17 @@ FLOOD_HAZARD_MAX_SIDE_M="${FLOOD_HAZARD_MAX_SIDE_M:-20000}"
FLOOD_HAZARD_MAX_PIXELS="${FLOOD_HAZARD_MAX_PIXELS:-12000000}" FLOOD_HAZARD_MAX_PIXELS="${FLOOD_HAZARD_MAX_PIXELS:-12000000}"
FLOOD_HAZARD_TIMEOUT_SECONDS="${FLOOD_HAZARD_TIMEOUT_SECONDS:-300}" FLOOD_HAZARD_TIMEOUT_SECONDS="${FLOOD_HAZARD_TIMEOUT_SECONDS:-300}"
FLOOD_HAZARD_MAX_RESPONSE_MB="${FLOOD_HAZARD_MAX_RESPONSE_MB:-160}" FLOOD_HAZARD_MAX_RESPONSE_MB="${FLOOD_HAZARD_MAX_RESPONSE_MB:-160}"
BATHYMETRY_PROFILES_ENABLED="${BATHYMETRY_PROFILES_ENABLED:-true}"
BATHYMETRY_PROFILES_LAYER_URL="${BATHYMETRY_PROFILES_LAYER_URL:-https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0}"
BATHYMETRY_WATERCOURSE_LAYER_URL="${BATHYMETRY_WATERCOURSE_LAYER_URL:-https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1}"
BATHYMETRY_PROFILES_PAGE_SIZE="${BATHYMETRY_PROFILES_PAGE_SIZE:-1000}"
BATHYMETRY_PROFILES_MAX_FEATURES="${BATHYMETRY_PROFILES_MAX_FEATURES:-50000}"
BATHYMETRY_PROFILES_TIMEOUT_SECONDS="${BATHYMETRY_PROFILES_TIMEOUT_SECONDS:-120}"
BATHYMETRY_PROFILES_MAX_RESPONSE_MB="${BATHYMETRY_PROFILES_MAX_RESPONSE_MB:-32}"
MDK_BATHYMETRY_PROBE_ENABLED="${MDK_BATHYMETRY_PROBE_ENABLED:-true}"
MDK_BATHYMETRY_WCS_URL="${MDK_BATHYMETRY_WCS_URL:-https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs}"
MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS="${MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS:-20}"
MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB="${MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB:-4}"
THEMATIC_RASTER_ENABLED="${THEMATIC_RASTER_ENABLED:-true}" THEMATIC_RASTER_ENABLED="${THEMATIC_RASTER_ENABLED:-true}"
THEMATIC_RASTER_WCS_URL="${THEMATIC_RASTER_WCS_URL:-https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs}" THEMATIC_RASTER_WCS_URL="${THEMATIC_RASTER_WCS_URL:-https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs}"
THEMATIC_RASTER_MIN_SIDE_M="${THEMATIC_RASTER_MIN_SIDE_M:-100}" THEMATIC_RASTER_MIN_SIDE_M="${THEMATIC_RASTER_MIN_SIDE_M:-100}"
@@ -158,6 +169,17 @@ docker run -d \
-e FLOOD_HAZARD_MAX_PIXELS="$FLOOD_HAZARD_MAX_PIXELS" \ -e FLOOD_HAZARD_MAX_PIXELS="$FLOOD_HAZARD_MAX_PIXELS" \
-e FLOOD_HAZARD_TIMEOUT_SECONDS="$FLOOD_HAZARD_TIMEOUT_SECONDS" \ -e FLOOD_HAZARD_TIMEOUT_SECONDS="$FLOOD_HAZARD_TIMEOUT_SECONDS" \
-e FLOOD_HAZARD_MAX_RESPONSE_MB="$FLOOD_HAZARD_MAX_RESPONSE_MB" \ -e FLOOD_HAZARD_MAX_RESPONSE_MB="$FLOOD_HAZARD_MAX_RESPONSE_MB" \
-e BATHYMETRY_PROFILES_ENABLED="$BATHYMETRY_PROFILES_ENABLED" \
-e BATHYMETRY_PROFILES_LAYER_URL="$BATHYMETRY_PROFILES_LAYER_URL" \
-e BATHYMETRY_WATERCOURSE_LAYER_URL="$BATHYMETRY_WATERCOURSE_LAYER_URL" \
-e BATHYMETRY_PROFILES_PAGE_SIZE="$BATHYMETRY_PROFILES_PAGE_SIZE" \
-e BATHYMETRY_PROFILES_MAX_FEATURES="$BATHYMETRY_PROFILES_MAX_FEATURES" \
-e BATHYMETRY_PROFILES_TIMEOUT_SECONDS="$BATHYMETRY_PROFILES_TIMEOUT_SECONDS" \
-e BATHYMETRY_PROFILES_MAX_RESPONSE_MB="$BATHYMETRY_PROFILES_MAX_RESPONSE_MB" \
-e MDK_BATHYMETRY_PROBE_ENABLED="$MDK_BATHYMETRY_PROBE_ENABLED" \
-e MDK_BATHYMETRY_WCS_URL="$MDK_BATHYMETRY_WCS_URL" \
-e MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS="$MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS" \
-e MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB="$MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB" \
-e THEMATIC_RASTER_ENABLED="$THEMATIC_RASTER_ENABLED" \ -e THEMATIC_RASTER_ENABLED="$THEMATIC_RASTER_ENABLED" \
-e THEMATIC_RASTER_WCS_URL="$THEMATIC_RASTER_WCS_URL" \ -e THEMATIC_RASTER_WCS_URL="$THEMATIC_RASTER_WCS_URL" \
-e THEMATIC_RASTER_MIN_SIDE_M="$THEMATIC_RASTER_MIN_SIDE_M" \ -e THEMATIC_RASTER_MIN_SIDE_M="$THEMATIC_RASTER_MIN_SIDE_M" \
+4
View File
@@ -61,6 +61,10 @@ services:
BATHYMETRY_PROFILES_MAX_FEATURES: ${BATHYMETRY_PROFILES_MAX_FEATURES:-50000} BATHYMETRY_PROFILES_MAX_FEATURES: ${BATHYMETRY_PROFILES_MAX_FEATURES:-50000}
BATHYMETRY_PROFILES_TIMEOUT_SECONDS: ${BATHYMETRY_PROFILES_TIMEOUT_SECONDS:-120} BATHYMETRY_PROFILES_TIMEOUT_SECONDS: ${BATHYMETRY_PROFILES_TIMEOUT_SECONDS:-120}
BATHYMETRY_PROFILES_MAX_RESPONSE_MB: ${BATHYMETRY_PROFILES_MAX_RESPONSE_MB:-32} BATHYMETRY_PROFILES_MAX_RESPONSE_MB: ${BATHYMETRY_PROFILES_MAX_RESPONSE_MB:-32}
MDK_BATHYMETRY_PROBE_ENABLED: ${MDK_BATHYMETRY_PROBE_ENABLED:-true}
MDK_BATHYMETRY_WCS_URL: ${MDK_BATHYMETRY_WCS_URL:-https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs}
MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS: ${MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS:-20}
MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB: ${MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB:-4}
THEMATIC_RASTER_ENABLED: ${THEMATIC_RASTER_ENABLED:-true} THEMATIC_RASTER_ENABLED: ${THEMATIC_RASTER_ENABLED:-true}
THEMATIC_RASTER_WCS_URL: ${THEMATIC_RASTER_WCS_URL:-https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs} THEMATIC_RASTER_WCS_URL: ${THEMATIC_RASTER_WCS_URL:-https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs}
THEMATIC_RASTER_MIN_SIDE_M: ${THEMATIC_RASTER_MIN_SIDE_M:-100} THEMATIC_RASTER_MIN_SIDE_M: ${THEMATIC_RASTER_MIN_SIDE_M:-100}
+29 -4
View File
@@ -2061,10 +2061,20 @@ sets an explicit Ollama context window and returns
### GET `/api/v1/projects/{project_id}/datasets/bathymetry/sources` ### GET `/api/v1/projects/{project_id}/datasets/bathymetry/sources`
Returns the governed bathymetry source registry in the canonical envelope. Returns the governed bathymetry source registry in the canonical envelope.
VHA inland profiles are `operational`. MDK Belgian Continental Shelf and SPW VHA inland profiles are `operational`. MDK Belgian Continental Shelf is
Walloon bathymetry remain `available_not_integrated`; they cannot be acquired `probe_only`; SPW Walloon bathymetry remains `available_not_integrated`.
through this contract until their raster/download and vertical-datum flows Neither source can be acquired until its raster/download and vertical-datum
pass live validation. flow passes live validation.
### GET `/api/v1/projects/{project_id}/datasets/bathymetry/sources/mdk_bcp_bathymetry/readiness`
Runs one bounded, read-only WCS 1.0.0 `GetCapabilities` request with mandatory
system TLS verification and a configured response-size limit. Status is one
of `disabled`, `invalid_configuration`, `tls_error`,
`endpoint_unavailable`, `invalid_capabilities` or `reachable`. A reachable
response lists coverage identifiers, advertised formats and CRS values, but
always returns `acquisition_supported=false`. There is no insecure TLS
fallback and no `GetCoverage` request.
### POST `/api/v1/projects/{project_id}/datasets/bathymetry/profiles/acquire` ### POST `/api/v1/projects/{project_id}/datasets/bathymetry/profiles/acquire`
@@ -2089,3 +2099,18 @@ The acquired Dataset uses
`reference_layer_name=bathymetry_profiles`. Existing vector content and `reference_layer_name=bathymetry_profiles`. Existing vector content and
selection endpoints provide GeoJSON and metrics. The contract does not expose selection endpoints provide GeoJSON and metrics. The contract does not expose
a continuous bathymetric surface, current water depth or water volume. a continuous bathymetric surface, current water depth or water volume.
### POST `/api/v1/projects/{project_id}/datasets/bathymetry/profiles/partitions/finalize`
Finalizes a complete municipality-partition manifest. The request supplies a
scope key, every expected municipality Area id, one ready VHA Dataset id for
each non-empty partition, explicit Area ids with zero source profiles, a
SHA-256 manifest identity and observation time. The backend verifies project,
Area, provider, Dataset and one-Dataset-per-Area consistency. Only a complete
accounting sets `partitioned_source_audit=true` and
`regional_partitions_complete=true`; partial operator runs remain hidden at
regional scope.
Future provider output continues to use DatasetService and, for vectors,
VectorFeatureService. Arbitrary service URLs, browser-side fetches, insecure
TLS bypasses and startup downloads remain forbidden.
+11 -2
View File
@@ -120,12 +120,21 @@ bed evolution.
## Implementation order ## Implementation order
1. Operate and validate the Mol VHA profile Dataset and map flow. 1. Operate and validate the Mol VHA profile Dataset and map flow. **Done.**
2. Add VHA municipal partition orchestration for Flanders. 2. Add VHA municipal partition orchestration for Flanders. **Implemented;
complete live provisioning remains an explicit operator run.**
3. Implement a bounded MDK WCS probe, then acquisition behind live evidence. 3. Implement a bounded MDK WCS probe, then acquisition behind live evidence.
**Probe implemented. Acquisition blocked because the live endpoint fails
strict hostname validation and does not expose usable capabilities.**
4. Implement SPW download staging and vertical-datum metadata validation. 4. Implement SPW download staging and vertical-datum metadata validation.
5. Add maritime boundaries as separate authoritative scope layers. 5. Add maritime boundaries as separate authoritative scope layers.
6. Add cross-source vertical-datum transformation only with authoritative 6. Add cross-source vertical-datum transformation only with authoritative
grids/parameters and uncertainty tests. grids/parameters and uncertainty tests.
7. Add volume only after a compatible measured or modeled water-surface source 7. Add volume only after a compatible measured or modeled water-surface source
is part of the same analysis contract. is part of the same analysis contract.
The Flanders scope is land-only and is dynamically derived from the complete
current VRBG municipality collection (285 members at the 2026-07-17
validation). Belgium is not represented by expanding that polygon. Wallonia,
Brussels, the territorial sea and the Belgian EEZ/continental shelf require
their own authoritative adapters and legal scope labels.
+38
View File
@@ -10054,3 +10054,41 @@ Validation:
historical survey range `1877-2020` and loaded all 828 persisted objects. historical survey range `1877-2020` and loaded all 828 persisted objects.
The browser console contained no warnings or errors and the 1265x720 The browser console contained no warnings or errors and the 1265x720
viewport had no horizontal document overflow. viewport had no horizontal document overflow.
## Sprint 236 - Flemish bathymetry partitions and safe North Sea probe (2026-07-17)
Implemented:
- Added a dynamic Flanders scope operator that discovers the complete current
official VRBG RefGem municipality inventory instead of freezing a fragile
hand-maintained list. It validates unique NIS codes/names, polygonal source
features and a 270..300 safety range before creating artifacts or API state.
- Added an atomic, resumable VHA municipality coordinator. Every partition is
exact-Area clipped and persisted through the existing acquisition endpoint;
failures, explicit zero-profile results and ready Dataset ids remain in the
checksum-bound coverage manifest.
- Added server-side partition finalization. Regional activation requires exact
accounting for every expected municipality, one ready VHA Dataset per
non-empty Area and explicit no-profile Area ids. Dataset and DatasetVersion
metadata retain the manifest identity and completeness flags.
- Added a strict-TLS, response-bounded MDK WCS GetCapabilities probe and a
concise Sources-workspace control. It never performs GetCoverage, never
disables certificate verification and always leaves acquisition disabled.
- Packaged the Flanders scope, VHA coordinator and MDK probe operators in the
all-in-one image and exposed their bounded runtime settings in Compose and
the Unraid deployment configuration.
Validation:
- Official VRBG fetch-only validation returned 285 unique municipalities, an
exact union of 13,625.73 km2 and WGS84 bounds
`[2.54132923, 50.68749237, 5.9111094, 51.50511313]`.
- The live MDK metadata endpoint currently fails strict hostname validation:
`bathy.agentschapmdk.be` presents a certificate for `*.l27powered.eu`.
Diagnostic requests to both the previously configured and current metadata
paths returned HTTP 404 after an explicitly external, non-application
insecure inspection. GeoIntel itself reports `tls_error` and provides no
bypass.
- The complete readiness gate passed 920 backend tests, backend compilation,
documentation/contract audits, 115 documented API routes, Alembic head
`202607160001`, frontend TypeScript typecheck and the production Vite build.
- Live Docker deployment and controlled Flanders provisioning are recorded
after the repository commit below.
+8
View File
@@ -2,6 +2,14 @@
Database: PostgreSQL + PostGIS. Database: PostgreSQL + PostGIS.
Bathymetry regional completeness requires no new table. Every VHA municipality
partition remains an ordinary `datasets` row plus `vector_features`. After a
complete server-validated manifest, Dataset and DatasetVersion metadata retain
`partition_scope_key`, partition counts, the manifest SHA-256,
`partitioned_source_audit=true` and
`regional_partitions_complete=true`. Empty source partitions are recorded in
the manifest provenance and never represented by fabricated features.
## Rules ## Rules
- Store geometries in PostGIS with explicit SRID. - Store geometries in PostGIS with explicit SRID.
+15 -1
View File
@@ -722,7 +722,11 @@ are not filled by fabricated OCR output.
The following sources are audited but not yet operational: The following sources are audited but not yet operational:
- MDK Dieptemodel Belgisch Continentaal Plat/Noordzee: 20 x 20 m continuous - MDK Dieptemodel Belgisch Continentaal Plat/Noordzee: 20 x 20 m continuous
raster in LAT, exposed through WCS/WMTS. raster in LAT, exposed through WCS/WMTS. GeoIntel now has a strict-TLS,
read-only GetCapabilities probe. On 2026-07-17 the official metadata
endpoint presented a certificate for another hostname and returned no
usable capabilities path; status therefore remains `tls_error` and raster
acquisition is disabled.
- SPW bathymetry of navigable waterways and reservoir lakes: 0.5 m bed - SPW bathymetry of navigable waterways and reservoir lakes: 0.5 m bed
elevation and XYZ data in mDNG. elevation and XYZ data in mDNG.
- Port of Antwerp-Bruges periodic soundings: catalog candidate pending a - Port of Antwerp-Bruges periodic soundings: catalog candidate pending a
@@ -732,3 +736,13 @@ See `docs/BATHYMETRY_EXPANSION_ROADMAP.md`. TAW, LAT and mDNG remain separate
until an authoritative vertical transformation is implemented and tested. until an authoritative vertical transformation is implemented and tested.
The territorial sea, EEZ and continental shelf are separate scope layers and The territorial sea, EEZ and continental shelf are separate scope layers and
must be labelled according to their legal meaning. must be labelled according to their legal meaning.
The official VRBG RefGem collection currently exposes 285 Flemish
municipalities. `scripts/provision_flanders_geographic_scope.py` validates the
complete inventory, produces checksum-bound land-boundary artifacts and
creates one persisted municipality Area per source feature. The separate
`scripts/provision_flanders_bathymetry_profiles.py` coordinator acquires VHA
profiles per Area, writes an atomic resumable manifest and requests regional
activation only after all 285 partitions are accounted for. A municipality
with zero source points is retained as an explicit no-profile partition, not
silently omitted.
+24 -1
View File
@@ -222,7 +222,30 @@ Raster / gemodelleerd overstromingsgevaar.
### Prioriteit ### Prioriteit
P5 overstromingsscenario's operationeel voor Mol; bathymetrie blijft open. P5 overstromingsscenario's operationeel. VHA-dwarsprofielen zijn als
historische puntmetingen operationeel voor Mol en partitioneerbaar voor alle
officiële Vlaamse gemeente-Areas. Ze blijven strikt gescheiden van dit
scenario-raster en leveren geen gebiedsdekkende bathymetrie of volume.
## Bathymetry partition metadata
Een VHA-profiel-Dataset behoudt `partition_area_id`,
`partition_area_name`, `municipality`, meetdatumbereik en exacte aantallen.
Regionale Vlaamse dekking is alleen geldig wanneer de server een volledig
manifest voor de actuele officiële gemeente-inventaris heeft gevalideerd.
Daarna bevatten Dataset en DatasetVersion:
- `partition_scope_key`
- `partition_count`, `data_partition_count` en `no_profile_partition_count`
- `partition_manifest_sha256`
- `partitioned_source_audit=true`
- `regional_partitions_complete=true`
Een no-profile partition is bronbewijs dat de begrensde VHA-query nul punten
opleverde. Er worden nooit lege of kunstmatige profielobjecten aangemaakt.
MDK Noordzee blijft `probe_only`: WCS-capabilities kunnen veilig worden
gecontroleerd, maar er bestaat nog geen toegestane rasteracquisitie.
## LAS/LAZ LiDAR ## LAS/LAZ LiDAR
+4 -3
View File
@@ -751,11 +751,12 @@ This file now starts with the current implementation status. Older preparation/b
creates a new immutable Dataset with official `YYYY.NN` source version. creates a new immutable Dataset with official `YYYY.NN` source version.
# Bathymetry follow-up # Bathymetry follow-up
- [ ] Add governed municipality partition orchestration for all of Flanders - [x] Add governed municipality partition orchestration for all of Flanders
after the Mol VHA operator passes live acceptance. after the Mol VHA operator passes live acceptance.
- [ ] Add read-only MDK WCS capability/TLS probe and maritime scope metadata. - [x] Add read-only MDK WCS capability/TLS probe and maritime scope metadata.
- [ ] Add bounded MDK GeoTIFF acquisition only after live CRS, LAT, nodata and - [ ] Add bounded MDK GeoTIFF acquisition only after live CRS, LAT, nodata and
response-limit validation. response-limit validation. Current official endpoint is blocked by TLS
hostname mismatch and unavailable capabilities; never bypass verification.
- [ ] Add SPW staged-download adapter with mDNG metadata and survey-epoch - [ ] Add SPW staged-download adapter with mDNG metadata and survey-epoch
coverage validation. coverage validation.
- [ ] Add authoritative territorial-sea, EEZ and continental-shelf boundary - [ ] Add authoritative territorial-sea, EEZ and continental-shelf boundary
+4 -2
View File
@@ -646,5 +646,7 @@ profile-document link.
The UI always labels the points as historical cross-sections. It does not The UI always labels the points as historical cross-sections. It does not
present them as a continuous bed raster and does not calculate water volume. present them as a continuous bed raster and does not calculate water volume.
The Sources workspace lists the future MDK North Sea and SPW Walloon Municipality partitions remain hidden for a regional Area until their backend
bathymetry integrations as planned until backend acquisition is operational. manifest reports complete coverage. The Sources workspace lists MDK North Sea
as read-only probe-only and SPW Walloon bathymetry as planned until bounded
raster/download acquisition is operational.
+1 -1
View File
@@ -806,7 +806,7 @@ function App(): JSX.Element {
{activeWorkspace === 'data' ? ( {activeWorkspace === 'data' ? (
<div className="workspace-grid workspace-grid-data"> <div className="workspace-grid workspace-grid-data">
<SourceCatalogPanel datasets={datasets} /> <SourceCatalogPanel datasets={datasets} projectId={selectedProjectId ?? undefined} />
<ProjectPanel <ProjectPanel
projects={projects} projects={projects}
selectedProjectId={selectedProjectId} selectedProjectId={selectedProjectId}
@@ -1,4 +1,6 @@
import type { DatasetCreateResponse } from '../../types' import { useState } from 'react'
import { datasetsApi } from '../../services/api'
import type { BathymetrySourceProbeRead, DatasetCreateResponse } from '../../types'
import { getDatasetSourceDisplayName } from '../../lib/datasetDisplay' import { getDatasetSourceDisplayName } from '../../lib/datasetDisplay'
import { import {
OFFICIAL_SOURCE_PORTFOLIO, OFFICIAL_SOURCE_PORTFOLIO,
@@ -10,6 +12,7 @@ import {
interface SourceCatalogPanelProps { interface SourceCatalogPanelProps {
datasets: DatasetCreateResponse[] datasets: DatasetCreateResponse[]
projectId?: string
} }
const THEME_LABELS: Record<string, string> = { const THEME_LABELS: Record<string, string> = {
@@ -63,7 +66,19 @@ function timelineSummary(
].filter(Boolean).join(' · ') ].filter(Boolean).join(' · ')
} }
export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.Element { function mdkStatusLabel(status: BathymetrySourceProbeRead['status']): string {
if (status === 'reachable') return 'Service bereikbaar'
if (status === 'tls_error') return 'Geblokkeerd: certificaat'
if (status === 'endpoint_unavailable') return 'Service niet bereikbaar'
if (status === 'invalid_capabilities') return 'Ongeldige service-informatie'
if (status === 'disabled') return 'Controle uitgeschakeld'
return 'Configuratie controleren'
}
export function SourceCatalogPanel({ datasets, projectId }: SourceCatalogPanelProps): JSX.Element {
const [mdkProbe, setMdkProbe] = useState<BathymetrySourceProbeRead | null>(null)
const [mdkProbeLoading, setMdkProbeLoading] = useState(false)
const [mdkProbeError, setMdkProbeError] = useState<string | null>(null)
const ready = datasets.filter((dataset) => dataset.status === 'ready') const ready = datasets.filter((dataset) => dataset.status === 'ready')
const waterinfoDatasets = ready.filter((dataset) => dataset.source_name === 'waterinfo') const waterinfoDatasets = ready.filter((dataset) => dataset.source_name === 'waterinfo')
const historicalOrthophotos = ready.filter( const historicalOrthophotos = ready.filter(
@@ -135,6 +150,19 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E
} }
}) })
const checkMdkReadiness = async (): Promise<void> => {
if (!projectId || mdkProbeLoading) return
setMdkProbeLoading(true)
setMdkProbeError(null)
try {
setMdkProbe(await datasetsApi.probeMdkBathymetry(projectId))
} catch (error) {
setMdkProbeError(error instanceof Error ? error.message : 'De Noordzee-bron kon niet worden gecontroleerd.')
} finally {
setMdkProbeLoading(false)
}
}
return ( return (
<section className="workspace-panel source-catalog-panel" aria-label="Beschikbare databronnen"> <section className="workspace-panel source-catalog-panel" aria-label="Beschikbare databronnen">
<div className="panel-title-row"> <div className="panel-title-row">
@@ -264,6 +292,29 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E
</details> </details>
) : null} ) : null}
<details className="source-loaded-disclosure">
<summary>Maritieme bronstatus</summary>
<div className="source-catalog-loaded">
<article>
<strong>MDK-dieptemodel Belgische Noordzee</strong>
<span>{mdkProbe ? mdkStatusLabel(mdkProbe.status) : 'Nog niet live gecontroleerd'}</span>
<p>
{mdkProbe?.message ??
'Controleert uitsluitend de officiële service-informatie met geldige TLS. Er wordt geen raster gedownload.'}
</p>
{mdkProbeError ? <small className="error-text">{mdkProbeError}</small> : null}
<button
className="secondary-button"
type="button"
disabled={!projectId || mdkProbeLoading}
onClick={() => void checkMdkReadiness()}
>
{mdkProbeLoading ? 'Controleren...' : 'Controleer Noordzee-bron'}
</button>
</article>
</div>
</details>
<details className="source-opportunity-list"> <details className="source-opportunity-list">
<summary>Officiële bronnen die hierna kunnen worden ingeladen</summary> <summary>Officiële bronnen die hierna kunnen worden ingeladen</summary>
<div className="source-opportunity-domains"> <div className="source-opportunity-domains">
+5
View File
@@ -23,6 +23,7 @@ import type {
FloodHazardProductRead, FloodHazardProductRead,
FloodHazardSelectionResponse, FloodHazardSelectionResponse,
BathymetryProfileAcquireRequest, BathymetryProfileAcquireRequest,
BathymetrySourceProbeRead,
BathymetrySourceRead, BathymetrySourceRead,
DhmvProductRead, DhmvProductRead,
TerrainSelectionResponse, TerrainSelectionResponse,
@@ -173,6 +174,10 @@ export const datasetsApi = {
apiGet<{ items: BathymetrySourceRead[]; total: number }>( apiGet<{ items: BathymetrySourceRead[]; total: number }>(
`/api/v1/projects/${projectId}/datasets/bathymetry/sources`, `/api/v1/projects/${projectId}/datasets/bathymetry/sources`,
), ),
probeMdkBathymetry: (projectId: string): Promise<BathymetrySourceProbeRead> =>
apiGet<BathymetrySourceProbeRead>(
`/api/v1/projects/${projectId}/datasets/bathymetry/sources/mdk_bcp_bathymetry/readiness`,
),
acquireBathymetryProfiles: (projectId: string, payload: BathymetryProfileAcquireRequest): Promise<JobRead> => acquireBathymetryProfiles: (projectId: string, payload: BathymetryProfileAcquireRequest): Promise<JobRead> =>
apiPost<JobRead>(`/api/v1/projects/${projectId}/datasets/bathymetry/profiles/acquire`, payload), apiPost<JobRead>(`/api/v1/projects/${projectId}/datasets/bathymetry/profiles/acquire`, payload),
acquireThematicRaster: (projectId: string, payload: ThematicRasterAcquireRequest): Promise<JobRead> => acquireThematicRaster: (projectId: string, payload: ThematicRasterAcquireRequest): Promise<JobRead> =>
+25 -1
View File
@@ -459,7 +459,7 @@ export interface BathymetrySourceRead {
vertical_reference: string vertical_reference: string
horizontal_crs: string horizontal_crs: string
native_resolution?: string | null native_resolution?: string | null
integration_status: 'operational' | 'available_not_integrated' | 'catalog_only' integration_status: 'operational' | 'probe_only' | 'available_not_integrated' | 'catalog_only'
acquisition_supported: boolean acquisition_supported: boolean
configured: boolean configured: boolean
service_url?: string | null service_url?: string | null
@@ -469,6 +469,30 @@ export interface BathymetrySourceRead {
limitation_message: string limitation_message: string
} }
export interface BathymetrySourceProbeRead {
source_key: 'mdk_bcp_bathymetry'
status:
| 'disabled'
| 'invalid_configuration'
| 'tls_error'
| 'endpoint_unavailable'
| 'invalid_capabilities'
| 'reachable'
configured_url: string
capabilities_url?: string | null
tls_verified: boolean
capabilities_reachable: boolean
acquisition_supported: false
wcs_version?: string | null
coverage_identifiers: string[]
advertised_formats: string[]
advertised_crs: string[]
response_sha256?: string | null
checked_at: string
message: string
limitation_message: string
}
export interface ThematicRasterAcquireRequest { export interface ThematicRasterAcquireRequest {
bbox: VectorSelectionBBox bbox: VectorSelectionBBox
area_id?: string | null area_id?: string | null
+29
View File
@@ -1909,3 +1909,32 @@ python scripts/provision_mol_bathymetry_profiles.py \
The output is a point dataset with historical evidence. It is not a continuous The output is a point dataset with historical evidence. It is not a continuous
water-bottom raster and cannot calculate current water volume. water-bottom raster and cannot calculate current water volume.
## Flanders VHA bathymetry partitions
Create the complete official Flemish land scope from the current VRBG
municipality collection:
```bash
docker exec geointel python /app/scripts/provision_flanders_geographic_scope.py
```
Acquire VHA profile partitions with atomic resume evidence:
```bash
docker exec geointel python /app/scripts/provision_flanders_bathymetry_profiles.py
```
Use `--members Mol Geel` or `--max-partitions 5` only for a partial operational
check. Partial runs do not activate regional coverage. The complete run
finalizes only when every official municipality has either one ready Dataset
or an explicit zero-profile source result.
Probe the official-metadata MDK WCS safely:
```bash
docker exec geointel python /app/scripts/probe_mdk_bathymetry.py
```
This performs only `GetCapabilities`, keeps strict TLS verification enabled
and returns exit code `2` for an honest non-ready source.
+57
View File
@@ -0,0 +1,57 @@
"""Run the safe MDK WCS readiness probe through the canonical GeoIntel API."""
from __future__ import annotations
import argparse
import json
import os
import requests
DEFAULT_API_URL = "http://127.0.0.1:8000"
DEFAULT_PROJECT_NAME = "Kempen Regional Workbench"
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Probe MDK bathymetry WCS readiness without downloading coverage.")
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
parser.add_argument("--project-name", default=DEFAULT_PROJECT_NAME)
parser.add_argument("--timeout", type=int, default=60)
return parser.parse_args()
def unwrap(response: requests.Response):
response.raise_for_status()
payload = response.json()
if not isinstance(payload, dict) or "data" not in payload:
raise RuntimeError(f"Non-canonical API response from {response.url}")
return payload["data"]
def main() -> int:
args = parse_args()
base_url = args.base_url.rstrip("/")
session = requests.Session()
projects = unwrap(
session.get(
f"{base_url}/api/v1/projects",
params={"name": args.project_name, "limit": 1},
timeout=args.timeout,
)
)["items"]
if not projects:
raise RuntimeError(f"Project {args.project_name!r} was not found")
result = unwrap(
session.get(
f"{base_url}/api/v1/projects/{projects[0]['id']}/datasets/"
"bathymetry/sources/mdk_bcp_bathymetry/readiness",
timeout=args.timeout,
)
)
print(json.dumps(result, ensure_ascii=False, indent=2))
return 0 if result["status"] == "reachable" else 2
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,400 @@
"""Provision VHA cross-section profiles for every persisted Flemish municipality.
The operator is resumable and calls only canonical GeoIntel API endpoints.
Regional completeness is finalized server-side only when every municipality
has either one ready profile Dataset or an explicit no-profile source result.
"""
from __future__ import annotations
import argparse
import hashlib
import json
import os
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Iterable
import requests
DEFAULT_API_URL = "http://127.0.0.1:8000"
DEFAULT_PROJECT_NAME = "Flanders Regional Workbench"
DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/bathymetry/vha-flanders/manifest.json")
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Provision partitioned VHA profiles for Flanders.")
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
parser.add_argument("--project-name", default=DEFAULT_PROJECT_NAME)
parser.add_argument(
"--manifest-path",
type=Path,
default=Path(os.environ.get("GEOINTEL_VHA_FLANDERS_MANIFEST", DEFAULT_MANIFEST_PATH)),
)
parser.add_argument("--timeout", type=int, default=900)
parser.add_argument("--force", action="store_true")
parser.add_argument("--continue-on-error", action="store_true")
parser.add_argument("--members", nargs="*", default=[])
parser.add_argument("--max-partitions", type=int, default=0)
return parser.parse_args()
def unwrap(response: requests.Response) -> Any:
try:
payload = response.json()
except ValueError as exc:
raise RuntimeError(f"GeoIntel returned non-JSON ({response.status_code}): {response.text[:300]}") from exc
if not response.ok:
error_code = payload.get("error") if isinstance(payload, dict) else None
message = payload.get("message") if isinstance(payload, dict) else None
raise RuntimeError(f"{error_code or response.status_code}: {message or response.text[:300]}")
if not isinstance(payload, dict) or "data" not in payload:
raise RuntimeError(f"Non-canonical API response from {response.url}")
return payload["data"]
def list_paginated(session: requests.Session, url: str, *, timeout: int) -> list[dict[str, Any]]:
items: list[dict[str, Any]] = []
offset = 0
total: int | None = None
while True:
page = unwrap(session.get(url, params={"limit": 200, "offset": offset}, timeout=timeout))
page_items = list(page.get("items") or [])
items.extend(page_items)
total = int(page.get("total") or 0) if total is None else total
if not page_items or len(items) >= total:
break
offset += len(page_items)
if total is not None and len(items) != total:
raise RuntimeError(f"Paginated API returned {len(items)} of {total} records for {url}")
return items
def coordinates(geometry: dict[str, Any]) -> Iterable[tuple[float, float]]:
def walk(value: Any):
if isinstance(value, list) and len(value) >= 2 and all(isinstance(item, (int, float)) for item in value[:2]):
yield float(value[0]), float(value[1])
return
if isinstance(value, list):
for child in value:
yield from walk(child)
yield from walk(geometry.get("coordinates", []))
def geometry_bbox(geometry: dict[str, Any]) -> dict[str, float | str]:
points = list(coordinates(geometry))
if not points:
raise RuntimeError("Persisted municipality Area contains no coordinates")
xs = [point[0] for point in points]
ys = [point[1] for point in points]
return {
"min_x": min(xs),
"min_y": min(ys),
"max_x": max(xs),
"max_y": max(ys),
"crs": "EPSG:4326",
}
def area_identity(area: dict[str, Any]) -> str:
canonical = {
"id": area.get("id"),
"name": area.get("name"),
"geometry": area.get("geometry"),
}
return hashlib.sha256(
json.dumps(canonical, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
).hexdigest()
def write_manifest(path: Path, payload: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
temporary = path.with_suffix(path.suffix + ".partial")
temporary.write_text(
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True),
encoding="utf-8",
)
temporary.replace(path)
def load_manifest(path: Path) -> dict[str, Any]:
if not path.is_file():
return {}
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return {}
return payload if isinstance(payload, dict) else {}
def source_error(response: requests.Response) -> tuple[str | None, str]:
try:
payload = response.json()
except ValueError:
return None, response.text[:300]
if not isinstance(payload, dict):
return None, response.text[:300]
return (
str(payload.get("error")) if payload.get("error") else None,
str(payload.get("message") or response.text[:300]),
)
def acquire_partition(
session: requests.Session,
*,
base_url: str,
project_id: str,
area: dict[str, Any],
timeout: int,
force: bool,
) -> dict[str, Any]:
bbox = geometry_bbox(area["geometry"])
response = session.post(
f"{base_url}/api/v1/projects/{project_id}/datasets/bathymetry/profiles/acquire",
json={"bbox": bbox, "area_id": area["id"], "force_refresh": force},
timeout=timeout,
)
if response.status_code == 404:
code, message = source_error(response)
if code == "BATHYMETRY_NO_PROFILES":
return {
"status": "no_profiles",
"area_id": area["id"],
"area_name": area["name"],
"area_identity_sha256": area_identity(area),
"bbox": bbox,
"message": message,
"completed_at": datetime.now(timezone.utc).isoformat(),
}
data = unwrap(response)
if data.get("status") != "success" or not data.get("output_dataset_id"):
raise RuntimeError(f"VHA acquisition failed for {area['name']}: {data.get('error_message') or data}")
result = data.get("result_json") or {}
return {
"status": "complete",
"area_id": area["id"],
"area_name": area["name"],
"area_identity_sha256": area_identity(area),
"bbox": bbox,
"dataset_id": data["output_dataset_id"],
"reused": bool(result.get("reused")),
"profile_count": int(result.get("profile_count") or 0),
"document_count": int(result.get("document_count") or 0),
"structured_depth_count": int(result.get("structured_depth_count") or 0),
"measurement_date_min": result.get("measurement_date_min"),
"measurement_date_max": result.get("measurement_date_max"),
"completed_at": datetime.now(timezone.utc).isoformat(),
}
def manifest_identity(
*,
project_id: str,
partitions: list[dict[str, Any]],
) -> tuple[dict[str, Any], str]:
identity = {
"schema_version": 1,
"source": "vmm_vha_bathymetry_profiles",
"partition_scope_key": "flanders",
"project_id": project_id,
"partitions": [
{
key: partition.get(key)
for key in (
"area_id",
"area_name",
"area_identity_sha256",
"status",
"dataset_id",
"profile_count",
"document_count",
"structured_depth_count",
"measurement_date_min",
"measurement_date_max",
)
}
for partition in sorted(partitions, key=lambda item: str(item["area_id"]))
],
}
digest = hashlib.sha256(
json.dumps(identity, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
).hexdigest()
return identity, digest
def main() -> int:
args = parse_args()
if args.max_partitions < 0:
print(json.dumps({"status": "error", "message": "--max-partitions cannot be negative"}), file=sys.stderr)
return 2
base_url = args.base_url.rstrip("/")
session = requests.Session()
session.headers.update({"User-Agent": "GeoIntel-VHA-Flanders-Operator/1.0"})
manifest = load_manifest(args.manifest_path)
try:
projects = list_paginated(session, f"{base_url}/api/v1/projects", timeout=60)
project = next((item for item in projects if item.get("name") == args.project_name), None)
if project is None:
raise RuntimeError(
f"Project {args.project_name!r} was not found; run provision_flanders_geographic_scope.py first"
)
all_areas = list_paginated(
session,
f"{base_url}/api/v1/projects/{project['id']}/areas",
timeout=120,
)
municipalities = sorted(
(area for area in all_areas if str(area.get("name") or "").casefold().startswith("gemeente ")),
key=lambda item: str(item["name"]).casefold(),
)
if not 270 <= len(municipalities) <= 300:
raise RuntimeError(
f"Flanders workspace exposes {len(municipalities)} municipality Areas; expected 270..300"
)
requested_names = {name.casefold() for name in args.members}
selected = [
area
for area in municipalities
if not requested_names
or str(area["name"]).removeprefix("Gemeente ").split(" - ", 1)[0].casefold() in requested_names
]
if requested_names:
found = {
str(area["name"]).removeprefix("Gemeente ").split(" - ", 1)[0].casefold()
for area in selected
}
missing = sorted(requested_names - found)
if missing:
raise RuntimeError(f"Unknown Flanders municipality selections: {', '.join(missing)}")
if args.max_partitions:
selected = selected[: args.max_partitions]
prior_by_area = {
str(item.get("area_id")): item
for item in (manifest.get("partitions") or [])
if isinstance(item, dict) and item.get("area_id")
}
current: list[dict[str, Any]] = []
failures: list[dict[str, Any]] = []
observed_at = datetime.now(timezone.utc).isoformat()
working_manifest = {
"schema_version": 1,
"status": "running",
"source": "vmm_vha_bathymetry_profiles",
"partition_scope_key": "flanders",
"project_id": project["id"],
"project_name": project["name"],
"municipality_inventory_count": len(municipalities),
"selected_partition_count": len(selected),
"observed_at": observed_at,
"partitions": current,
}
for index, area in enumerate(selected, start=1):
identity = area_identity(area)
prior = prior_by_area.get(str(area["id"]))
if (
not args.force
and prior
and prior.get("area_identity_sha256") == identity
and prior.get("status") in {"complete", "no_profiles"}
):
result = prior
else:
try:
result = acquire_partition(
session,
base_url=base_url,
project_id=str(project["id"]),
area=area,
timeout=args.timeout,
force=args.force,
)
except (RuntimeError, requests.RequestException) as exc:
result = {
"status": "failed",
"area_id": area["id"],
"area_name": area["name"],
"area_identity_sha256": identity,
"message": str(exc),
"failed_at": datetime.now(timezone.utc).isoformat(),
}
failures.append(result)
current.append(result)
working_manifest["completed_partition_count"] = index
working_manifest["partitions"] = current
write_manifest(args.manifest_path, working_manifest)
if result["status"] == "failed" and not args.continue_on_error:
raise RuntimeError(f"Partition failed for {area['name']}: {result['message']}")
full_inventory_selected = len(selected) == len(municipalities) and not requested_names and not args.max_partitions
complete = not failures and all(item["status"] in {"complete", "no_profiles"} for item in current)
identity_payload, identity_sha = manifest_identity(
project_id=str(project["id"]),
partitions=current,
)
finalization = None
if complete and full_inventory_selected:
dataset_ids = [item["dataset_id"] for item in current if item["status"] == "complete"]
no_profile_area_ids = [item["area_id"] for item in current if item["status"] == "no_profiles"]
finalization = unwrap(
session.post(
f"{base_url}/api/v1/projects/{project['id']}/datasets/bathymetry/profiles/partitions/finalize",
json={
"partition_scope_key": "flanders",
"expected_area_ids": [area["id"] for area in municipalities],
"dataset_ids": dataset_ids,
"no_profile_area_ids": no_profile_area_ids,
"manifest_sha256": identity_sha,
"observed_at": observed_at,
},
timeout=args.timeout,
)
)
working_manifest.update(
{
"status": "complete" if complete else "failed",
"completed_at": datetime.now(timezone.utc).isoformat(),
"coverage_manifest": identity_payload,
"coverage_manifest_sha256": identity_sha,
"regional_finalized": finalization is not None,
"finalization": finalization,
"failed_partition_count": len(failures),
}
)
write_manifest(args.manifest_path, working_manifest)
except (OSError, RuntimeError, ValueError, KeyError, requests.RequestException) as exc:
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
return 1
print(
json.dumps(
{
"status": working_manifest["status"],
"project_id": project["id"],
"municipality_inventory_count": len(municipalities),
"processed_partition_count": len(current),
"data_partition_count": sum(item["status"] == "complete" for item in current),
"no_profile_partition_count": sum(item["status"] == "no_profiles" for item in current),
"failed_partition_count": len(failures),
"profile_count": sum(int(item.get("profile_count") or 0) for item in current),
"document_count": sum(int(item.get("document_count") or 0) for item in current),
"regional_finalized": finalization is not None,
"manifest_path": str(args.manifest_path),
"manifest_sha256": identity_sha,
"finalization": finalization,
},
ensure_ascii=False,
indent=2,
)
)
return 0 if working_manifest["status"] == "complete" else 1
if __name__ == "__main__":
raise SystemExit(main())
@@ -0,0 +1,220 @@
"""Provision all current official Flemish municipality Areas.
The municipality inventory is discovered from the official VRBG RefGem
collection at runtime. The script uses the existing geographic-scope artifact
and API persistence flow; it never writes directly to PostGIS.
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
import requests
from geographic_scopes import GeographicScope, ScopeMember
from provision_geographic_scope import (
VRBG_ATTRIBUTION,
VRBG_ITEMS_URL,
build_scope_payloads,
build_source_session,
provision_scope,
sha256_file,
write_json_atomic,
)
DEFAULT_API_URL = "http://127.0.0.1:8000"
DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-data/geographic-scopes")
MIN_EXPECTED_MUNICIPALITIES = 270
MAX_EXPECTED_MUNICIPALITIES = 300
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Provision the current official Flanders municipality scope.")
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
parser.add_argument(
"--output-root",
type=Path,
default=Path(os.environ.get("GEOINTEL_SCOPE_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)),
)
parser.add_argument("--request-timeout", type=int, default=180)
parser.add_argument("--import-timeout", type=int, default=3600)
parser.add_argument("--force", action="store_true")
parser.add_argument("--fetch-only", action="store_true")
parser.add_argument("--min-municipalities", type=int, default=MIN_EXPECTED_MUNICIPALITIES)
parser.add_argument("--max-municipalities", type=int, default=MAX_EXPECTED_MUNICIPALITIES)
return parser.parse_args()
def discover_flanders_scope(
session: requests.Session,
*,
timeout: int,
min_municipalities: int,
max_municipalities: int,
) -> tuple[GeographicScope, list[dict[str, Any]], str]:
if min_municipalities <= 0 or max_municipalities < min_municipalities:
raise ValueError("Municipality count safety limits are invalid")
response = session.get(
VRBG_ITEMS_URL,
params={"f": "application/geo+json", "limit": "1000"},
timeout=timeout,
)
response.raise_for_status()
payload = response.json()
features = payload.get("features")
if not isinstance(features, list):
raise RuntimeError("Official VRBG response does not contain a feature list")
if not min_municipalities <= len(features) <= max_municipalities:
raise RuntimeError(
f"Official VRBG returned {len(features)} municipalities; expected "
f"{min_municipalities}..{max_municipalities}. Refusing an incomplete or broadened scope."
)
by_code: dict[str, dict[str, Any]] = {}
names: set[str] = set()
for feature in features:
if not isinstance(feature, dict) or not isinstance(feature.get("geometry"), dict):
raise RuntimeError("Official VRBG contains a malformed municipality feature")
properties = feature.get("properties")
if not isinstance(properties, dict):
raise RuntimeError("Official VRBG municipality is missing properties")
nis_code = str(properties.get("NISCODE") or "").strip()
name = str(properties.get("NAAM") or "").strip()
if len(nis_code) != 5 or not nis_code.isdigit() or not name:
raise RuntimeError(f"Official VRBG municipality identity is invalid: {nis_code!r} / {name!r}")
if nis_code in by_code or name.casefold() in names:
raise RuntimeError(f"Official VRBG contains a duplicate municipality: {nis_code} / {name}")
by_code[nis_code] = feature
names.add(name.casefold())
ordered = [by_code[code] for code in sorted(by_code)]
members = tuple(
ScopeMember(
name=str(feature["properties"]["NAAM"]).strip(),
nis_code=str(feature["properties"]["NISCODE"]).strip(),
)
for feature in ordered
)
scope = GeographicScope(
key="flanders",
display_name=f"Vlaanderen ({len(members)} gemeenten)",
project_name="Flanders Regional Workbench",
project_region="Vlaanderen, België",
area_name="Vlaanderen - officiële operationele grens",
authority_name="Digitaal Vlaanderen VRBG",
authority_url=VRBG_ITEMS_URL,
scope_type="region",
limitation_message=(
"Officiële unie van de actuele VRBG-gemeentegrenzen. De operationele regio omvat het Vlaamse "
"landgebied; territoriale zee, EEZ en continentaal plat zijn afzonderlijke maritieme scopes."
),
members=members,
)
return scope, ordered, response.url
def prepare_artifacts(
args: argparse.Namespace,
scope: GeographicScope,
features: list[dict[str, Any]],
source_url: str,
) -> tuple[Path, Path, dict[str, Any]]:
output_dir = args.output_root / scope.key
manifest_path = output_dir / "flanders_scope_manifest.json"
generated_at = datetime.now(timezone.utc).isoformat()
snapshot_date = generated_at[:10]
boundary_path = output_dir / f"flanders_boundary_{snapshot_date}.geojson"
members_path = output_dir / f"flanders_municipalities_{snapshot_date}.geojson"
member_codes = list(scope.nis_codes)
if not args.force and manifest_path.is_file():
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
cached_boundary = output_dir / str(manifest.get("boundary_filename") or "")
cached_members = output_dir / str(manifest.get("municipalities_filename") or "")
if (
manifest.get("status") == "complete"
and manifest.get("member_nis_codes") == member_codes
and cached_boundary.is_file()
and cached_members.is_file()
and sha256_file(cached_boundary) == manifest.get("boundary_sha256")
and sha256_file(cached_members) == manifest.get("municipalities_sha256")
):
return cached_boundary, cached_members, manifest
boundary_payload, members_payload, summary = build_scope_payloads(
scope,
features,
source_url=source_url,
generated_at=generated_at,
)
write_json_atomic(boundary_path, boundary_payload)
write_json_atomic(members_path, members_payload)
manifest = {
"schema_version": 1,
"status": "complete",
"generated_at": generated_at,
"observed_at": f"{snapshot_date}T00:00:00Z",
"boundary_filename": boundary_path.name,
"boundary_sha256": sha256_file(boundary_path),
"municipalities_filename": members_path.name,
"municipalities_sha256": sha256_file(members_path),
"vrbg_source_url": source_url,
"vrbg_attribution": VRBG_ATTRIBUTION,
"scope_authority_name": scope.authority_name,
"scope_authority_url": scope.authority_url,
"scope_limitation": scope.limitation_message,
**summary,
}
write_json_atomic(manifest_path, manifest, pretty=True)
return boundary_path, members_path, manifest
def main() -> int:
args = parse_args()
args.operator_tool = "provision_flanders_geographic_scope.py"
try:
with build_source_session() as session:
scope, features, source_url = discover_flanders_scope(
session,
timeout=args.request_timeout,
min_municipalities=args.min_municipalities,
max_municipalities=args.max_municipalities,
)
boundary_path, members_path, manifest = prepare_artifacts(args, scope, features, source_url)
workspace = None if args.fetch_only else provision_scope(
args, scope, boundary_path, members_path, manifest
)
except (OSError, RuntimeError, ValueError, KeyError, requests.RequestException) as exc:
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
return 1
print(
json.dumps(
{
"status": "ok",
"mode": "fetch_only" if args.fetch_only else "provisioned",
"scope": scope.key,
"member_count": manifest["member_count"],
"area_km2": manifest["area_km2"],
"wgs84_bbox": manifest["wgs84_bbox"],
"boundary_path": str(boundary_path),
"municipalities_path": str(members_path),
"workspace": workspace,
"limitation": scope.limitation_message,
},
ensure_ascii=False,
indent=2,
)
)
return 0
if __name__ == "__main__":
raise SystemExit(main())
+1 -1
View File
@@ -478,7 +478,7 @@ def provision_scope(
"attribution": VRBG_ATTRIBUTION, "attribution": VRBG_ATTRIBUTION,
} }
common_provenance = { common_provenance = {
"operator_tool": "provision_geographic_scope.py", "operator_tool": getattr(args, "operator_tool", "provision_geographic_scope.py"),
"operator_explicit_fetch": True, "operator_explicit_fetch": True,
"manifest_path": str(args.output_root / scope.key / f"{scope.key.replace('-', '_')}_scope_manifest.json"), "manifest_path": str(args.output_root / scope.key / f"{scope.key.replace('-', '_')}_scope_manifest.json"),
"source_url": manifest["vrbg_source_url"], "source_url": manifest["vrbg_source_url"],
+3
View File
@@ -51,6 +51,9 @@ ${PYTHON_BIN} -m py_compile scripts/provision_mol_historical_landuse.py
${PYTHON_BIN} -m py_compile scripts/provision_regional_historical_landuse.py ${PYTHON_BIN} -m py_compile scripts/provision_regional_historical_landuse.py
${PYTHON_BIN} -m py_compile scripts/provision_official_landuse_timeseries.py ${PYTHON_BIN} -m py_compile scripts/provision_official_landuse_timeseries.py
${PYTHON_BIN} -m py_compile scripts/provision_waterinfo_station_history.py ${PYTHON_BIN} -m py_compile scripts/provision_waterinfo_station_history.py
${PYTHON_BIN} -m py_compile scripts/provision_flanders_geographic_scope.py
${PYTHON_BIN} -m py_compile scripts/provision_flanders_bathymetry_profiles.py
${PYTHON_BIN} -m py_compile scripts/probe_mdk_bathymetry.py
${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py ${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py
${PYTHON_BIN} -m py_compile scripts/provision_regional_bwk_natura2000.py ${PYTHON_BIN} -m py_compile scripts/provision_regional_bwk_natura2000.py
${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py ${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py