Add Flemish bathymetry partition workflow
This commit is contained in:
@@ -43,6 +43,10 @@ BATHYMETRY_PROFILES_PAGE_SIZE=1000
|
||||
BATHYMETRY_PROFILES_MAX_FEATURES=50000
|
||||
BATHYMETRY_PROFILES_TIMEOUT_SECONDS=120
|
||||
BATHYMETRY_PROFILES_MAX_RESPONSE_MB=32
|
||||
MDK_BATHYMETRY_PROBE_ENABLED=true
|
||||
MDK_BATHYMETRY_WCS_URL=https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs
|
||||
MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS=20
|
||||
MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB=4
|
||||
THEMATIC_RASTER_ENABLED=true
|
||||
THEMATIC_RASTER_WCS_URL=https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs
|
||||
THEMATIC_RASTER_MIN_SIDE_M=100
|
||||
|
||||
@@ -7,6 +7,17 @@
|
||||
|
||||
# Changelog
|
||||
|
||||
## Sprint 236 Flemish bathymetry partitions and safe North Sea probe (2026-07-17)
|
||||
|
||||
- Added dynamic provisioning of the complete official VRBG Flemish
|
||||
municipality inventory, validated at 285 current Areas.
|
||||
- Added an atomic, resumable VHA profile coordinator and server-side manifest
|
||||
finalization that prevents partial partitions from appearing regional.
|
||||
- Added a strict-TLS, size-bounded MDK WCS GetCapabilities readiness probe
|
||||
without coverage download or insecure fallback.
|
||||
- Exposed truthful MDK readiness and bathymetry partition-finalization API
|
||||
contracts and packaged all operator scripts in the all-in-one image.
|
||||
|
||||
## Sprint 235 Governed bathymetry profiles and Belgian scale architecture (2026-07-17)
|
||||
|
||||
- Added bounded official VHA cross-section acquisition with exact Area
|
||||
|
||||
@@ -1728,3 +1728,29 @@ The Dataset exposes profile count and nullable structured depth/width metrics.
|
||||
It does not claim a continuous bed model, current depth or volume. Use
|
||||
`scripts/provision_mol_bathymetry_profiles.py` for the canonical Mol operator
|
||||
flow.
|
||||
|
||||
Provision the complete current Flemish land scope and then run the resumable
|
||||
VHA municipality coordinator:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_flanders_geographic_scope.py
|
||||
docker exec geointel python /app/scripts/provision_flanders_bathymetry_profiles.py
|
||||
```
|
||||
|
||||
The first command discovers all current VRBG RefGem municipalities, validates
|
||||
a 270..300 safety range and persists their exact Areas. The second writes its
|
||||
manifest after every partition. Repeating it reuses completed source
|
||||
identities; `--force` deliberately refreshes them. A partial `--members` or
|
||||
`--max-partitions` run never marks regional coverage complete.
|
||||
|
||||
Inspect the MDK North Sea WCS without downloading coverage:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/probe_mdk_bathymetry.py
|
||||
```
|
||||
|
||||
Exit code `0` means verified capabilities; `2` means a truthful blocked
|
||||
readiness state such as TLS or endpoint failure. Runtime controls are
|
||||
`MDK_BATHYMETRY_PROBE_ENABLED`, `MDK_BATHYMETRY_WCS_URL`,
|
||||
`MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS` and
|
||||
`MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB`. TLS verification cannot be disabled.
|
||||
|
||||
@@ -27,6 +27,7 @@ from app.schemas import (
|
||||
FloodHazardAcquireRequest,
|
||||
FloodHazardPartitionSelectionRequest,
|
||||
FloodHazardSelectionRequest,
|
||||
BathymetryPartitionFinalizeRequest,
|
||||
BathymetryProfileAcquireRequest,
|
||||
ThematicRasterAcquireRequest,
|
||||
ThematicRasterSelectionRequest,
|
||||
@@ -54,6 +55,7 @@ from app.services.terrain_analysis_service import TerrainAnalysisService
|
||||
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
|
||||
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
|
||||
from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
|
||||
from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService
|
||||
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
|
||||
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
|
||||
from app.utils.response import envelope
|
||||
@@ -223,6 +225,13 @@ def list_bathymetry_sources(project_id: UUID, db: Session = Depends(get_db)):
|
||||
return envelope({"items": items, "total": len(items)})
|
||||
|
||||
|
||||
@router.get("/datasets/bathymetry/sources/mdk_bcp_bathymetry/readiness", response_model=dict)
|
||||
def probe_mdk_bathymetry_readiness(project_id: UUID, db: Session = Depends(get_db)):
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
return envelope(MdkBathymetryProbeService.probe())
|
||||
|
||||
|
||||
@router.post("/datasets/bathymetry/profiles/acquire", response_model=dict)
|
||||
def acquire_bounded_bathymetry_profiles(
|
||||
project_id: UUID,
|
||||
@@ -239,6 +248,15 @@ def acquire_bounded_bathymetry_profiles(
|
||||
return envelope(job)
|
||||
|
||||
|
||||
@router.post("/datasets/bathymetry/profiles/partitions/finalize", response_model=dict)
|
||||
def finalize_bathymetry_profile_partitions(
|
||||
project_id: UUID,
|
||||
payload: BathymetryPartitionFinalizeRequest,
|
||||
db: Session = Depends(get_db),
|
||||
):
|
||||
return envelope(BathymetryProfileAcquisitionService.finalize_partitions(db, project_id, payload))
|
||||
|
||||
|
||||
@router.post("/datasets/thematic-raster/acquire", response_model=dict)
|
||||
def acquire_bounded_thematic_raster(
|
||||
project_id: UUID,
|
||||
|
||||
@@ -123,6 +123,23 @@ class Settings(BaseSettings):
|
||||
le=256,
|
||||
validation_alias="BATHYMETRY_PROFILES_MAX_RESPONSE_MB",
|
||||
)
|
||||
mdk_bathymetry_probe_enabled: bool = Field(default=True, validation_alias="MDK_BATHYMETRY_PROBE_ENABLED")
|
||||
mdk_bathymetry_wcs_url: str = Field(
|
||||
default="https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs",
|
||||
validation_alias="MDK_BATHYMETRY_WCS_URL",
|
||||
)
|
||||
mdk_bathymetry_probe_timeout_seconds: int = Field(
|
||||
default=20,
|
||||
ge=1,
|
||||
le=120,
|
||||
validation_alias="MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS",
|
||||
)
|
||||
mdk_bathymetry_probe_max_response_mb: int = Field(
|
||||
default=4,
|
||||
ge=1,
|
||||
le=16,
|
||||
validation_alias="MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB",
|
||||
)
|
||||
thematic_raster_enabled: bool = Field(default=True, validation_alias="THEMATIC_RASTER_ENABLED")
|
||||
thematic_raster_wcs_url: str = Field(
|
||||
default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs",
|
||||
|
||||
@@ -66,8 +66,11 @@ from .flood_hazard import (
|
||||
FloodHazardSelectionSummary,
|
||||
)
|
||||
from .bathymetry import (
|
||||
BathymetryPartitionFinalizeRequest,
|
||||
BathymetryPartitionFinalizationResult,
|
||||
BathymetryProfileAcquireRequest,
|
||||
BathymetryProfileAcquisitionResult,
|
||||
BathymetrySourceProbeRead,
|
||||
BathymetrySourceRead,
|
||||
)
|
||||
from .thematic_raster import (
|
||||
@@ -209,6 +212,9 @@ __all__ = [
|
||||
"FloodHazardSelectionSummary",
|
||||
"BathymetryProfileAcquireRequest",
|
||||
"BathymetryProfileAcquisitionResult",
|
||||
"BathymetryPartitionFinalizeRequest",
|
||||
"BathymetryPartitionFinalizationResult",
|
||||
"BathymetrySourceProbeRead",
|
||||
"BathymetrySourceRead",
|
||||
"ThematicRasterAcquireRequest",
|
||||
"ThematicRasterAcquisitionResult",
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from typing import Literal
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic import BaseModel, Field, field_validator
|
||||
|
||||
from .operations import VectorSelectionBBox
|
||||
|
||||
@@ -25,7 +26,7 @@ class BathymetrySourceRead(BaseModel):
|
||||
vertical_reference: str
|
||||
horizontal_crs: str
|
||||
native_resolution: str | None = None
|
||||
integration_status: Literal["operational", "available_not_integrated", "catalog_only"]
|
||||
integration_status: Literal["operational", "probe_only", "available_not_integrated", "catalog_only"]
|
||||
acquisition_supported: bool
|
||||
configured: bool
|
||||
service_url: str | None = None
|
||||
@@ -50,3 +51,61 @@ class BathymetryProfileAcquisitionResult(BaseModel):
|
||||
measurement_date_max: str | None = None
|
||||
attribution: str
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class BathymetryPartitionFinalizeRequest(BaseModel):
|
||||
partition_scope_key: str = Field(min_length=1, max_length=120, pattern=r"^[a-z0-9][a-z0-9_-]*$")
|
||||
expected_area_ids: list[UUID] = Field(min_length=1, max_length=500)
|
||||
dataset_ids: list[UUID] = Field(default_factory=list, max_length=500)
|
||||
no_profile_area_ids: list[UUID] = Field(default_factory=list, max_length=500)
|
||||
manifest_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||
observed_at: datetime
|
||||
|
||||
@field_validator("expected_area_ids", "dataset_ids", "no_profile_area_ids")
|
||||
@classmethod
|
||||
def require_unique_ids(cls, value: list[UUID]) -> list[UUID]:
|
||||
if len(value) != len(set(value)):
|
||||
raise ValueError("Partition identifiers must be unique")
|
||||
return value
|
||||
|
||||
|
||||
class BathymetryPartitionFinalizationResult(BaseModel):
|
||||
partition_scope_key: str
|
||||
regional_partitions_complete: bool
|
||||
partition_count: int = Field(ge=1)
|
||||
data_partition_count: int = Field(ge=0)
|
||||
no_profile_partition_count: int = Field(ge=0)
|
||||
profile_count: int = Field(ge=0)
|
||||
document_count: int = Field(ge=0)
|
||||
structured_depth_count: int = Field(ge=0)
|
||||
measurement_date_min: str | None = None
|
||||
measurement_date_max: str | None = None
|
||||
dataset_ids: list[UUID]
|
||||
manifest_sha256: str
|
||||
observed_at: datetime
|
||||
limitation_message: str
|
||||
|
||||
|
||||
class BathymetrySourceProbeRead(BaseModel):
|
||||
source_key: str
|
||||
status: Literal[
|
||||
"disabled",
|
||||
"invalid_configuration",
|
||||
"tls_error",
|
||||
"endpoint_unavailable",
|
||||
"invalid_capabilities",
|
||||
"reachable",
|
||||
]
|
||||
configured_url: str
|
||||
capabilities_url: str | None = None
|
||||
tls_verified: bool
|
||||
capabilities_reachable: bool
|
||||
acquisition_supported: bool = False
|
||||
wcs_version: str | None = None
|
||||
coverage_identifiers: list[str] = Field(default_factory=list)
|
||||
advertised_formats: list[str] = Field(default_factory=list)
|
||||
advertised_crs: list[str] = Field(default_factory=list)
|
||||
response_sha256: str | None = None
|
||||
checked_at: datetime
|
||||
message: str
|
||||
limitation_message: str
|
||||
|
||||
@@ -15,8 +15,10 @@ from shapely.geometry import Point, box, mapping
|
||||
|
||||
from app.core.config import Settings, get_settings
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset, Project
|
||||
from app.models import Area, Dataset, DatasetVersion, Project
|
||||
from app.schemas.bathymetry import (
|
||||
BathymetryPartitionFinalizeRequest,
|
||||
BathymetryPartitionFinalizationResult,
|
||||
BathymetryProfileAcquireRequest,
|
||||
BathymetryProfileAcquisitionResult,
|
||||
BathymetrySourceRead,
|
||||
@@ -70,16 +72,16 @@ class BathymetryProfileAcquisitionService:
|
||||
"vertical_reference": "LAT",
|
||||
"horizontal_crs": "bronafhankelijk; expliciet per WCS-respons",
|
||||
"native_resolution": "20 x 20 m",
|
||||
"integration_status": "available_not_integrated",
|
||||
"integration_status": "probe_only",
|
||||
"acquisition_supported": False,
|
||||
"configured": False,
|
||||
"service_url": "https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/WCS_Public",
|
||||
"service_url": "https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs",
|
||||
"catalog_url": "https://www.vlaanderen.be/datavindplaats/catalogus/dieptemodel-van-de-zeebodem-belgisch-continentaal-plat-noordzee",
|
||||
"attribution": "Agentschap Maritieme Dienstverlening en Kust",
|
||||
"license_note": "Zie de officiële datasetmetadata en gebruiksvoorwaarden.",
|
||||
"limitation_message": (
|
||||
"Niet operationeel in GeoIntel totdat WCS, maritieme begrenzing, tegels, LAT-semantiek en "
|
||||
"servercertificaten in een live integratietest zijn gevalideerd."
|
||||
"Alleen een read-only GetCapabilities-probe is beschikbaar. Rasteracquisitie blijft uit totdat "
|
||||
"WCS, maritieme begrenzing, tegels, LAT-semantiek en servercertificaten live zijn gevalideerd."
|
||||
),
|
||||
},
|
||||
{
|
||||
@@ -404,6 +406,8 @@ class BathymetryProfileAcquisitionService:
|
||||
raw_features: list[dict[str, Any]],
|
||||
scope_geometry,
|
||||
watercourse_names: dict[int, dict[str, str | None]],
|
||||
*,
|
||||
partition_properties: dict[str, Any] | None = None,
|
||||
) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
normalized: list[dict[str, Any]] = []
|
||||
dates: list[str] = []
|
||||
@@ -465,6 +469,7 @@ class BathymetryProfileAcquisitionService:
|
||||
"provider": BathymetryProfileAcquisitionService.PROVIDER,
|
||||
"measurement_semantics": "historical_cross_section_profile_point",
|
||||
"vertical_reference": "document-specific",
|
||||
**(partition_properties or {}),
|
||||
},
|
||||
"geometry": mapping(point),
|
||||
}
|
||||
@@ -487,6 +492,17 @@ class BathymetryProfileAcquisitionService:
|
||||
},
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _municipality_name(area: Area | None) -> str | None:
|
||||
if area is None:
|
||||
return None
|
||||
normalized = str(area.name or "").strip()
|
||||
prefix = "Gemeente "
|
||||
if not normalized.casefold().startswith(prefix.casefold()):
|
||||
return None
|
||||
municipality = normalized[len(prefix) :].split(" - ", 1)[0].strip()
|
||||
return municipality or None
|
||||
|
||||
@staticmethod
|
||||
def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None:
|
||||
return (
|
||||
@@ -541,6 +557,8 @@ class BathymetryProfileAcquisitionService:
|
||||
scope_geometry = BathymetryProfileAcquisitionService._scope_geometry(
|
||||
db, project_id, payload.area_id, bbox_values
|
||||
)
|
||||
area = db.get(Area, payload.area_id) if payload.area_id else None
|
||||
municipality = BathymetryProfileAcquisitionService._municipality_name(area)
|
||||
exact_bbox = tuple(float(value) for value in scope_geometry.bounds)
|
||||
request_identity = {
|
||||
"provider": BathymetryProfileAcquisitionService.PROVIDER,
|
||||
@@ -568,7 +586,16 @@ class BathymetryProfileAcquisitionService:
|
||||
vhag_codes, resolved_settings, opener
|
||||
)
|
||||
feature_collection, summary = BathymetryProfileAcquisitionService._normalize_features(
|
||||
raw_features, scope_geometry, names
|
||||
raw_features,
|
||||
scope_geometry,
|
||||
names,
|
||||
partition_properties={
|
||||
"partition_area_id": str(area.id),
|
||||
"partition_area_name": area.name,
|
||||
**({"municipality": municipality} if municipality else {}),
|
||||
}
|
||||
if area
|
||||
else None,
|
||||
)
|
||||
if summary["profile_count"] == 0:
|
||||
raise AppError(
|
||||
@@ -585,6 +612,11 @@ class BathymetryProfileAcquisitionService:
|
||||
"theme": "bathymetry",
|
||||
"layer_name": "bathymetry_profiles",
|
||||
"coverage_scope": "municipality" if payload.area_id else "bounded_selection",
|
||||
"partition_area_id": str(area.id) if area else None,
|
||||
"partition_area_name": area.name if area else None,
|
||||
"municipality": municipality,
|
||||
"partitioned_source_audit": False,
|
||||
"regional_partitions_complete": False,
|
||||
"bbox_epsg4326": list(exact_bbox),
|
||||
**summary,
|
||||
"selection_aggregation": {
|
||||
@@ -669,3 +701,160 @@ class BathymetryProfileAcquisitionService:
|
||||
)
|
||||
persisted = db.get(Dataset, dataset.id)
|
||||
return BathymetryProfileAcquisitionService._result(persisted, reused=False)
|
||||
|
||||
@staticmethod
|
||||
def finalize_partitions(
|
||||
db,
|
||||
project_id: UUID,
|
||||
payload: BathymetryPartitionFinalizeRequest,
|
||||
) -> dict[str, Any]:
|
||||
if not db.get(Project, project_id):
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
|
||||
expected_area_ids = set(payload.expected_area_ids)
|
||||
dataset_ids = set(payload.dataset_ids)
|
||||
no_profile_area_ids = set(payload.no_profile_area_ids)
|
||||
|
||||
areas: dict[UUID, Area] = {}
|
||||
for area_id in expected_area_ids:
|
||||
area = db.get(Area, area_id)
|
||||
if area is None or area.project_id != project_id:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PARTITION_AREA_INVALID",
|
||||
message="Every expected partition Area must belong to the project",
|
||||
details={"area_id": str(area_id)},
|
||||
status_code=400,
|
||||
)
|
||||
if BathymetryProfileAcquisitionService._municipality_name(area) is None:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PARTITION_AREA_INVALID",
|
||||
message="Bathymetry partitions must use persisted municipality Areas",
|
||||
details={"area_id": str(area_id), "area_name": area.name},
|
||||
status_code=400,
|
||||
)
|
||||
areas[area_id] = area
|
||||
|
||||
if not no_profile_area_ids.issubset(expected_area_ids):
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PARTITION_MANIFEST_INVALID",
|
||||
message="No-profile partitions must be part of the expected Area set",
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
datasets: list[Dataset] = []
|
||||
data_area_ids: set[UUID] = set()
|
||||
for dataset_id in payload.dataset_ids:
|
||||
dataset = db.get(Dataset, dataset_id)
|
||||
if (
|
||||
dataset is None
|
||||
or dataset.project_id != project_id
|
||||
or dataset.source_name != BathymetryProfileAcquisitionService.PROVIDER
|
||||
or dataset.status != "ready"
|
||||
or dataset.area_id not in expected_area_ids
|
||||
):
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PARTITION_DATASET_INVALID",
|
||||
message="Every partition Dataset must be a ready VHA profile Dataset scoped to an expected Area",
|
||||
details={"dataset_id": str(dataset_id)},
|
||||
status_code=400,
|
||||
)
|
||||
if dataset.area_id in data_area_ids:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PARTITION_DATASET_DUPLICATE",
|
||||
message="A complete manifest may reference only one profile Dataset per Area",
|
||||
details={"area_id": str(dataset.area_id)},
|
||||
status_code=400,
|
||||
)
|
||||
data_area_ids.add(dataset.area_id)
|
||||
datasets.append(dataset)
|
||||
|
||||
accounted_area_ids = data_area_ids.union(no_profile_area_ids)
|
||||
if accounted_area_ids != expected_area_ids:
|
||||
raise AppError(
|
||||
code="BATHYMETRY_PARTITION_MANIFEST_INCOMPLETE",
|
||||
message="Every expected Area must have one ready Dataset or an explicit no-profile result",
|
||||
details={
|
||||
"missing_area_ids": sorted(str(value) for value in expected_area_ids - accounted_area_ids),
|
||||
"unexpected_area_ids": sorted(str(value) for value in accounted_area_ids - expected_area_ids),
|
||||
},
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
profile_count = 0
|
||||
document_count = 0
|
||||
structured_depth_count = 0
|
||||
dates_min: list[str] = []
|
||||
dates_max: list[str] = []
|
||||
shared_metadata = {
|
||||
"partition_scope_key": payload.partition_scope_key,
|
||||
"partition_count": len(expected_area_ids),
|
||||
"data_partition_count": len(datasets),
|
||||
"no_profile_partition_count": len(no_profile_area_ids),
|
||||
"partition_manifest_sha256": payload.manifest_sha256,
|
||||
"partition_manifest_observed_at": payload.observed_at.isoformat(),
|
||||
"partitioned_source_audit": True,
|
||||
"regional_partitions_complete": True,
|
||||
}
|
||||
shared_provenance = {
|
||||
"partition_manifest_sha256": payload.manifest_sha256,
|
||||
"partition_manifest_observed_at": payload.observed_at.isoformat(),
|
||||
"partition_scope_key": payload.partition_scope_key,
|
||||
"regional_partitions_complete": True,
|
||||
"no_profile_area_ids": sorted(str(value) for value in no_profile_area_ids),
|
||||
}
|
||||
|
||||
for dataset in datasets:
|
||||
source_metadata = dict(dataset.source_metadata or {})
|
||||
provenance_metadata = dict(dataset.provenance_metadata or {})
|
||||
profile_count += int(source_metadata.get("profile_count") or 0)
|
||||
document_count += int(source_metadata.get("document_count") or 0)
|
||||
structured_depth_count += int(source_metadata.get("structured_depth_count") or 0)
|
||||
if source_metadata.get("measurement_date_min"):
|
||||
dates_min.append(str(source_metadata["measurement_date_min"]))
|
||||
if source_metadata.get("measurement_date_max"):
|
||||
dates_max.append(str(source_metadata["measurement_date_max"]))
|
||||
area = areas[dataset.area_id]
|
||||
source_metadata.update(
|
||||
{
|
||||
**shared_metadata,
|
||||
"coverage_scope": payload.partition_scope_key,
|
||||
"partition_area_id": str(area.id),
|
||||
"partition_area_name": area.name,
|
||||
"municipality": BathymetryProfileAcquisitionService._municipality_name(area),
|
||||
}
|
||||
)
|
||||
provenance_metadata.update(shared_provenance)
|
||||
dataset.source_metadata = source_metadata
|
||||
dataset.provenance_metadata = provenance_metadata
|
||||
|
||||
if dataset_ids:
|
||||
versions = (
|
||||
db.query(DatasetVersion)
|
||||
.filter(DatasetVersion.dataset_id.in_(dataset_ids))
|
||||
.all()
|
||||
)
|
||||
for version in versions:
|
||||
version.source_metadata = dict(
|
||||
next(dataset.source_metadata for dataset in datasets if dataset.id == version.dataset_id)
|
||||
)
|
||||
version.provenance_metadata = dict(
|
||||
next(dataset.provenance_metadata for dataset in datasets if dataset.id == version.dataset_id)
|
||||
)
|
||||
|
||||
db.commit()
|
||||
return BathymetryPartitionFinalizationResult(
|
||||
partition_scope_key=payload.partition_scope_key,
|
||||
regional_partitions_complete=True,
|
||||
partition_count=len(expected_area_ids),
|
||||
data_partition_count=len(datasets),
|
||||
no_profile_partition_count=len(no_profile_area_ids),
|
||||
profile_count=profile_count,
|
||||
document_count=document_count,
|
||||
structured_depth_count=structured_depth_count,
|
||||
measurement_date_min=min(dates_min) if dates_min else None,
|
||||
measurement_date_max=max(dates_max) if dates_max else None,
|
||||
dataset_ids=payload.dataset_ids,
|
||||
manifest_sha256=payload.manifest_sha256,
|
||||
observed_at=payload.observed_at,
|
||||
limitation_message=BathymetryProfileAcquisitionService.LIMITATION,
|
||||
).model_dump(mode="json")
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
import hashlib
|
||||
import ssl
|
||||
from typing import Any, Callable
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
from urllib.request import Request, urlopen
|
||||
from xml.etree import ElementTree
|
||||
|
||||
from app.core.config import Settings, get_settings
|
||||
from app.schemas.bathymetry import BathymetrySourceProbeRead
|
||||
|
||||
|
||||
class MdkBathymetryProbeService:
|
||||
SOURCE_KEY = "mdk_bcp_bathymetry"
|
||||
LIMITATION = (
|
||||
"Deze probe leest alleen WCS GetCapabilities met strikte TLS-controle. "
|
||||
"GeoIntel downloadt of activeert geen Noordzee-raster totdat endpoint, coverage-id, CRS, LAT, "
|
||||
"nodata, resolutie, begrenzing en responslimieten live zijn gevalideerd."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _capabilities_url(configured_url: str) -> str:
|
||||
parsed = urlsplit(configured_url.strip())
|
||||
if parsed.scheme.lower() != "https" or not parsed.hostname:
|
||||
raise ValueError("The MDK WCS probe requires an absolute HTTPS URL")
|
||||
parameters = dict(parse_qsl(parsed.query, keep_blank_values=True))
|
||||
parameters.update({"service": "WCS", "request": "GetCapabilities", "version": "1.0.0"})
|
||||
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(parameters), ""))
|
||||
|
||||
@staticmethod
|
||||
def _read_capabilities(
|
||||
capabilities_url: str,
|
||||
settings: Settings,
|
||||
opener: Callable[..., Any] | None,
|
||||
) -> tuple[bytes, str]:
|
||||
request = Request(
|
||||
capabilities_url,
|
||||
headers={
|
||||
"Accept": "application/xml,text/xml;q=0.9,*/*;q=0.1",
|
||||
"User-Agent": "GeoIntel/1.0 MDK-bathymetry-readiness-probe",
|
||||
},
|
||||
)
|
||||
with (opener or urlopen)(request, timeout=settings.mdk_bathymetry_probe_timeout_seconds) as response:
|
||||
limit = settings.mdk_bathymetry_probe_max_response_mb * 1024 * 1024
|
||||
content = response.read(limit + 1)
|
||||
if len(content) > limit:
|
||||
raise ValueError("MDK WCS capabilities response exceeded the configured size limit")
|
||||
content_type = str(response.headers.get("Content-Type") or "") if hasattr(response, "headers") else ""
|
||||
return content, content_type
|
||||
|
||||
@staticmethod
|
||||
def _local_name(tag: str) -> str:
|
||||
return tag.rsplit("}", 1)[-1].casefold()
|
||||
|
||||
@staticmethod
|
||||
def _parse_capabilities(content: bytes) -> dict[str, Any]:
|
||||
root = ElementTree.fromstring(content)
|
||||
root_name = MdkBathymetryProbeService._local_name(root.tag)
|
||||
if "capabilities" not in root_name:
|
||||
raise ValueError("MDK endpoint did not return a WCS capabilities document")
|
||||
|
||||
coverage_identifiers: set[str] = set()
|
||||
advertised_formats: set[str] = set()
|
||||
advertised_crs: set[str] = set()
|
||||
for element in root.iter():
|
||||
local_name = MdkBathymetryProbeService._local_name(element.tag)
|
||||
text = (element.text or "").strip()
|
||||
if local_name in {"coverageofferingbrief", "coverageoffering"}:
|
||||
for child in element:
|
||||
if MdkBathymetryProbeService._local_name(child.tag) in {"name", "identifier"}:
|
||||
identifier = (child.text or "").strip()
|
||||
if identifier:
|
||||
coverage_identifiers.add(identifier)
|
||||
break
|
||||
if local_name in {"format", "formats"} and text:
|
||||
advertised_formats.add(text)
|
||||
if local_name in {"requestresponsecrss", "requestcrss", "responsecrss", "nativecrss", "crs"} and text:
|
||||
advertised_crs.add(text)
|
||||
for attribute_value in element.attrib.values():
|
||||
normalized = str(attribute_value).strip()
|
||||
if "EPSG" in normalized.upper() or "CRS:" in normalized.upper():
|
||||
advertised_crs.add(normalized)
|
||||
|
||||
return {
|
||||
"wcs_version": str(root.attrib.get("version") or "") or None,
|
||||
"coverage_identifiers": sorted(coverage_identifiers),
|
||||
"advertised_formats": sorted(advertised_formats),
|
||||
"advertised_crs": sorted(advertised_crs),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _result(
|
||||
*,
|
||||
settings: Settings,
|
||||
status: str,
|
||||
checked_at: datetime,
|
||||
message: str,
|
||||
capabilities_url: str | None = None,
|
||||
tls_verified: bool = False,
|
||||
capabilities_reachable: bool = False,
|
||||
response_sha256: str | None = None,
|
||||
parsed: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
parsed = parsed or {}
|
||||
return BathymetrySourceProbeRead(
|
||||
source_key=MdkBathymetryProbeService.SOURCE_KEY,
|
||||
status=status,
|
||||
configured_url=settings.mdk_bathymetry_wcs_url,
|
||||
capabilities_url=capabilities_url,
|
||||
tls_verified=tls_verified,
|
||||
capabilities_reachable=capabilities_reachable,
|
||||
acquisition_supported=False,
|
||||
wcs_version=parsed.get("wcs_version"),
|
||||
coverage_identifiers=parsed.get("coverage_identifiers") or [],
|
||||
advertised_formats=parsed.get("advertised_formats") or [],
|
||||
advertised_crs=parsed.get("advertised_crs") or [],
|
||||
response_sha256=response_sha256,
|
||||
checked_at=checked_at,
|
||||
message=message,
|
||||
limitation_message=MdkBathymetryProbeService.LIMITATION,
|
||||
).model_dump(mode="json")
|
||||
|
||||
@staticmethod
|
||||
def probe(
|
||||
*,
|
||||
settings: Settings | None = None,
|
||||
opener: Callable[..., Any] | None = None,
|
||||
checked_at: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
resolved_settings = settings or get_settings()
|
||||
now = checked_at or datetime.now(UTC)
|
||||
if not resolved_settings.mdk_bathymetry_probe_enabled:
|
||||
return MdkBathymetryProbeService._result(
|
||||
settings=resolved_settings,
|
||||
status="disabled",
|
||||
checked_at=now,
|
||||
message="MDK bathymetry readiness probing is disabled.",
|
||||
)
|
||||
try:
|
||||
capabilities_url = MdkBathymetryProbeService._capabilities_url(
|
||||
resolved_settings.mdk_bathymetry_wcs_url
|
||||
)
|
||||
except ValueError as exc:
|
||||
return MdkBathymetryProbeService._result(
|
||||
settings=resolved_settings,
|
||||
status="invalid_configuration",
|
||||
checked_at=now,
|
||||
message=str(exc),
|
||||
)
|
||||
|
||||
try:
|
||||
content, content_type = MdkBathymetryProbeService._read_capabilities(
|
||||
capabilities_url, resolved_settings, opener
|
||||
)
|
||||
except HTTPError as exc:
|
||||
return MdkBathymetryProbeService._result(
|
||||
settings=resolved_settings,
|
||||
status="endpoint_unavailable",
|
||||
checked_at=now,
|
||||
capabilities_url=capabilities_url,
|
||||
tls_verified=True,
|
||||
message=f"MDK WCS GetCapabilities returned HTTP {exc.code}.",
|
||||
)
|
||||
except ssl.SSLCertVerificationError:
|
||||
return MdkBathymetryProbeService._result(
|
||||
settings=resolved_settings,
|
||||
status="tls_error",
|
||||
checked_at=now,
|
||||
capabilities_url=capabilities_url,
|
||||
message="MDK WCS TLS certificate validation failed; insecure fallback is prohibited.",
|
||||
)
|
||||
except URLError as exc:
|
||||
reason = exc.reason
|
||||
is_tls_error = isinstance(reason, (ssl.SSLError, ssl.CertificateError)) or "certificate" in str(
|
||||
reason
|
||||
).casefold()
|
||||
return MdkBathymetryProbeService._result(
|
||||
settings=resolved_settings,
|
||||
status="tls_error" if is_tls_error else "endpoint_unavailable",
|
||||
checked_at=now,
|
||||
capabilities_url=capabilities_url,
|
||||
message=(
|
||||
"MDK WCS TLS certificate validation failed; insecure fallback is prohibited."
|
||||
if is_tls_error
|
||||
else "MDK WCS GetCapabilities could not be reached."
|
||||
),
|
||||
)
|
||||
except (TimeoutError, OSError) as exc:
|
||||
return MdkBathymetryProbeService._result(
|
||||
settings=resolved_settings,
|
||||
status="endpoint_unavailable",
|
||||
checked_at=now,
|
||||
capabilities_url=capabilities_url,
|
||||
message=f"MDK WCS GetCapabilities could not be reached ({type(exc).__name__}).",
|
||||
)
|
||||
except ValueError as exc:
|
||||
return MdkBathymetryProbeService._result(
|
||||
settings=resolved_settings,
|
||||
status="invalid_capabilities",
|
||||
checked_at=now,
|
||||
capabilities_url=capabilities_url,
|
||||
tls_verified=True,
|
||||
capabilities_reachable=True,
|
||||
message=str(exc),
|
||||
)
|
||||
|
||||
response_sha256 = hashlib.sha256(content).hexdigest()
|
||||
try:
|
||||
parsed = MdkBathymetryProbeService._parse_capabilities(content)
|
||||
except (ElementTree.ParseError, ValueError) as exc:
|
||||
detail = " ".join(str(exc).split())
|
||||
if content_type:
|
||||
detail = f"{detail} Content-Type: {content_type}."
|
||||
return MdkBathymetryProbeService._result(
|
||||
settings=resolved_settings,
|
||||
status="invalid_capabilities",
|
||||
checked_at=now,
|
||||
capabilities_url=capabilities_url,
|
||||
tls_verified=True,
|
||||
capabilities_reachable=True,
|
||||
response_sha256=response_sha256,
|
||||
message=detail,
|
||||
)
|
||||
|
||||
if not parsed["coverage_identifiers"]:
|
||||
return MdkBathymetryProbeService._result(
|
||||
settings=resolved_settings,
|
||||
status="invalid_capabilities",
|
||||
checked_at=now,
|
||||
capabilities_url=capabilities_url,
|
||||
tls_verified=True,
|
||||
capabilities_reachable=True,
|
||||
response_sha256=response_sha256,
|
||||
parsed=parsed,
|
||||
message="MDK WCS capabilities are reachable but advertise no coverage identifier.",
|
||||
)
|
||||
return MdkBathymetryProbeService._result(
|
||||
settings=resolved_settings,
|
||||
status="reachable",
|
||||
checked_at=now,
|
||||
capabilities_url=capabilities_url,
|
||||
tls_verified=True,
|
||||
capabilities_reachable=True,
|
||||
response_sha256=response_sha256,
|
||||
parsed=parsed,
|
||||
message=(
|
||||
"MDK WCS capabilities are reachable with verified TLS. "
|
||||
"Raster acquisition remains disabled pending bounded coverage validation."
|
||||
),
|
||||
)
|
||||
@@ -282,8 +282,11 @@ def test_bathymetry_acquisition_persists_reference_dataset_through_dataset_servi
|
||||
assert captured["reference_layer_name"] == "bathymetry_profiles"
|
||||
assert captured["source_metadata"]["theme"] == "bathymetry"
|
||||
assert captured["source_metadata"]["volume_supported"] is False
|
||||
assert captured["provenance_metadata"]["water_volume_available"] is False
|
||||
assert captured["source_metadata"]["municipality"] == "Mol"
|
||||
assert captured["source_metadata"]["regional_partitions_complete"] is False
|
||||
payload = json.loads(captured["content"])
|
||||
assert payload["features"][0]["properties"]["municipality"] == "Mol"
|
||||
assert captured["provenance_metadata"]["water_volume_available"] is False
|
||||
assert len(payload["features"]) == 2
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
import json
|
||||
from pathlib import Path
|
||||
import ssl
|
||||
import sys
|
||||
from types import SimpleNamespace
|
||||
from urllib.error import URLError
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
import pytest
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import AppError
|
||||
from app.db.session import get_db
|
||||
from app.main import app
|
||||
from app.models import Area, Dataset, DatasetVersion, Project
|
||||
from app.schemas.bathymetry import BathymetryPartitionFinalizeRequest
|
||||
from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
|
||||
from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
import provision_flanders_geographic_scope as flanders_scope # noqa: E402
|
||||
|
||||
|
||||
class BinaryResponse:
|
||||
def __init__(self, content: bytes, *, content_type: str = "application/xml"):
|
||||
self.content = content
|
||||
self.headers = {"Content-Type": content_type}
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return False
|
||||
|
||||
def read(self, size=-1):
|
||||
return self.content if size < 0 else self.content[:size]
|
||||
|
||||
|
||||
class JsonResponse:
|
||||
def __init__(self, payload, *, url="https://geo.api.vlaanderen.be/VRBG/items"):
|
||||
self.payload = payload
|
||||
self.url = url
|
||||
|
||||
def raise_for_status(self):
|
||||
return None
|
||||
|
||||
def json(self):
|
||||
return self.payload
|
||||
|
||||
|
||||
class SourceSession:
|
||||
def __init__(self, payload):
|
||||
self.payload = payload
|
||||
|
||||
def get(self, *_args, **_kwargs):
|
||||
return JsonResponse(self.payload)
|
||||
|
||||
|
||||
class VersionQuery:
|
||||
def __init__(self, versions):
|
||||
self.versions = versions
|
||||
|
||||
def filter(self, *_args):
|
||||
return self
|
||||
|
||||
def all(self):
|
||||
return self.versions
|
||||
|
||||
|
||||
class FinalizeSession:
|
||||
def __init__(self, rows, versions=None):
|
||||
self.rows = rows
|
||||
self.versions = versions or []
|
||||
self.commit_count = 0
|
||||
|
||||
def get(self, model, row_id):
|
||||
return self.rows.get((model, row_id))
|
||||
|
||||
def query(self, model):
|
||||
assert model is DatasetVersion
|
||||
return VersionQuery(self.versions)
|
||||
|
||||
def commit(self):
|
||||
self.commit_count += 1
|
||||
|
||||
|
||||
def capabilities_xml() -> bytes:
|
||||
return b"""<?xml version="1.0"?>
|
||||
<WCS_Capabilities version="1.0.0" xmlns="http://www.opengis.net/wcs">
|
||||
<ContentMetadata>
|
||||
<CoverageOfferingBrief>
|
||||
<name>EL.GridCoverage</name>
|
||||
<label>Belgian Continental Shelf bathymetry</label>
|
||||
<lonLatEnvelope srsName="urn:ogc:def:crs:OGC:1.3:CRS84" />
|
||||
</CoverageOfferingBrief>
|
||||
</ContentMetadata>
|
||||
<Capability><Request><GetCoverage><resultFormat><formats>GeoTIFF</formats></resultFormat></GetCoverage></Request></Capability>
|
||||
</WCS_Capabilities>"""
|
||||
|
||||
|
||||
def test_mdk_probe_parses_capabilities_without_enabling_acquisition() -> None:
|
||||
seen = {}
|
||||
|
||||
def opener(request, timeout):
|
||||
seen["url"] = request.full_url
|
||||
seen["timeout"] = timeout
|
||||
return BinaryResponse(capabilities_xml())
|
||||
|
||||
result = MdkBathymetryProbeService.probe(
|
||||
settings=Settings(_env_file=None),
|
||||
opener=opener,
|
||||
checked_at=datetime(2026, 7, 17, tzinfo=UTC),
|
||||
)
|
||||
|
||||
assert result["status"] == "reachable"
|
||||
assert result["tls_verified"] is True
|
||||
assert result["capabilities_reachable"] is True
|
||||
assert result["acquisition_supported"] is False
|
||||
assert result["coverage_identifiers"] == ["EL.GridCoverage"]
|
||||
assert result["advertised_formats"] == ["GeoTIFF"]
|
||||
assert result["response_sha256"]
|
||||
assert "request=GetCapabilities" in seen["url"]
|
||||
assert seen["timeout"] == 20
|
||||
|
||||
|
||||
def test_mdk_probe_reports_tls_failure_and_never_uses_insecure_fallback() -> None:
|
||||
calls = 0
|
||||
|
||||
def opener(_request, timeout):
|
||||
nonlocal calls
|
||||
assert timeout == 20
|
||||
calls += 1
|
||||
raise URLError(ssl.SSLCertVerificationError("hostname mismatch"))
|
||||
|
||||
result = MdkBathymetryProbeService.probe(
|
||||
settings=Settings(_env_file=None),
|
||||
opener=opener,
|
||||
)
|
||||
|
||||
assert calls == 1
|
||||
assert result["status"] == "tls_error"
|
||||
assert result["tls_verified"] is False
|
||||
assert result["capabilities_reachable"] is False
|
||||
assert "insecure fallback is prohibited" in result["message"]
|
||||
|
||||
|
||||
def test_mdk_readiness_api_uses_canonical_envelope(monkeypatch) -> None:
|
||||
project_id = uuid4()
|
||||
db = SimpleNamespace(get=lambda model, row_id: Project(id=project_id, name="Mol") if model is Project else None)
|
||||
monkeypatch.setattr(
|
||||
MdkBathymetryProbeService,
|
||||
"probe",
|
||||
lambda: {
|
||||
"source_key": "mdk_bcp_bathymetry",
|
||||
"status": "tls_error",
|
||||
"acquisition_supported": False,
|
||||
},
|
||||
)
|
||||
app.dependency_overrides[get_db] = lambda: db
|
||||
try:
|
||||
response = TestClient(app).get(
|
||||
f"/api/v1/projects/{project_id}/datasets/bathymetry/sources/mdk_bcp_bathymetry/readiness"
|
||||
)
|
||||
finally:
|
||||
app.dependency_overrides.clear()
|
||||
|
||||
assert response.status_code == 200
|
||||
assert set(response.json()) == {"data"}
|
||||
assert response.json()["data"]["status"] == "tls_error"
|
||||
assert response.json()["data"]["acquisition_supported"] is False
|
||||
|
||||
|
||||
def test_partition_finalization_requires_complete_area_accounting_and_updates_versions() -> None:
|
||||
project_id = uuid4()
|
||||
area_ids = [uuid4(), uuid4()]
|
||||
dataset_id = uuid4()
|
||||
project = Project(id=project_id, name="Flanders")
|
||||
areas = [
|
||||
Area(id=area_ids[0], project_id=project_id, name="Gemeente Mol - officiële grens"),
|
||||
Area(id=area_ids[1], project_id=project_id, name="Gemeente Geel - officiële grens"),
|
||||
]
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
area_id=area_ids[0],
|
||||
name="vha.geojson",
|
||||
dataset_type="vector",
|
||||
source="VHA",
|
||||
source_name=BathymetryProfileAcquisitionService.PROVIDER,
|
||||
source_metadata={
|
||||
"profile_count": 3,
|
||||
"document_count": 2,
|
||||
"structured_depth_count": 1,
|
||||
"measurement_date_min": "1990-01-01",
|
||||
"measurement_date_max": "2020-01-01",
|
||||
},
|
||||
provenance_metadata={},
|
||||
status="ready",
|
||||
)
|
||||
version = DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1)
|
||||
rows = {
|
||||
(Project, project_id): project,
|
||||
(Area, area_ids[0]): areas[0],
|
||||
(Area, area_ids[1]): areas[1],
|
||||
(Dataset, dataset_id): dataset,
|
||||
}
|
||||
db = FinalizeSession(rows, [version])
|
||||
payload = BathymetryPartitionFinalizeRequest(
|
||||
partition_scope_key="flanders",
|
||||
expected_area_ids=area_ids,
|
||||
dataset_ids=[dataset_id],
|
||||
no_profile_area_ids=[area_ids[1]],
|
||||
manifest_sha256="a" * 64,
|
||||
observed_at=datetime(2026, 7, 17, tzinfo=UTC),
|
||||
)
|
||||
|
||||
result = BathymetryProfileAcquisitionService.finalize_partitions(db, project_id, payload)
|
||||
|
||||
assert result["regional_partitions_complete"] is True
|
||||
assert result["partition_count"] == 2
|
||||
assert result["data_partition_count"] == 1
|
||||
assert result["no_profile_partition_count"] == 1
|
||||
assert result["profile_count"] == 3
|
||||
assert dataset.source_metadata["coverage_scope"] == "flanders"
|
||||
assert dataset.source_metadata["municipality"] == "Mol"
|
||||
assert dataset.source_metadata["partitioned_source_audit"] is True
|
||||
assert version.source_metadata == dataset.source_metadata
|
||||
assert version.provenance_metadata == dataset.provenance_metadata
|
||||
assert db.commit_count == 1
|
||||
|
||||
incomplete = payload.model_copy(update={"no_profile_area_ids": []})
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
BathymetryProfileAcquisitionService.finalize_partitions(
|
||||
FinalizeSession(rows, [version]),
|
||||
project_id,
|
||||
incomplete,
|
||||
)
|
||||
assert exc_info.value.code == "BATHYMETRY_PARTITION_MANIFEST_INCOMPLETE"
|
||||
|
||||
|
||||
def test_flanders_scope_discovery_uses_complete_unique_vrbg_inventory() -> None:
|
||||
features = [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": f"Refgem.{index:05d}",
|
||||
"properties": {"NISCODE": f"{index:05d}", "NAAM": f"Gemeente {index:03d}"},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [[[4.0, 50.5], [4.1, 50.5], [4.1, 50.6], [4.0, 50.5]]],
|
||||
},
|
||||
}
|
||||
for index in range(1, 286)
|
||||
]
|
||||
scope, selected, source_url = flanders_scope.discover_flanders_scope(
|
||||
SourceSession({"features": list(reversed(features))}),
|
||||
timeout=30,
|
||||
min_municipalities=270,
|
||||
max_municipalities=300,
|
||||
)
|
||||
|
||||
assert scope.key == "flanders"
|
||||
assert scope.project_name == "Flanders Regional Workbench"
|
||||
assert len(scope.members) == 285
|
||||
assert len(set(scope.nis_codes)) == 285
|
||||
assert [item["properties"]["NISCODE"] for item in selected] == sorted(scope.nis_codes)
|
||||
assert source_url.startswith("https://geo.api.vlaanderen.be/")
|
||||
|
||||
|
||||
def test_expansion_scripts_are_packaged_and_readiness_checked() -> None:
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
for script in (
|
||||
"provision_flanders_geographic_scope.py",
|
||||
"provision_flanders_bathymetry_profiles.py",
|
||||
"probe_mdk_bathymetry.py",
|
||||
):
|
||||
assert f"COPY scripts/{script}" in dockerfile
|
||||
assert f"py_compile scripts/{script}" in readiness
|
||||
|
||||
sources = {
|
||||
item["key"]: item
|
||||
for item in BathymetryProfileAcquisitionService.list_sources()
|
||||
}
|
||||
assert sources["mdk_bcp_bathymetry"]["integration_status"] == "probe_only"
|
||||
assert sources["mdk_bcp_bathymetry"]["acquisition_supported"] is False
|
||||
assert "EL_wcs" in sources["mdk_bcp_bathymetry"]["service_url"]
|
||||
@@ -89,6 +89,9 @@ COPY scripts/provision_regional_historical_landuse.py /app/scripts/provision_reg
|
||||
COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_official_landuse_timeseries.py
|
||||
COPY scripts/provision_waterinfo_station_history.py /app/scripts/provision_waterinfo_station_history.py
|
||||
COPY scripts/provision_mol_bathymetry_profiles.py /app/scripts/provision_mol_bathymetry_profiles.py
|
||||
COPY scripts/provision_flanders_geographic_scope.py /app/scripts/provision_flanders_geographic_scope.py
|
||||
COPY scripts/provision_flanders_bathymetry_profiles.py /app/scripts/provision_flanders_bathymetry_profiles.py
|
||||
COPY scripts/probe_mdk_bathymetry.py /app/scripts/probe_mdk_bathymetry.py
|
||||
COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natura2000.py
|
||||
COPY scripts/provision_regional_bwk_natura2000.py /app/scripts/provision_regional_bwk_natura2000.py
|
||||
COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py
|
||||
|
||||
@@ -51,6 +51,11 @@
|
||||
<Config Name="VMM Flood Hazard WCS URL" Target="FLOOD_HAZARD_WCS_URL" Default="https://geoservice.waterinfo.be/OGRK/wcs" Mode="" Description="Official VMM OGRK WCS endpoint for governed flood-depth scenarios." Type="Variable" Display="advanced" Required="true" Mask="false">https://geoservice.waterinfo.be/OGRK/wcs</Config>
|
||||
<Config Name="Flood Hazard Analysis Resolution (m)" Target="FLOOD_HAZARD_RESOLUTION_M" Default="5.0" Mode="" Description="Stored analysis grid resolution; official source values are converted from centimetres to metres." Type="Variable" Display="advanced" Required="true" Mask="false">5.0</Config>
|
||||
<Config Name="Flood Hazard Maximum Cells" Target="FLOOD_HAZARD_MAX_PIXELS" Default="12000000" Mode="" Description="Maximum raster cells per flood-hazard acquisition or selection analysis." Type="Variable" Display="advanced" Required="true" Mask="false">12000000</Config>
|
||||
<Config Name="VHA Bathymetry Profiles" Target="BATHYMETRY_PROFILES_ENABLED" Default="true" Mode="" Description="Allow explicit bounded acquisition of official VHA cross-section profile points." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
|
||||
<Config Name="VHA Profile Maximum Features" Target="BATHYMETRY_PROFILES_MAX_FEATURES" Default="50000" Mode="" Description="Safety limit per municipality or bounded VHA profile request." Type="Variable" Display="advanced" Required="true" Mask="false">50000</Config>
|
||||
<Config Name="MDK Bathymetry Readiness Probe" Target="MDK_BATHYMETRY_PROBE_ENABLED" Default="true" Mode="" Description="Enable strict-TLS read-only GetCapabilities checks. This never downloads or activates a North Sea raster." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
|
||||
<Config Name="MDK Bathymetry WCS URL" Target="MDK_BATHYMETRY_WCS_URL" Default="https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs" Mode="" Description="Official metadata WCS endpoint. TLS verification is mandatory and cannot be bypassed." Type="Variable" Display="advanced" Required="true" Mask="false">https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs</Config>
|
||||
<Config Name="MDK Probe Timeout Seconds" Target="MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS" Default="20" Mode="" Description="Maximum wait for one read-only MDK GetCapabilities request." Type="Variable" Display="advanced" Required="true" Mask="false">20</Config>
|
||||
<Config Name="Official Thematic Raster Acquisition" Target="THEMATIC_RASTER_ENABLED" Default="true" Mode="" Description="Allow bounded official Departement Omgeving rasters for space, population, accessibility and services." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
|
||||
<Config Name="Thematic Raster WCS URL" Target="THEMATIC_RASTER_WCS_URL" Default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs" Mode="" Description="Official public MercatorNet WCS endpoint. Product identifiers remain server allowlisted." Type="Variable" Display="advanced" Required="true" Mask="false">https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs</Config>
|
||||
<Config Name="Thematic Raster Maximum Side (m)" Target="THEMATIC_RASTER_MAX_SIDE_M" Default="60000" Mode="" Description="Maximum side length for one allowlisted thematic raster scope; supports the complete Kempen work area while WCS transfers remain tiled." Type="Variable" Display="advanced" Required="true" Mask="false">60000</Config>
|
||||
|
||||
@@ -59,6 +59,17 @@ FLOOD_HAZARD_MAX_SIDE_M=20000
|
||||
FLOOD_HAZARD_MAX_PIXELS=12000000
|
||||
FLOOD_HAZARD_TIMEOUT_SECONDS=300
|
||||
FLOOD_HAZARD_MAX_RESPONSE_MB=160
|
||||
BATHYMETRY_PROFILES_ENABLED=true
|
||||
BATHYMETRY_PROFILES_LAYER_URL=https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0
|
||||
BATHYMETRY_WATERCOURSE_LAYER_URL=https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1
|
||||
BATHYMETRY_PROFILES_PAGE_SIZE=1000
|
||||
BATHYMETRY_PROFILES_MAX_FEATURES=50000
|
||||
BATHYMETRY_PROFILES_TIMEOUT_SECONDS=120
|
||||
BATHYMETRY_PROFILES_MAX_RESPONSE_MB=32
|
||||
MDK_BATHYMETRY_PROBE_ENABLED=true
|
||||
MDK_BATHYMETRY_WCS_URL=https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs
|
||||
MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS=20
|
||||
MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB=4
|
||||
|
||||
# Allowlisted Departement Omgeving policy rasters. Regional requests are
|
||||
# transferred as fixed 10 km WCS tiles before exact Area clipping.
|
||||
|
||||
@@ -51,6 +51,17 @@ FLOOD_HAZARD_MAX_SIDE_M="${FLOOD_HAZARD_MAX_SIDE_M:-20000}"
|
||||
FLOOD_HAZARD_MAX_PIXELS="${FLOOD_HAZARD_MAX_PIXELS:-12000000}"
|
||||
FLOOD_HAZARD_TIMEOUT_SECONDS="${FLOOD_HAZARD_TIMEOUT_SECONDS:-300}"
|
||||
FLOOD_HAZARD_MAX_RESPONSE_MB="${FLOOD_HAZARD_MAX_RESPONSE_MB:-160}"
|
||||
BATHYMETRY_PROFILES_ENABLED="${BATHYMETRY_PROFILES_ENABLED:-true}"
|
||||
BATHYMETRY_PROFILES_LAYER_URL="${BATHYMETRY_PROFILES_LAYER_URL:-https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0}"
|
||||
BATHYMETRY_WATERCOURSE_LAYER_URL="${BATHYMETRY_WATERCOURSE_LAYER_URL:-https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1}"
|
||||
BATHYMETRY_PROFILES_PAGE_SIZE="${BATHYMETRY_PROFILES_PAGE_SIZE:-1000}"
|
||||
BATHYMETRY_PROFILES_MAX_FEATURES="${BATHYMETRY_PROFILES_MAX_FEATURES:-50000}"
|
||||
BATHYMETRY_PROFILES_TIMEOUT_SECONDS="${BATHYMETRY_PROFILES_TIMEOUT_SECONDS:-120}"
|
||||
BATHYMETRY_PROFILES_MAX_RESPONSE_MB="${BATHYMETRY_PROFILES_MAX_RESPONSE_MB:-32}"
|
||||
MDK_BATHYMETRY_PROBE_ENABLED="${MDK_BATHYMETRY_PROBE_ENABLED:-true}"
|
||||
MDK_BATHYMETRY_WCS_URL="${MDK_BATHYMETRY_WCS_URL:-https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs}"
|
||||
MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS="${MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS:-20}"
|
||||
MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB="${MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB:-4}"
|
||||
THEMATIC_RASTER_ENABLED="${THEMATIC_RASTER_ENABLED:-true}"
|
||||
THEMATIC_RASTER_WCS_URL="${THEMATIC_RASTER_WCS_URL:-https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs}"
|
||||
THEMATIC_RASTER_MIN_SIDE_M="${THEMATIC_RASTER_MIN_SIDE_M:-100}"
|
||||
@@ -158,6 +169,17 @@ docker run -d \
|
||||
-e FLOOD_HAZARD_MAX_PIXELS="$FLOOD_HAZARD_MAX_PIXELS" \
|
||||
-e FLOOD_HAZARD_TIMEOUT_SECONDS="$FLOOD_HAZARD_TIMEOUT_SECONDS" \
|
||||
-e FLOOD_HAZARD_MAX_RESPONSE_MB="$FLOOD_HAZARD_MAX_RESPONSE_MB" \
|
||||
-e BATHYMETRY_PROFILES_ENABLED="$BATHYMETRY_PROFILES_ENABLED" \
|
||||
-e BATHYMETRY_PROFILES_LAYER_URL="$BATHYMETRY_PROFILES_LAYER_URL" \
|
||||
-e BATHYMETRY_WATERCOURSE_LAYER_URL="$BATHYMETRY_WATERCOURSE_LAYER_URL" \
|
||||
-e BATHYMETRY_PROFILES_PAGE_SIZE="$BATHYMETRY_PROFILES_PAGE_SIZE" \
|
||||
-e BATHYMETRY_PROFILES_MAX_FEATURES="$BATHYMETRY_PROFILES_MAX_FEATURES" \
|
||||
-e BATHYMETRY_PROFILES_TIMEOUT_SECONDS="$BATHYMETRY_PROFILES_TIMEOUT_SECONDS" \
|
||||
-e BATHYMETRY_PROFILES_MAX_RESPONSE_MB="$BATHYMETRY_PROFILES_MAX_RESPONSE_MB" \
|
||||
-e MDK_BATHYMETRY_PROBE_ENABLED="$MDK_BATHYMETRY_PROBE_ENABLED" \
|
||||
-e MDK_BATHYMETRY_WCS_URL="$MDK_BATHYMETRY_WCS_URL" \
|
||||
-e MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS="$MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS" \
|
||||
-e MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB="$MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB" \
|
||||
-e THEMATIC_RASTER_ENABLED="$THEMATIC_RASTER_ENABLED" \
|
||||
-e THEMATIC_RASTER_WCS_URL="$THEMATIC_RASTER_WCS_URL" \
|
||||
-e THEMATIC_RASTER_MIN_SIDE_M="$THEMATIC_RASTER_MIN_SIDE_M" \
|
||||
|
||||
@@ -61,6 +61,10 @@ services:
|
||||
BATHYMETRY_PROFILES_MAX_FEATURES: ${BATHYMETRY_PROFILES_MAX_FEATURES:-50000}
|
||||
BATHYMETRY_PROFILES_TIMEOUT_SECONDS: ${BATHYMETRY_PROFILES_TIMEOUT_SECONDS:-120}
|
||||
BATHYMETRY_PROFILES_MAX_RESPONSE_MB: ${BATHYMETRY_PROFILES_MAX_RESPONSE_MB:-32}
|
||||
MDK_BATHYMETRY_PROBE_ENABLED: ${MDK_BATHYMETRY_PROBE_ENABLED:-true}
|
||||
MDK_BATHYMETRY_WCS_URL: ${MDK_BATHYMETRY_WCS_URL:-https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs}
|
||||
MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS: ${MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS:-20}
|
||||
MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB: ${MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB:-4}
|
||||
THEMATIC_RASTER_ENABLED: ${THEMATIC_RASTER_ENABLED:-true}
|
||||
THEMATIC_RASTER_WCS_URL: ${THEMATIC_RASTER_WCS_URL:-https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs}
|
||||
THEMATIC_RASTER_MIN_SIDE_M: ${THEMATIC_RASTER_MIN_SIDE_M:-100}
|
||||
|
||||
+29
-4
@@ -2061,10 +2061,20 @@ sets an explicit Ollama context window and returns
|
||||
### GET `/api/v1/projects/{project_id}/datasets/bathymetry/sources`
|
||||
|
||||
Returns the governed bathymetry source registry in the canonical envelope.
|
||||
VHA inland profiles are `operational`. MDK Belgian Continental Shelf and SPW
|
||||
Walloon bathymetry remain `available_not_integrated`; they cannot be acquired
|
||||
through this contract until their raster/download and vertical-datum flows
|
||||
pass live validation.
|
||||
VHA inland profiles are `operational`. MDK Belgian Continental Shelf is
|
||||
`probe_only`; SPW Walloon bathymetry remains `available_not_integrated`.
|
||||
Neither source can be acquired until its raster/download and vertical-datum
|
||||
flow passes live validation.
|
||||
|
||||
### GET `/api/v1/projects/{project_id}/datasets/bathymetry/sources/mdk_bcp_bathymetry/readiness`
|
||||
|
||||
Runs one bounded, read-only WCS 1.0.0 `GetCapabilities` request with mandatory
|
||||
system TLS verification and a configured response-size limit. Status is one
|
||||
of `disabled`, `invalid_configuration`, `tls_error`,
|
||||
`endpoint_unavailable`, `invalid_capabilities` or `reachable`. A reachable
|
||||
response lists coverage identifiers, advertised formats and CRS values, but
|
||||
always returns `acquisition_supported=false`. There is no insecure TLS
|
||||
fallback and no `GetCoverage` request.
|
||||
|
||||
### POST `/api/v1/projects/{project_id}/datasets/bathymetry/profiles/acquire`
|
||||
|
||||
@@ -2089,3 +2099,18 @@ The acquired Dataset uses
|
||||
`reference_layer_name=bathymetry_profiles`. Existing vector content and
|
||||
selection endpoints provide GeoJSON and metrics. The contract does not expose
|
||||
a continuous bathymetric surface, current water depth or water volume.
|
||||
|
||||
### POST `/api/v1/projects/{project_id}/datasets/bathymetry/profiles/partitions/finalize`
|
||||
|
||||
Finalizes a complete municipality-partition manifest. The request supplies a
|
||||
scope key, every expected municipality Area id, one ready VHA Dataset id for
|
||||
each non-empty partition, explicit Area ids with zero source profiles, a
|
||||
SHA-256 manifest identity and observation time. The backend verifies project,
|
||||
Area, provider, Dataset and one-Dataset-per-Area consistency. Only a complete
|
||||
accounting sets `partitioned_source_audit=true` and
|
||||
`regional_partitions_complete=true`; partial operator runs remain hidden at
|
||||
regional scope.
|
||||
|
||||
Future provider output continues to use DatasetService and, for vectors,
|
||||
VectorFeatureService. Arbitrary service URLs, browser-side fetches, insecure
|
||||
TLS bypasses and startup downloads remain forbidden.
|
||||
|
||||
@@ -120,12 +120,21 @@ bed evolution.
|
||||
|
||||
## Implementation order
|
||||
|
||||
1. Operate and validate the Mol VHA profile Dataset and map flow.
|
||||
2. Add VHA municipal partition orchestration for Flanders.
|
||||
1. Operate and validate the Mol VHA profile Dataset and map flow. **Done.**
|
||||
2. Add VHA municipal partition orchestration for Flanders. **Implemented;
|
||||
complete live provisioning remains an explicit operator run.**
|
||||
3. Implement a bounded MDK WCS probe, then acquisition behind live evidence.
|
||||
**Probe implemented. Acquisition blocked because the live endpoint fails
|
||||
strict hostname validation and does not expose usable capabilities.**
|
||||
4. Implement SPW download staging and vertical-datum metadata validation.
|
||||
5. Add maritime boundaries as separate authoritative scope layers.
|
||||
6. Add cross-source vertical-datum transformation only with authoritative
|
||||
grids/parameters and uncertainty tests.
|
||||
7. Add volume only after a compatible measured or modeled water-surface source
|
||||
is part of the same analysis contract.
|
||||
|
||||
The Flanders scope is land-only and is dynamically derived from the complete
|
||||
current VRBG municipality collection (285 members at the 2026-07-17
|
||||
validation). Belgium is not represented by expanding that polygon. Wallonia,
|
||||
Brussels, the territorial sea and the Belgian EEZ/continental shelf require
|
||||
their own authoritative adapters and legal scope labels.
|
||||
|
||||
@@ -10054,3 +10054,41 @@ Validation:
|
||||
historical survey range `1877-2020` and loaded all 828 persisted objects.
|
||||
The browser console contained no warnings or errors and the 1265x720
|
||||
viewport had no horizontal document overflow.
|
||||
|
||||
## Sprint 236 - Flemish bathymetry partitions and safe North Sea probe (2026-07-17)
|
||||
|
||||
Implemented:
|
||||
- Added a dynamic Flanders scope operator that discovers the complete current
|
||||
official VRBG RefGem municipality inventory instead of freezing a fragile
|
||||
hand-maintained list. It validates unique NIS codes/names, polygonal source
|
||||
features and a 270..300 safety range before creating artifacts or API state.
|
||||
- Added an atomic, resumable VHA municipality coordinator. Every partition is
|
||||
exact-Area clipped and persisted through the existing acquisition endpoint;
|
||||
failures, explicit zero-profile results and ready Dataset ids remain in the
|
||||
checksum-bound coverage manifest.
|
||||
- Added server-side partition finalization. Regional activation requires exact
|
||||
accounting for every expected municipality, one ready VHA Dataset per
|
||||
non-empty Area and explicit no-profile Area ids. Dataset and DatasetVersion
|
||||
metadata retain the manifest identity and completeness flags.
|
||||
- Added a strict-TLS, response-bounded MDK WCS GetCapabilities probe and a
|
||||
concise Sources-workspace control. It never performs GetCoverage, never
|
||||
disables certificate verification and always leaves acquisition disabled.
|
||||
- Packaged the Flanders scope, VHA coordinator and MDK probe operators in the
|
||||
all-in-one image and exposed their bounded runtime settings in Compose and
|
||||
the Unraid deployment configuration.
|
||||
|
||||
Validation:
|
||||
- Official VRBG fetch-only validation returned 285 unique municipalities, an
|
||||
exact union of 13,625.73 km2 and WGS84 bounds
|
||||
`[2.54132923, 50.68749237, 5.9111094, 51.50511313]`.
|
||||
- The live MDK metadata endpoint currently fails strict hostname validation:
|
||||
`bathy.agentschapmdk.be` presents a certificate for `*.l27powered.eu`.
|
||||
Diagnostic requests to both the previously configured and current metadata
|
||||
paths returned HTTP 404 after an explicitly external, non-application
|
||||
insecure inspection. GeoIntel itself reports `tls_error` and provides no
|
||||
bypass.
|
||||
- The complete readiness gate passed 920 backend tests, backend compilation,
|
||||
documentation/contract audits, 115 documented API routes, Alembic head
|
||||
`202607160001`, frontend TypeScript typecheck and the production Vite build.
|
||||
- Live Docker deployment and controlled Flanders provisioning are recorded
|
||||
after the repository commit below.
|
||||
|
||||
@@ -2,6 +2,14 @@
|
||||
|
||||
Database: PostgreSQL + PostGIS.
|
||||
|
||||
Bathymetry regional completeness requires no new table. Every VHA municipality
|
||||
partition remains an ordinary `datasets` row plus `vector_features`. After a
|
||||
complete server-validated manifest, Dataset and DatasetVersion metadata retain
|
||||
`partition_scope_key`, partition counts, the manifest SHA-256,
|
||||
`partitioned_source_audit=true` and
|
||||
`regional_partitions_complete=true`. Empty source partitions are recorded in
|
||||
the manifest provenance and never represented by fabricated features.
|
||||
|
||||
## Rules
|
||||
|
||||
- Store geometries in PostGIS with explicit SRID.
|
||||
|
||||
+15
-1
@@ -722,7 +722,11 @@ are not filled by fabricated OCR output.
|
||||
The following sources are audited but not yet operational:
|
||||
|
||||
- MDK Dieptemodel Belgisch Continentaal Plat/Noordzee: 20 x 20 m continuous
|
||||
raster in LAT, exposed through WCS/WMTS.
|
||||
raster in LAT, exposed through WCS/WMTS. GeoIntel now has a strict-TLS,
|
||||
read-only GetCapabilities probe. On 2026-07-17 the official metadata
|
||||
endpoint presented a certificate for another hostname and returned no
|
||||
usable capabilities path; status therefore remains `tls_error` and raster
|
||||
acquisition is disabled.
|
||||
- SPW bathymetry of navigable waterways and reservoir lakes: 0.5 m bed
|
||||
elevation and XYZ data in mDNG.
|
||||
- Port of Antwerp-Bruges periodic soundings: catalog candidate pending a
|
||||
@@ -732,3 +736,13 @@ See `docs/BATHYMETRY_EXPANSION_ROADMAP.md`. TAW, LAT and mDNG remain separate
|
||||
until an authoritative vertical transformation is implemented and tested.
|
||||
The territorial sea, EEZ and continental shelf are separate scope layers and
|
||||
must be labelled according to their legal meaning.
|
||||
|
||||
The official VRBG RefGem collection currently exposes 285 Flemish
|
||||
municipalities. `scripts/provision_flanders_geographic_scope.py` validates the
|
||||
complete inventory, produces checksum-bound land-boundary artifacts and
|
||||
creates one persisted municipality Area per source feature. The separate
|
||||
`scripts/provision_flanders_bathymetry_profiles.py` coordinator acquires VHA
|
||||
profiles per Area, writes an atomic resumable manifest and requests regional
|
||||
activation only after all 285 partitions are accounted for. A municipality
|
||||
with zero source points is retained as an explicit no-profile partition, not
|
||||
silently omitted.
|
||||
|
||||
@@ -222,7 +222,30 @@ Raster / gemodelleerd overstromingsgevaar.
|
||||
|
||||
### Prioriteit
|
||||
|
||||
P5 overstromingsscenario's operationeel voor Mol; bathymetrie blijft open.
|
||||
P5 overstromingsscenario's operationeel. VHA-dwarsprofielen zijn als
|
||||
historische puntmetingen operationeel voor Mol en partitioneerbaar voor alle
|
||||
officiële Vlaamse gemeente-Areas. Ze blijven strikt gescheiden van dit
|
||||
scenario-raster en leveren geen gebiedsdekkende bathymetrie of volume.
|
||||
|
||||
## Bathymetry partition metadata
|
||||
|
||||
Een VHA-profiel-Dataset behoudt `partition_area_id`,
|
||||
`partition_area_name`, `municipality`, meetdatumbereik en exacte aantallen.
|
||||
Regionale Vlaamse dekking is alleen geldig wanneer de server een volledig
|
||||
manifest voor de actuele officiële gemeente-inventaris heeft gevalideerd.
|
||||
Daarna bevatten Dataset en DatasetVersion:
|
||||
|
||||
- `partition_scope_key`
|
||||
- `partition_count`, `data_partition_count` en `no_profile_partition_count`
|
||||
- `partition_manifest_sha256`
|
||||
- `partitioned_source_audit=true`
|
||||
- `regional_partitions_complete=true`
|
||||
|
||||
Een no-profile partition is bronbewijs dat de begrensde VHA-query nul punten
|
||||
opleverde. Er worden nooit lege of kunstmatige profielobjecten aangemaakt.
|
||||
|
||||
MDK Noordzee blijft `probe_only`: WCS-capabilities kunnen veilig worden
|
||||
gecontroleerd, maar er bestaat nog geen toegestane rasteracquisitie.
|
||||
|
||||
## LAS/LAZ LiDAR
|
||||
|
||||
|
||||
+4
-3
@@ -751,11 +751,12 @@ This file now starts with the current implementation status. Older preparation/b
|
||||
creates a new immutable Dataset with official `YYYY.NN` source version.
|
||||
# Bathymetry follow-up
|
||||
|
||||
- [ ] Add governed municipality partition orchestration for all of Flanders
|
||||
- [x] Add governed municipality partition orchestration for all of Flanders
|
||||
after the Mol VHA operator passes live acceptance.
|
||||
- [ ] Add read-only MDK WCS capability/TLS probe and maritime scope metadata.
|
||||
- [x] Add read-only MDK WCS capability/TLS probe and maritime scope metadata.
|
||||
- [ ] Add bounded MDK GeoTIFF acquisition only after live CRS, LAT, nodata and
|
||||
response-limit validation.
|
||||
response-limit validation. Current official endpoint is blocked by TLS
|
||||
hostname mismatch and unavailable capabilities; never bypass verification.
|
||||
- [ ] Add SPW staged-download adapter with mDNG metadata and survey-epoch
|
||||
coverage validation.
|
||||
- [ ] Add authoritative territorial-sea, EEZ and continental-shelf boundary
|
||||
|
||||
+4
-2
@@ -646,5 +646,7 @@ profile-document link.
|
||||
|
||||
The UI always labels the points as historical cross-sections. It does not
|
||||
present them as a continuous bed raster and does not calculate water volume.
|
||||
The Sources workspace lists the future MDK North Sea and SPW Walloon
|
||||
bathymetry integrations as planned until backend acquisition is operational.
|
||||
Municipality partitions remain hidden for a regional Area until their backend
|
||||
manifest reports complete coverage. The Sources workspace lists MDK North Sea
|
||||
as read-only probe-only and SPW Walloon bathymetry as planned until bounded
|
||||
raster/download acquisition is operational.
|
||||
|
||||
@@ -806,7 +806,7 @@ function App(): JSX.Element {
|
||||
|
||||
{activeWorkspace === 'data' ? (
|
||||
<div className="workspace-grid workspace-grid-data">
|
||||
<SourceCatalogPanel datasets={datasets} />
|
||||
<SourceCatalogPanel datasets={datasets} projectId={selectedProjectId ?? undefined} />
|
||||
<ProjectPanel
|
||||
projects={projects}
|
||||
selectedProjectId={selectedProjectId}
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
import type { DatasetCreateResponse } from '../../types'
|
||||
import { useState } from 'react'
|
||||
import { datasetsApi } from '../../services/api'
|
||||
import type { BathymetrySourceProbeRead, DatasetCreateResponse } from '../../types'
|
||||
import { getDatasetSourceDisplayName } from '../../lib/datasetDisplay'
|
||||
import {
|
||||
OFFICIAL_SOURCE_PORTFOLIO,
|
||||
@@ -10,6 +12,7 @@ import {
|
||||
|
||||
interface SourceCatalogPanelProps {
|
||||
datasets: DatasetCreateResponse[]
|
||||
projectId?: string
|
||||
}
|
||||
|
||||
const THEME_LABELS: Record<string, string> = {
|
||||
@@ -63,7 +66,19 @@ function timelineSummary(
|
||||
].filter(Boolean).join(' · ')
|
||||
}
|
||||
|
||||
export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.Element {
|
||||
function mdkStatusLabel(status: BathymetrySourceProbeRead['status']): string {
|
||||
if (status === 'reachable') return 'Service bereikbaar'
|
||||
if (status === 'tls_error') return 'Geblokkeerd: certificaat'
|
||||
if (status === 'endpoint_unavailable') return 'Service niet bereikbaar'
|
||||
if (status === 'invalid_capabilities') return 'Ongeldige service-informatie'
|
||||
if (status === 'disabled') return 'Controle uitgeschakeld'
|
||||
return 'Configuratie controleren'
|
||||
}
|
||||
|
||||
export function SourceCatalogPanel({ datasets, projectId }: SourceCatalogPanelProps): JSX.Element {
|
||||
const [mdkProbe, setMdkProbe] = useState<BathymetrySourceProbeRead | null>(null)
|
||||
const [mdkProbeLoading, setMdkProbeLoading] = useState(false)
|
||||
const [mdkProbeError, setMdkProbeError] = useState<string | null>(null)
|
||||
const ready = datasets.filter((dataset) => dataset.status === 'ready')
|
||||
const waterinfoDatasets = ready.filter((dataset) => dataset.source_name === 'waterinfo')
|
||||
const historicalOrthophotos = ready.filter(
|
||||
@@ -135,6 +150,19 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E
|
||||
}
|
||||
})
|
||||
|
||||
const checkMdkReadiness = async (): Promise<void> => {
|
||||
if (!projectId || mdkProbeLoading) return
|
||||
setMdkProbeLoading(true)
|
||||
setMdkProbeError(null)
|
||||
try {
|
||||
setMdkProbe(await datasetsApi.probeMdkBathymetry(projectId))
|
||||
} catch (error) {
|
||||
setMdkProbeError(error instanceof Error ? error.message : 'De Noordzee-bron kon niet worden gecontroleerd.')
|
||||
} finally {
|
||||
setMdkProbeLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<section className="workspace-panel source-catalog-panel" aria-label="Beschikbare databronnen">
|
||||
<div className="panel-title-row">
|
||||
@@ -264,6 +292,29 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E
|
||||
</details>
|
||||
) : null}
|
||||
|
||||
<details className="source-loaded-disclosure">
|
||||
<summary>Maritieme bronstatus</summary>
|
||||
<div className="source-catalog-loaded">
|
||||
<article>
|
||||
<strong>MDK-dieptemodel Belgische Noordzee</strong>
|
||||
<span>{mdkProbe ? mdkStatusLabel(mdkProbe.status) : 'Nog niet live gecontroleerd'}</span>
|
||||
<p>
|
||||
{mdkProbe?.message ??
|
||||
'Controleert uitsluitend de officiële service-informatie met geldige TLS. Er wordt geen raster gedownload.'}
|
||||
</p>
|
||||
{mdkProbeError ? <small className="error-text">{mdkProbeError}</small> : null}
|
||||
<button
|
||||
className="secondary-button"
|
||||
type="button"
|
||||
disabled={!projectId || mdkProbeLoading}
|
||||
onClick={() => void checkMdkReadiness()}
|
||||
>
|
||||
{mdkProbeLoading ? 'Controleren...' : 'Controleer Noordzee-bron'}
|
||||
</button>
|
||||
</article>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<details className="source-opportunity-list">
|
||||
<summary>Officiële bronnen die hierna kunnen worden ingeladen</summary>
|
||||
<div className="source-opportunity-domains">
|
||||
|
||||
@@ -23,6 +23,7 @@ import type {
|
||||
FloodHazardProductRead,
|
||||
FloodHazardSelectionResponse,
|
||||
BathymetryProfileAcquireRequest,
|
||||
BathymetrySourceProbeRead,
|
||||
BathymetrySourceRead,
|
||||
DhmvProductRead,
|
||||
TerrainSelectionResponse,
|
||||
@@ -173,6 +174,10 @@ export const datasetsApi = {
|
||||
apiGet<{ items: BathymetrySourceRead[]; total: number }>(
|
||||
`/api/v1/projects/${projectId}/datasets/bathymetry/sources`,
|
||||
),
|
||||
probeMdkBathymetry: (projectId: string): Promise<BathymetrySourceProbeRead> =>
|
||||
apiGet<BathymetrySourceProbeRead>(
|
||||
`/api/v1/projects/${projectId}/datasets/bathymetry/sources/mdk_bcp_bathymetry/readiness`,
|
||||
),
|
||||
acquireBathymetryProfiles: (projectId: string, payload: BathymetryProfileAcquireRequest): Promise<JobRead> =>
|
||||
apiPost<JobRead>(`/api/v1/projects/${projectId}/datasets/bathymetry/profiles/acquire`, payload),
|
||||
acquireThematicRaster: (projectId: string, payload: ThematicRasterAcquireRequest): Promise<JobRead> =>
|
||||
|
||||
+25
-1
@@ -459,7 +459,7 @@ export interface BathymetrySourceRead {
|
||||
vertical_reference: string
|
||||
horizontal_crs: string
|
||||
native_resolution?: string | null
|
||||
integration_status: 'operational' | 'available_not_integrated' | 'catalog_only'
|
||||
integration_status: 'operational' | 'probe_only' | 'available_not_integrated' | 'catalog_only'
|
||||
acquisition_supported: boolean
|
||||
configured: boolean
|
||||
service_url?: string | null
|
||||
@@ -469,6 +469,30 @@ export interface BathymetrySourceRead {
|
||||
limitation_message: string
|
||||
}
|
||||
|
||||
export interface BathymetrySourceProbeRead {
|
||||
source_key: 'mdk_bcp_bathymetry'
|
||||
status:
|
||||
| 'disabled'
|
||||
| 'invalid_configuration'
|
||||
| 'tls_error'
|
||||
| 'endpoint_unavailable'
|
||||
| 'invalid_capabilities'
|
||||
| 'reachable'
|
||||
configured_url: string
|
||||
capabilities_url?: string | null
|
||||
tls_verified: boolean
|
||||
capabilities_reachable: boolean
|
||||
acquisition_supported: false
|
||||
wcs_version?: string | null
|
||||
coverage_identifiers: string[]
|
||||
advertised_formats: string[]
|
||||
advertised_crs: string[]
|
||||
response_sha256?: string | null
|
||||
checked_at: string
|
||||
message: string
|
||||
limitation_message: string
|
||||
}
|
||||
|
||||
export interface ThematicRasterAcquireRequest {
|
||||
bbox: VectorSelectionBBox
|
||||
area_id?: string | null
|
||||
|
||||
@@ -1909,3 +1909,32 @@ python scripts/provision_mol_bathymetry_profiles.py \
|
||||
|
||||
The output is a point dataset with historical evidence. It is not a continuous
|
||||
water-bottom raster and cannot calculate current water volume.
|
||||
|
||||
## Flanders VHA bathymetry partitions
|
||||
|
||||
Create the complete official Flemish land scope from the current VRBG
|
||||
municipality collection:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_flanders_geographic_scope.py
|
||||
```
|
||||
|
||||
Acquire VHA profile partitions with atomic resume evidence:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_flanders_bathymetry_profiles.py
|
||||
```
|
||||
|
||||
Use `--members Mol Geel` or `--max-partitions 5` only for a partial operational
|
||||
check. Partial runs do not activate regional coverage. The complete run
|
||||
finalizes only when every official municipality has either one ready Dataset
|
||||
or an explicit zero-profile source result.
|
||||
|
||||
Probe the official-metadata MDK WCS safely:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/probe_mdk_bathymetry.py
|
||||
```
|
||||
|
||||
This performs only `GetCapabilities`, keeps strict TLS verification enabled
|
||||
and returns exit code `2` for an honest non-ready source.
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Run the safe MDK WCS readiness probe through the canonical GeoIntel API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||
DEFAULT_PROJECT_NAME = "Kempen Regional Workbench"
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Probe MDK bathymetry WCS readiness without downloading coverage.")
|
||||
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
|
||||
parser.add_argument("--project-name", default=DEFAULT_PROJECT_NAME)
|
||||
parser.add_argument("--timeout", type=int, default=60)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def unwrap(response: requests.Response):
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
if not isinstance(payload, dict) or "data" not in payload:
|
||||
raise RuntimeError(f"Non-canonical API response from {response.url}")
|
||||
return payload["data"]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
base_url = args.base_url.rstrip("/")
|
||||
session = requests.Session()
|
||||
projects = unwrap(
|
||||
session.get(
|
||||
f"{base_url}/api/v1/projects",
|
||||
params={"name": args.project_name, "limit": 1},
|
||||
timeout=args.timeout,
|
||||
)
|
||||
)["items"]
|
||||
if not projects:
|
||||
raise RuntimeError(f"Project {args.project_name!r} was not found")
|
||||
result = unwrap(
|
||||
session.get(
|
||||
f"{base_url}/api/v1/projects/{projects[0]['id']}/datasets/"
|
||||
"bathymetry/sources/mdk_bcp_bathymetry/readiness",
|
||||
timeout=args.timeout,
|
||||
)
|
||||
)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0 if result["status"] == "reachable" else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,400 @@
|
||||
"""Provision VHA cross-section profiles for every persisted Flemish municipality.
|
||||
|
||||
The operator is resumable and calls only canonical GeoIntel API endpoints.
|
||||
Regional completeness is finalized server-side only when every municipality
|
||||
has either one ready profile Dataset or an explicit no-profile source result.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||
DEFAULT_PROJECT_NAME = "Flanders Regional Workbench"
|
||||
DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/bathymetry/vha-flanders/manifest.json")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Provision partitioned VHA profiles for Flanders.")
|
||||
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
|
||||
parser.add_argument("--project-name", default=DEFAULT_PROJECT_NAME)
|
||||
parser.add_argument(
|
||||
"--manifest-path",
|
||||
type=Path,
|
||||
default=Path(os.environ.get("GEOINTEL_VHA_FLANDERS_MANIFEST", DEFAULT_MANIFEST_PATH)),
|
||||
)
|
||||
parser.add_argument("--timeout", type=int, default=900)
|
||||
parser.add_argument("--force", action="store_true")
|
||||
parser.add_argument("--continue-on-error", action="store_true")
|
||||
parser.add_argument("--members", nargs="*", default=[])
|
||||
parser.add_argument("--max-partitions", type=int, default=0)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def unwrap(response: requests.Response) -> Any:
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError as exc:
|
||||
raise RuntimeError(f"GeoIntel returned non-JSON ({response.status_code}): {response.text[:300]}") from exc
|
||||
if not response.ok:
|
||||
error_code = payload.get("error") if isinstance(payload, dict) else None
|
||||
message = payload.get("message") if isinstance(payload, dict) else None
|
||||
raise RuntimeError(f"{error_code or response.status_code}: {message or response.text[:300]}")
|
||||
if not isinstance(payload, dict) or "data" not in payload:
|
||||
raise RuntimeError(f"Non-canonical API response from {response.url}")
|
||||
return payload["data"]
|
||||
|
||||
|
||||
def list_paginated(session: requests.Session, url: str, *, timeout: int) -> list[dict[str, Any]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
offset = 0
|
||||
total: int | None = None
|
||||
while True:
|
||||
page = unwrap(session.get(url, params={"limit": 200, "offset": offset}, timeout=timeout))
|
||||
page_items = list(page.get("items") or [])
|
||||
items.extend(page_items)
|
||||
total = int(page.get("total") or 0) if total is None else total
|
||||
if not page_items or len(items) >= total:
|
||||
break
|
||||
offset += len(page_items)
|
||||
if total is not None and len(items) != total:
|
||||
raise RuntimeError(f"Paginated API returned {len(items)} of {total} records for {url}")
|
||||
return items
|
||||
|
||||
|
||||
def coordinates(geometry: dict[str, Any]) -> Iterable[tuple[float, float]]:
|
||||
def walk(value: Any):
|
||||
if isinstance(value, list) and len(value) >= 2 and all(isinstance(item, (int, float)) for item in value[:2]):
|
||||
yield float(value[0]), float(value[1])
|
||||
return
|
||||
if isinstance(value, list):
|
||||
for child in value:
|
||||
yield from walk(child)
|
||||
|
||||
yield from walk(geometry.get("coordinates", []))
|
||||
|
||||
|
||||
def geometry_bbox(geometry: dict[str, Any]) -> dict[str, float | str]:
|
||||
points = list(coordinates(geometry))
|
||||
if not points:
|
||||
raise RuntimeError("Persisted municipality Area contains no coordinates")
|
||||
xs = [point[0] for point in points]
|
||||
ys = [point[1] for point in points]
|
||||
return {
|
||||
"min_x": min(xs),
|
||||
"min_y": min(ys),
|
||||
"max_x": max(xs),
|
||||
"max_y": max(ys),
|
||||
"crs": "EPSG:4326",
|
||||
}
|
||||
|
||||
|
||||
def area_identity(area: dict[str, Any]) -> str:
|
||||
canonical = {
|
||||
"id": area.get("id"),
|
||||
"name": area.get("name"),
|
||||
"geometry": area.get("geometry"),
|
||||
}
|
||||
return hashlib.sha256(
|
||||
json.dumps(canonical, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def write_manifest(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_suffix(path.suffix + ".partial")
|
||||
temporary.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def load_manifest(path: Path) -> dict[str, Any]:
|
||||
if not path.is_file():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def source_error(response: requests.Response) -> tuple[str | None, str]:
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError:
|
||||
return None, response.text[:300]
|
||||
if not isinstance(payload, dict):
|
||||
return None, response.text[:300]
|
||||
return (
|
||||
str(payload.get("error")) if payload.get("error") else None,
|
||||
str(payload.get("message") or response.text[:300]),
|
||||
)
|
||||
|
||||
|
||||
def acquire_partition(
|
||||
session: requests.Session,
|
||||
*,
|
||||
base_url: str,
|
||||
project_id: str,
|
||||
area: dict[str, Any],
|
||||
timeout: int,
|
||||
force: bool,
|
||||
) -> dict[str, Any]:
|
||||
bbox = geometry_bbox(area["geometry"])
|
||||
response = session.post(
|
||||
f"{base_url}/api/v1/projects/{project_id}/datasets/bathymetry/profiles/acquire",
|
||||
json={"bbox": bbox, "area_id": area["id"], "force_refresh": force},
|
||||
timeout=timeout,
|
||||
)
|
||||
if response.status_code == 404:
|
||||
code, message = source_error(response)
|
||||
if code == "BATHYMETRY_NO_PROFILES":
|
||||
return {
|
||||
"status": "no_profiles",
|
||||
"area_id": area["id"],
|
||||
"area_name": area["name"],
|
||||
"area_identity_sha256": area_identity(area),
|
||||
"bbox": bbox,
|
||||
"message": message,
|
||||
"completed_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
data = unwrap(response)
|
||||
if data.get("status") != "success" or not data.get("output_dataset_id"):
|
||||
raise RuntimeError(f"VHA acquisition failed for {area['name']}: {data.get('error_message') or data}")
|
||||
result = data.get("result_json") or {}
|
||||
return {
|
||||
"status": "complete",
|
||||
"area_id": area["id"],
|
||||
"area_name": area["name"],
|
||||
"area_identity_sha256": area_identity(area),
|
||||
"bbox": bbox,
|
||||
"dataset_id": data["output_dataset_id"],
|
||||
"reused": bool(result.get("reused")),
|
||||
"profile_count": int(result.get("profile_count") or 0),
|
||||
"document_count": int(result.get("document_count") or 0),
|
||||
"structured_depth_count": int(result.get("structured_depth_count") or 0),
|
||||
"measurement_date_min": result.get("measurement_date_min"),
|
||||
"measurement_date_max": result.get("measurement_date_max"),
|
||||
"completed_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def manifest_identity(
|
||||
*,
|
||||
project_id: str,
|
||||
partitions: list[dict[str, Any]],
|
||||
) -> tuple[dict[str, Any], str]:
|
||||
identity = {
|
||||
"schema_version": 1,
|
||||
"source": "vmm_vha_bathymetry_profiles",
|
||||
"partition_scope_key": "flanders",
|
||||
"project_id": project_id,
|
||||
"partitions": [
|
||||
{
|
||||
key: partition.get(key)
|
||||
for key in (
|
||||
"area_id",
|
||||
"area_name",
|
||||
"area_identity_sha256",
|
||||
"status",
|
||||
"dataset_id",
|
||||
"profile_count",
|
||||
"document_count",
|
||||
"structured_depth_count",
|
||||
"measurement_date_min",
|
||||
"measurement_date_max",
|
||||
)
|
||||
}
|
||||
for partition in sorted(partitions, key=lambda item: str(item["area_id"]))
|
||||
],
|
||||
}
|
||||
digest = hashlib.sha256(
|
||||
json.dumps(identity, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||
).hexdigest()
|
||||
return identity, digest
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
if args.max_partitions < 0:
|
||||
print(json.dumps({"status": "error", "message": "--max-partitions cannot be negative"}), file=sys.stderr)
|
||||
return 2
|
||||
base_url = args.base_url.rstrip("/")
|
||||
session = requests.Session()
|
||||
session.headers.update({"User-Agent": "GeoIntel-VHA-Flanders-Operator/1.0"})
|
||||
manifest = load_manifest(args.manifest_path)
|
||||
|
||||
try:
|
||||
projects = list_paginated(session, f"{base_url}/api/v1/projects", timeout=60)
|
||||
project = next((item for item in projects if item.get("name") == args.project_name), None)
|
||||
if project is None:
|
||||
raise RuntimeError(
|
||||
f"Project {args.project_name!r} was not found; run provision_flanders_geographic_scope.py first"
|
||||
)
|
||||
all_areas = list_paginated(
|
||||
session,
|
||||
f"{base_url}/api/v1/projects/{project['id']}/areas",
|
||||
timeout=120,
|
||||
)
|
||||
municipalities = sorted(
|
||||
(area for area in all_areas if str(area.get("name") or "").casefold().startswith("gemeente ")),
|
||||
key=lambda item: str(item["name"]).casefold(),
|
||||
)
|
||||
if not 270 <= len(municipalities) <= 300:
|
||||
raise RuntimeError(
|
||||
f"Flanders workspace exposes {len(municipalities)} municipality Areas; expected 270..300"
|
||||
)
|
||||
requested_names = {name.casefold() for name in args.members}
|
||||
selected = [
|
||||
area
|
||||
for area in municipalities
|
||||
if not requested_names
|
||||
or str(area["name"]).removeprefix("Gemeente ").split(" - ", 1)[0].casefold() in requested_names
|
||||
]
|
||||
if requested_names:
|
||||
found = {
|
||||
str(area["name"]).removeprefix("Gemeente ").split(" - ", 1)[0].casefold()
|
||||
for area in selected
|
||||
}
|
||||
missing = sorted(requested_names - found)
|
||||
if missing:
|
||||
raise RuntimeError(f"Unknown Flanders municipality selections: {', '.join(missing)}")
|
||||
if args.max_partitions:
|
||||
selected = selected[: args.max_partitions]
|
||||
|
||||
prior_by_area = {
|
||||
str(item.get("area_id")): item
|
||||
for item in (manifest.get("partitions") or [])
|
||||
if isinstance(item, dict) and item.get("area_id")
|
||||
}
|
||||
current: list[dict[str, Any]] = []
|
||||
failures: list[dict[str, Any]] = []
|
||||
observed_at = datetime.now(timezone.utc).isoformat()
|
||||
working_manifest = {
|
||||
"schema_version": 1,
|
||||
"status": "running",
|
||||
"source": "vmm_vha_bathymetry_profiles",
|
||||
"partition_scope_key": "flanders",
|
||||
"project_id": project["id"],
|
||||
"project_name": project["name"],
|
||||
"municipality_inventory_count": len(municipalities),
|
||||
"selected_partition_count": len(selected),
|
||||
"observed_at": observed_at,
|
||||
"partitions": current,
|
||||
}
|
||||
for index, area in enumerate(selected, start=1):
|
||||
identity = area_identity(area)
|
||||
prior = prior_by_area.get(str(area["id"]))
|
||||
if (
|
||||
not args.force
|
||||
and prior
|
||||
and prior.get("area_identity_sha256") == identity
|
||||
and prior.get("status") in {"complete", "no_profiles"}
|
||||
):
|
||||
result = prior
|
||||
else:
|
||||
try:
|
||||
result = acquire_partition(
|
||||
session,
|
||||
base_url=base_url,
|
||||
project_id=str(project["id"]),
|
||||
area=area,
|
||||
timeout=args.timeout,
|
||||
force=args.force,
|
||||
)
|
||||
except (RuntimeError, requests.RequestException) as exc:
|
||||
result = {
|
||||
"status": "failed",
|
||||
"area_id": area["id"],
|
||||
"area_name": area["name"],
|
||||
"area_identity_sha256": identity,
|
||||
"message": str(exc),
|
||||
"failed_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
failures.append(result)
|
||||
current.append(result)
|
||||
working_manifest["completed_partition_count"] = index
|
||||
working_manifest["partitions"] = current
|
||||
write_manifest(args.manifest_path, working_manifest)
|
||||
if result["status"] == "failed" and not args.continue_on_error:
|
||||
raise RuntimeError(f"Partition failed for {area['name']}: {result['message']}")
|
||||
|
||||
full_inventory_selected = len(selected) == len(municipalities) and not requested_names and not args.max_partitions
|
||||
complete = not failures and all(item["status"] in {"complete", "no_profiles"} for item in current)
|
||||
identity_payload, identity_sha = manifest_identity(
|
||||
project_id=str(project["id"]),
|
||||
partitions=current,
|
||||
)
|
||||
finalization = None
|
||||
if complete and full_inventory_selected:
|
||||
dataset_ids = [item["dataset_id"] for item in current if item["status"] == "complete"]
|
||||
no_profile_area_ids = [item["area_id"] for item in current if item["status"] == "no_profiles"]
|
||||
finalization = unwrap(
|
||||
session.post(
|
||||
f"{base_url}/api/v1/projects/{project['id']}/datasets/bathymetry/profiles/partitions/finalize",
|
||||
json={
|
||||
"partition_scope_key": "flanders",
|
||||
"expected_area_ids": [area["id"] for area in municipalities],
|
||||
"dataset_ids": dataset_ids,
|
||||
"no_profile_area_ids": no_profile_area_ids,
|
||||
"manifest_sha256": identity_sha,
|
||||
"observed_at": observed_at,
|
||||
},
|
||||
timeout=args.timeout,
|
||||
)
|
||||
)
|
||||
|
||||
working_manifest.update(
|
||||
{
|
||||
"status": "complete" if complete else "failed",
|
||||
"completed_at": datetime.now(timezone.utc).isoformat(),
|
||||
"coverage_manifest": identity_payload,
|
||||
"coverage_manifest_sha256": identity_sha,
|
||||
"regional_finalized": finalization is not None,
|
||||
"finalization": finalization,
|
||||
"failed_partition_count": len(failures),
|
||||
}
|
||||
)
|
||||
write_manifest(args.manifest_path, working_manifest)
|
||||
except (OSError, RuntimeError, ValueError, KeyError, requests.RequestException) as exc:
|
||||
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": working_manifest["status"],
|
||||
"project_id": project["id"],
|
||||
"municipality_inventory_count": len(municipalities),
|
||||
"processed_partition_count": len(current),
|
||||
"data_partition_count": sum(item["status"] == "complete" for item in current),
|
||||
"no_profile_partition_count": sum(item["status"] == "no_profiles" for item in current),
|
||||
"failed_partition_count": len(failures),
|
||||
"profile_count": sum(int(item.get("profile_count") or 0) for item in current),
|
||||
"document_count": sum(int(item.get("document_count") or 0) for item in current),
|
||||
"regional_finalized": finalization is not None,
|
||||
"manifest_path": str(args.manifest_path),
|
||||
"manifest_sha256": identity_sha,
|
||||
"finalization": finalization,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0 if working_manifest["status"] == "complete" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,220 @@
|
||||
"""Provision all current official Flemish municipality Areas.
|
||||
|
||||
The municipality inventory is discovered from the official VRBG RefGem
|
||||
collection at runtime. The script uses the existing geographic-scope artifact
|
||||
and API persistence flow; it never writes directly to PostGIS.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from geographic_scopes import GeographicScope, ScopeMember
|
||||
from provision_geographic_scope import (
|
||||
VRBG_ATTRIBUTION,
|
||||
VRBG_ITEMS_URL,
|
||||
build_scope_payloads,
|
||||
build_source_session,
|
||||
provision_scope,
|
||||
sha256_file,
|
||||
write_json_atomic,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||
DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-data/geographic-scopes")
|
||||
MIN_EXPECTED_MUNICIPALITIES = 270
|
||||
MAX_EXPECTED_MUNICIPALITIES = 300
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Provision the current official Flanders municipality scope.")
|
||||
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
|
||||
parser.add_argument(
|
||||
"--output-root",
|
||||
type=Path,
|
||||
default=Path(os.environ.get("GEOINTEL_SCOPE_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)),
|
||||
)
|
||||
parser.add_argument("--request-timeout", type=int, default=180)
|
||||
parser.add_argument("--import-timeout", type=int, default=3600)
|
||||
parser.add_argument("--force", action="store_true")
|
||||
parser.add_argument("--fetch-only", action="store_true")
|
||||
parser.add_argument("--min-municipalities", type=int, default=MIN_EXPECTED_MUNICIPALITIES)
|
||||
parser.add_argument("--max-municipalities", type=int, default=MAX_EXPECTED_MUNICIPALITIES)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def discover_flanders_scope(
|
||||
session: requests.Session,
|
||||
*,
|
||||
timeout: int,
|
||||
min_municipalities: int,
|
||||
max_municipalities: int,
|
||||
) -> tuple[GeographicScope, list[dict[str, Any]], str]:
|
||||
if min_municipalities <= 0 or max_municipalities < min_municipalities:
|
||||
raise ValueError("Municipality count safety limits are invalid")
|
||||
response = session.get(
|
||||
VRBG_ITEMS_URL,
|
||||
params={"f": "application/geo+json", "limit": "1000"},
|
||||
timeout=timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
features = payload.get("features")
|
||||
if not isinstance(features, list):
|
||||
raise RuntimeError("Official VRBG response does not contain a feature list")
|
||||
if not min_municipalities <= len(features) <= max_municipalities:
|
||||
raise RuntimeError(
|
||||
f"Official VRBG returned {len(features)} municipalities; expected "
|
||||
f"{min_municipalities}..{max_municipalities}. Refusing an incomplete or broadened scope."
|
||||
)
|
||||
|
||||
by_code: dict[str, dict[str, Any]] = {}
|
||||
names: set[str] = set()
|
||||
for feature in features:
|
||||
if not isinstance(feature, dict) or not isinstance(feature.get("geometry"), dict):
|
||||
raise RuntimeError("Official VRBG contains a malformed municipality feature")
|
||||
properties = feature.get("properties")
|
||||
if not isinstance(properties, dict):
|
||||
raise RuntimeError("Official VRBG municipality is missing properties")
|
||||
nis_code = str(properties.get("NISCODE") or "").strip()
|
||||
name = str(properties.get("NAAM") or "").strip()
|
||||
if len(nis_code) != 5 or not nis_code.isdigit() or not name:
|
||||
raise RuntimeError(f"Official VRBG municipality identity is invalid: {nis_code!r} / {name!r}")
|
||||
if nis_code in by_code or name.casefold() in names:
|
||||
raise RuntimeError(f"Official VRBG contains a duplicate municipality: {nis_code} / {name}")
|
||||
by_code[nis_code] = feature
|
||||
names.add(name.casefold())
|
||||
|
||||
ordered = [by_code[code] for code in sorted(by_code)]
|
||||
members = tuple(
|
||||
ScopeMember(
|
||||
name=str(feature["properties"]["NAAM"]).strip(),
|
||||
nis_code=str(feature["properties"]["NISCODE"]).strip(),
|
||||
)
|
||||
for feature in ordered
|
||||
)
|
||||
scope = GeographicScope(
|
||||
key="flanders",
|
||||
display_name=f"Vlaanderen ({len(members)} gemeenten)",
|
||||
project_name="Flanders Regional Workbench",
|
||||
project_region="Vlaanderen, België",
|
||||
area_name="Vlaanderen - officiële operationele grens",
|
||||
authority_name="Digitaal Vlaanderen VRBG",
|
||||
authority_url=VRBG_ITEMS_URL,
|
||||
scope_type="region",
|
||||
limitation_message=(
|
||||
"Officiële unie van de actuele VRBG-gemeentegrenzen. De operationele regio omvat het Vlaamse "
|
||||
"landgebied; territoriale zee, EEZ en continentaal plat zijn afzonderlijke maritieme scopes."
|
||||
),
|
||||
members=members,
|
||||
)
|
||||
return scope, ordered, response.url
|
||||
|
||||
|
||||
def prepare_artifacts(
|
||||
args: argparse.Namespace,
|
||||
scope: GeographicScope,
|
||||
features: list[dict[str, Any]],
|
||||
source_url: str,
|
||||
) -> tuple[Path, Path, dict[str, Any]]:
|
||||
output_dir = args.output_root / scope.key
|
||||
manifest_path = output_dir / "flanders_scope_manifest.json"
|
||||
generated_at = datetime.now(timezone.utc).isoformat()
|
||||
snapshot_date = generated_at[:10]
|
||||
boundary_path = output_dir / f"flanders_boundary_{snapshot_date}.geojson"
|
||||
members_path = output_dir / f"flanders_municipalities_{snapshot_date}.geojson"
|
||||
member_codes = list(scope.nis_codes)
|
||||
|
||||
if not args.force and manifest_path.is_file():
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
cached_boundary = output_dir / str(manifest.get("boundary_filename") or "")
|
||||
cached_members = output_dir / str(manifest.get("municipalities_filename") or "")
|
||||
if (
|
||||
manifest.get("status") == "complete"
|
||||
and manifest.get("member_nis_codes") == member_codes
|
||||
and cached_boundary.is_file()
|
||||
and cached_members.is_file()
|
||||
and sha256_file(cached_boundary) == manifest.get("boundary_sha256")
|
||||
and sha256_file(cached_members) == manifest.get("municipalities_sha256")
|
||||
):
|
||||
return cached_boundary, cached_members, manifest
|
||||
|
||||
boundary_payload, members_payload, summary = build_scope_payloads(
|
||||
scope,
|
||||
features,
|
||||
source_url=source_url,
|
||||
generated_at=generated_at,
|
||||
)
|
||||
write_json_atomic(boundary_path, boundary_payload)
|
||||
write_json_atomic(members_path, members_payload)
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"status": "complete",
|
||||
"generated_at": generated_at,
|
||||
"observed_at": f"{snapshot_date}T00:00:00Z",
|
||||
"boundary_filename": boundary_path.name,
|
||||
"boundary_sha256": sha256_file(boundary_path),
|
||||
"municipalities_filename": members_path.name,
|
||||
"municipalities_sha256": sha256_file(members_path),
|
||||
"vrbg_source_url": source_url,
|
||||
"vrbg_attribution": VRBG_ATTRIBUTION,
|
||||
"scope_authority_name": scope.authority_name,
|
||||
"scope_authority_url": scope.authority_url,
|
||||
"scope_limitation": scope.limitation_message,
|
||||
**summary,
|
||||
}
|
||||
write_json_atomic(manifest_path, manifest, pretty=True)
|
||||
return boundary_path, members_path, manifest
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
args.operator_tool = "provision_flanders_geographic_scope.py"
|
||||
try:
|
||||
with build_source_session() as session:
|
||||
scope, features, source_url = discover_flanders_scope(
|
||||
session,
|
||||
timeout=args.request_timeout,
|
||||
min_municipalities=args.min_municipalities,
|
||||
max_municipalities=args.max_municipalities,
|
||||
)
|
||||
boundary_path, members_path, manifest = prepare_artifacts(args, scope, features, source_url)
|
||||
workspace = None if args.fetch_only else provision_scope(
|
||||
args, scope, boundary_path, members_path, manifest
|
||||
)
|
||||
except (OSError, RuntimeError, ValueError, KeyError, requests.RequestException) as exc:
|
||||
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "ok",
|
||||
"mode": "fetch_only" if args.fetch_only else "provisioned",
|
||||
"scope": scope.key,
|
||||
"member_count": manifest["member_count"],
|
||||
"area_km2": manifest["area_km2"],
|
||||
"wgs84_bbox": manifest["wgs84_bbox"],
|
||||
"boundary_path": str(boundary_path),
|
||||
"municipalities_path": str(members_path),
|
||||
"workspace": workspace,
|
||||
"limitation": scope.limitation_message,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -478,7 +478,7 @@ def provision_scope(
|
||||
"attribution": VRBG_ATTRIBUTION,
|
||||
}
|
||||
common_provenance = {
|
||||
"operator_tool": "provision_geographic_scope.py",
|
||||
"operator_tool": getattr(args, "operator_tool", "provision_geographic_scope.py"),
|
||||
"operator_explicit_fetch": True,
|
||||
"manifest_path": str(args.output_root / scope.key / f"{scope.key.replace('-', '_')}_scope_manifest.json"),
|
||||
"source_url": manifest["vrbg_source_url"],
|
||||
|
||||
@@ -51,6 +51,9 @@ ${PYTHON_BIN} -m py_compile scripts/provision_mol_historical_landuse.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_historical_landuse.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_official_landuse_timeseries.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_waterinfo_station_history.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_flanders_geographic_scope.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_flanders_bathymetry_profiles.py
|
||||
${PYTHON_BIN} -m py_compile scripts/probe_mdk_bathymetry.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_bwk_natura2000.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py
|
||||
|
||||
Reference in New Issue
Block a user