Add Flemish bathymetry partition workflow
This commit is contained in:
@@ -43,6 +43,10 @@ BATHYMETRY_PROFILES_PAGE_SIZE=1000
|
|||||||
BATHYMETRY_PROFILES_MAX_FEATURES=50000
|
BATHYMETRY_PROFILES_MAX_FEATURES=50000
|
||||||
BATHYMETRY_PROFILES_TIMEOUT_SECONDS=120
|
BATHYMETRY_PROFILES_TIMEOUT_SECONDS=120
|
||||||
BATHYMETRY_PROFILES_MAX_RESPONSE_MB=32
|
BATHYMETRY_PROFILES_MAX_RESPONSE_MB=32
|
||||||
|
MDK_BATHYMETRY_PROBE_ENABLED=true
|
||||||
|
MDK_BATHYMETRY_WCS_URL=https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs
|
||||||
|
MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS=20
|
||||||
|
MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB=4
|
||||||
THEMATIC_RASTER_ENABLED=true
|
THEMATIC_RASTER_ENABLED=true
|
||||||
THEMATIC_RASTER_WCS_URL=https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs
|
THEMATIC_RASTER_WCS_URL=https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs
|
||||||
THEMATIC_RASTER_MIN_SIDE_M=100
|
THEMATIC_RASTER_MIN_SIDE_M=100
|
||||||
|
|||||||
@@ -7,6 +7,17 @@
|
|||||||
|
|
||||||
# Changelog
|
# Changelog
|
||||||
|
|
||||||
|
## Sprint 236 Flemish bathymetry partitions and safe North Sea probe (2026-07-17)
|
||||||
|
|
||||||
|
- Added dynamic provisioning of the complete official VRBG Flemish
|
||||||
|
municipality inventory, validated at 285 current Areas.
|
||||||
|
- Added an atomic, resumable VHA profile coordinator and server-side manifest
|
||||||
|
finalization that prevents partial partitions from appearing regional.
|
||||||
|
- Added a strict-TLS, size-bounded MDK WCS GetCapabilities readiness probe
|
||||||
|
without coverage download or insecure fallback.
|
||||||
|
- Exposed truthful MDK readiness and bathymetry partition-finalization API
|
||||||
|
contracts and packaged all operator scripts in the all-in-one image.
|
||||||
|
|
||||||
## Sprint 235 Governed bathymetry profiles and Belgian scale architecture (2026-07-17)
|
## Sprint 235 Governed bathymetry profiles and Belgian scale architecture (2026-07-17)
|
||||||
|
|
||||||
- Added bounded official VHA cross-section acquisition with exact Area
|
- Added bounded official VHA cross-section acquisition with exact Area
|
||||||
|
|||||||
@@ -1728,3 +1728,29 @@ The Dataset exposes profile count and nullable structured depth/width metrics.
|
|||||||
It does not claim a continuous bed model, current depth or volume. Use
|
It does not claim a continuous bed model, current depth or volume. Use
|
||||||
`scripts/provision_mol_bathymetry_profiles.py` for the canonical Mol operator
|
`scripts/provision_mol_bathymetry_profiles.py` for the canonical Mol operator
|
||||||
flow.
|
flow.
|
||||||
|
|
||||||
|
Provision the complete current Flemish land scope and then run the resumable
|
||||||
|
VHA municipality coordinator:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec geointel python /app/scripts/provision_flanders_geographic_scope.py
|
||||||
|
docker exec geointel python /app/scripts/provision_flanders_bathymetry_profiles.py
|
||||||
|
```
|
||||||
|
|
||||||
|
The first command discovers all current VRBG RefGem municipalities, validates
|
||||||
|
a 270..300 safety range and persists their exact Areas. The second writes its
|
||||||
|
manifest after every partition. Repeating it reuses completed source
|
||||||
|
identities; `--force` deliberately refreshes them. A partial `--members` or
|
||||||
|
`--max-partitions` run never marks regional coverage complete.
|
||||||
|
|
||||||
|
Inspect the MDK North Sea WCS without downloading coverage:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec geointel python /app/scripts/probe_mdk_bathymetry.py
|
||||||
|
```
|
||||||
|
|
||||||
|
Exit code `0` means verified capabilities; `2` means a truthful blocked
|
||||||
|
readiness state such as TLS or endpoint failure. Runtime controls are
|
||||||
|
`MDK_BATHYMETRY_PROBE_ENABLED`, `MDK_BATHYMETRY_WCS_URL`,
|
||||||
|
`MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS` and
|
||||||
|
`MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB`. TLS verification cannot be disabled.
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ from app.schemas import (
|
|||||||
FloodHazardAcquireRequest,
|
FloodHazardAcquireRequest,
|
||||||
FloodHazardPartitionSelectionRequest,
|
FloodHazardPartitionSelectionRequest,
|
||||||
FloodHazardSelectionRequest,
|
FloodHazardSelectionRequest,
|
||||||
|
BathymetryPartitionFinalizeRequest,
|
||||||
BathymetryProfileAcquireRequest,
|
BathymetryProfileAcquireRequest,
|
||||||
ThematicRasterAcquireRequest,
|
ThematicRasterAcquireRequest,
|
||||||
ThematicRasterSelectionRequest,
|
ThematicRasterSelectionRequest,
|
||||||
@@ -54,6 +55,7 @@ from app.services.terrain_analysis_service import TerrainAnalysisService
|
|||||||
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
|
from app.services.flood_hazard_acquisition_service import FloodHazardAcquisitionService
|
||||||
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
|
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
|
||||||
from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
|
from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
|
||||||
|
from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService
|
||||||
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
|
from app.services.thematic_raster_acquisition_service import ThematicRasterAcquisitionService
|
||||||
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
|
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
|
||||||
from app.utils.response import envelope
|
from app.utils.response import envelope
|
||||||
@@ -223,6 +225,13 @@ def list_bathymetry_sources(project_id: UUID, db: Session = Depends(get_db)):
|
|||||||
return envelope({"items": items, "total": len(items)})
|
return envelope({"items": items, "total": len(items)})
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/datasets/bathymetry/sources/mdk_bcp_bathymetry/readiness", response_model=dict)
|
||||||
|
def probe_mdk_bathymetry_readiness(project_id: UUID, db: Session = Depends(get_db)):
|
||||||
|
if not db.get(Project, project_id):
|
||||||
|
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||||
|
return envelope(MdkBathymetryProbeService.probe())
|
||||||
|
|
||||||
|
|
||||||
@router.post("/datasets/bathymetry/profiles/acquire", response_model=dict)
|
@router.post("/datasets/bathymetry/profiles/acquire", response_model=dict)
|
||||||
def acquire_bounded_bathymetry_profiles(
|
def acquire_bounded_bathymetry_profiles(
|
||||||
project_id: UUID,
|
project_id: UUID,
|
||||||
@@ -239,6 +248,15 @@ def acquire_bounded_bathymetry_profiles(
|
|||||||
return envelope(job)
|
return envelope(job)
|
||||||
|
|
||||||
|
|
||||||
|
@router.post("/datasets/bathymetry/profiles/partitions/finalize", response_model=dict)
|
||||||
|
def finalize_bathymetry_profile_partitions(
|
||||||
|
project_id: UUID,
|
||||||
|
payload: BathymetryPartitionFinalizeRequest,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
):
|
||||||
|
return envelope(BathymetryProfileAcquisitionService.finalize_partitions(db, project_id, payload))
|
||||||
|
|
||||||
|
|
||||||
@router.post("/datasets/thematic-raster/acquire", response_model=dict)
|
@router.post("/datasets/thematic-raster/acquire", response_model=dict)
|
||||||
def acquire_bounded_thematic_raster(
|
def acquire_bounded_thematic_raster(
|
||||||
project_id: UUID,
|
project_id: UUID,
|
||||||
|
|||||||
@@ -123,6 +123,23 @@ class Settings(BaseSettings):
|
|||||||
le=256,
|
le=256,
|
||||||
validation_alias="BATHYMETRY_PROFILES_MAX_RESPONSE_MB",
|
validation_alias="BATHYMETRY_PROFILES_MAX_RESPONSE_MB",
|
||||||
)
|
)
|
||||||
|
mdk_bathymetry_probe_enabled: bool = Field(default=True, validation_alias="MDK_BATHYMETRY_PROBE_ENABLED")
|
||||||
|
mdk_bathymetry_wcs_url: str = Field(
|
||||||
|
default="https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs",
|
||||||
|
validation_alias="MDK_BATHYMETRY_WCS_URL",
|
||||||
|
)
|
||||||
|
mdk_bathymetry_probe_timeout_seconds: int = Field(
|
||||||
|
default=20,
|
||||||
|
ge=1,
|
||||||
|
le=120,
|
||||||
|
validation_alias="MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS",
|
||||||
|
)
|
||||||
|
mdk_bathymetry_probe_max_response_mb: int = Field(
|
||||||
|
default=4,
|
||||||
|
ge=1,
|
||||||
|
le=16,
|
||||||
|
validation_alias="MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB",
|
||||||
|
)
|
||||||
thematic_raster_enabled: bool = Field(default=True, validation_alias="THEMATIC_RASTER_ENABLED")
|
thematic_raster_enabled: bool = Field(default=True, validation_alias="THEMATIC_RASTER_ENABLED")
|
||||||
thematic_raster_wcs_url: str = Field(
|
thematic_raster_wcs_url: str = Field(
|
||||||
default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs",
|
default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs",
|
||||||
|
|||||||
@@ -66,8 +66,11 @@ from .flood_hazard import (
|
|||||||
FloodHazardSelectionSummary,
|
FloodHazardSelectionSummary,
|
||||||
)
|
)
|
||||||
from .bathymetry import (
|
from .bathymetry import (
|
||||||
|
BathymetryPartitionFinalizeRequest,
|
||||||
|
BathymetryPartitionFinalizationResult,
|
||||||
BathymetryProfileAcquireRequest,
|
BathymetryProfileAcquireRequest,
|
||||||
BathymetryProfileAcquisitionResult,
|
BathymetryProfileAcquisitionResult,
|
||||||
|
BathymetrySourceProbeRead,
|
||||||
BathymetrySourceRead,
|
BathymetrySourceRead,
|
||||||
)
|
)
|
||||||
from .thematic_raster import (
|
from .thematic_raster import (
|
||||||
@@ -209,6 +212,9 @@ __all__ = [
|
|||||||
"FloodHazardSelectionSummary",
|
"FloodHazardSelectionSummary",
|
||||||
"BathymetryProfileAcquireRequest",
|
"BathymetryProfileAcquireRequest",
|
||||||
"BathymetryProfileAcquisitionResult",
|
"BathymetryProfileAcquisitionResult",
|
||||||
|
"BathymetryPartitionFinalizeRequest",
|
||||||
|
"BathymetryPartitionFinalizationResult",
|
||||||
|
"BathymetrySourceProbeRead",
|
||||||
"BathymetrySourceRead",
|
"BathymetrySourceRead",
|
||||||
"ThematicRasterAcquireRequest",
|
"ThematicRasterAcquireRequest",
|
||||||
"ThematicRasterAcquisitionResult",
|
"ThematicRasterAcquisitionResult",
|
||||||
|
|||||||
@@ -1,9 +1,10 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
from typing import Literal
|
from typing import Literal
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
|
|
||||||
from pydantic import BaseModel, Field
|
from pydantic import BaseModel, Field, field_validator
|
||||||
|
|
||||||
from .operations import VectorSelectionBBox
|
from .operations import VectorSelectionBBox
|
||||||
|
|
||||||
@@ -25,7 +26,7 @@ class BathymetrySourceRead(BaseModel):
|
|||||||
vertical_reference: str
|
vertical_reference: str
|
||||||
horizontal_crs: str
|
horizontal_crs: str
|
||||||
native_resolution: str | None = None
|
native_resolution: str | None = None
|
||||||
integration_status: Literal["operational", "available_not_integrated", "catalog_only"]
|
integration_status: Literal["operational", "probe_only", "available_not_integrated", "catalog_only"]
|
||||||
acquisition_supported: bool
|
acquisition_supported: bool
|
||||||
configured: bool
|
configured: bool
|
||||||
service_url: str | None = None
|
service_url: str | None = None
|
||||||
@@ -50,3 +51,61 @@ class BathymetryProfileAcquisitionResult(BaseModel):
|
|||||||
measurement_date_max: str | None = None
|
measurement_date_max: str | None = None
|
||||||
attribution: str
|
attribution: str
|
||||||
limitation_message: str
|
limitation_message: str
|
||||||
|
|
||||||
|
|
||||||
|
class BathymetryPartitionFinalizeRequest(BaseModel):
|
||||||
|
partition_scope_key: str = Field(min_length=1, max_length=120, pattern=r"^[a-z0-9][a-z0-9_-]*$")
|
||||||
|
expected_area_ids: list[UUID] = Field(min_length=1, max_length=500)
|
||||||
|
dataset_ids: list[UUID] = Field(default_factory=list, max_length=500)
|
||||||
|
no_profile_area_ids: list[UUID] = Field(default_factory=list, max_length=500)
|
||||||
|
manifest_sha256: str = Field(pattern=r"^[a-f0-9]{64}$")
|
||||||
|
observed_at: datetime
|
||||||
|
|
||||||
|
@field_validator("expected_area_ids", "dataset_ids", "no_profile_area_ids")
|
||||||
|
@classmethod
|
||||||
|
def require_unique_ids(cls, value: list[UUID]) -> list[UUID]:
|
||||||
|
if len(value) != len(set(value)):
|
||||||
|
raise ValueError("Partition identifiers must be unique")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
class BathymetryPartitionFinalizationResult(BaseModel):
|
||||||
|
partition_scope_key: str
|
||||||
|
regional_partitions_complete: bool
|
||||||
|
partition_count: int = Field(ge=1)
|
||||||
|
data_partition_count: int = Field(ge=0)
|
||||||
|
no_profile_partition_count: int = Field(ge=0)
|
||||||
|
profile_count: int = Field(ge=0)
|
||||||
|
document_count: int = Field(ge=0)
|
||||||
|
structured_depth_count: int = Field(ge=0)
|
||||||
|
measurement_date_min: str | None = None
|
||||||
|
measurement_date_max: str | None = None
|
||||||
|
dataset_ids: list[UUID]
|
||||||
|
manifest_sha256: str
|
||||||
|
observed_at: datetime
|
||||||
|
limitation_message: str
|
||||||
|
|
||||||
|
|
||||||
|
class BathymetrySourceProbeRead(BaseModel):
|
||||||
|
source_key: str
|
||||||
|
status: Literal[
|
||||||
|
"disabled",
|
||||||
|
"invalid_configuration",
|
||||||
|
"tls_error",
|
||||||
|
"endpoint_unavailable",
|
||||||
|
"invalid_capabilities",
|
||||||
|
"reachable",
|
||||||
|
]
|
||||||
|
configured_url: str
|
||||||
|
capabilities_url: str | None = None
|
||||||
|
tls_verified: bool
|
||||||
|
capabilities_reachable: bool
|
||||||
|
acquisition_supported: bool = False
|
||||||
|
wcs_version: str | None = None
|
||||||
|
coverage_identifiers: list[str] = Field(default_factory=list)
|
||||||
|
advertised_formats: list[str] = Field(default_factory=list)
|
||||||
|
advertised_crs: list[str] = Field(default_factory=list)
|
||||||
|
response_sha256: str | None = None
|
||||||
|
checked_at: datetime
|
||||||
|
message: str
|
||||||
|
limitation_message: str
|
||||||
|
|||||||
@@ -15,8 +15,10 @@ from shapely.geometry import Point, box, mapping
|
|||||||
|
|
||||||
from app.core.config import Settings, get_settings
|
from app.core.config import Settings, get_settings
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.models import Area, Dataset, Project
|
from app.models import Area, Dataset, DatasetVersion, Project
|
||||||
from app.schemas.bathymetry import (
|
from app.schemas.bathymetry import (
|
||||||
|
BathymetryPartitionFinalizeRequest,
|
||||||
|
BathymetryPartitionFinalizationResult,
|
||||||
BathymetryProfileAcquireRequest,
|
BathymetryProfileAcquireRequest,
|
||||||
BathymetryProfileAcquisitionResult,
|
BathymetryProfileAcquisitionResult,
|
||||||
BathymetrySourceRead,
|
BathymetrySourceRead,
|
||||||
@@ -70,16 +72,16 @@ class BathymetryProfileAcquisitionService:
|
|||||||
"vertical_reference": "LAT",
|
"vertical_reference": "LAT",
|
||||||
"horizontal_crs": "bronafhankelijk; expliciet per WCS-respons",
|
"horizontal_crs": "bronafhankelijk; expliciet per WCS-respons",
|
||||||
"native_resolution": "20 x 20 m",
|
"native_resolution": "20 x 20 m",
|
||||||
"integration_status": "available_not_integrated",
|
"integration_status": "probe_only",
|
||||||
"acquisition_supported": False,
|
"acquisition_supported": False,
|
||||||
"configured": False,
|
"configured": False,
|
||||||
"service_url": "https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/WCS_Public",
|
"service_url": "https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs",
|
||||||
"catalog_url": "https://www.vlaanderen.be/datavindplaats/catalogus/dieptemodel-van-de-zeebodem-belgisch-continentaal-plat-noordzee",
|
"catalog_url": "https://www.vlaanderen.be/datavindplaats/catalogus/dieptemodel-van-de-zeebodem-belgisch-continentaal-plat-noordzee",
|
||||||
"attribution": "Agentschap Maritieme Dienstverlening en Kust",
|
"attribution": "Agentschap Maritieme Dienstverlening en Kust",
|
||||||
"license_note": "Zie de officiële datasetmetadata en gebruiksvoorwaarden.",
|
"license_note": "Zie de officiële datasetmetadata en gebruiksvoorwaarden.",
|
||||||
"limitation_message": (
|
"limitation_message": (
|
||||||
"Niet operationeel in GeoIntel totdat WCS, maritieme begrenzing, tegels, LAT-semantiek en "
|
"Alleen een read-only GetCapabilities-probe is beschikbaar. Rasteracquisitie blijft uit totdat "
|
||||||
"servercertificaten in een live integratietest zijn gevalideerd."
|
"WCS, maritieme begrenzing, tegels, LAT-semantiek en servercertificaten live zijn gevalideerd."
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -404,6 +406,8 @@ class BathymetryProfileAcquisitionService:
|
|||||||
raw_features: list[dict[str, Any]],
|
raw_features: list[dict[str, Any]],
|
||||||
scope_geometry,
|
scope_geometry,
|
||||||
watercourse_names: dict[int, dict[str, str | None]],
|
watercourse_names: dict[int, dict[str, str | None]],
|
||||||
|
*,
|
||||||
|
partition_properties: dict[str, Any] | None = None,
|
||||||
) -> tuple[dict[str, Any], dict[str, Any]]:
|
) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||||
normalized: list[dict[str, Any]] = []
|
normalized: list[dict[str, Any]] = []
|
||||||
dates: list[str] = []
|
dates: list[str] = []
|
||||||
@@ -465,6 +469,7 @@ class BathymetryProfileAcquisitionService:
|
|||||||
"provider": BathymetryProfileAcquisitionService.PROVIDER,
|
"provider": BathymetryProfileAcquisitionService.PROVIDER,
|
||||||
"measurement_semantics": "historical_cross_section_profile_point",
|
"measurement_semantics": "historical_cross_section_profile_point",
|
||||||
"vertical_reference": "document-specific",
|
"vertical_reference": "document-specific",
|
||||||
|
**(partition_properties or {}),
|
||||||
},
|
},
|
||||||
"geometry": mapping(point),
|
"geometry": mapping(point),
|
||||||
}
|
}
|
||||||
@@ -487,6 +492,17 @@ class BathymetryProfileAcquisitionService:
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _municipality_name(area: Area | None) -> str | None:
|
||||||
|
if area is None:
|
||||||
|
return None
|
||||||
|
normalized = str(area.name or "").strip()
|
||||||
|
prefix = "Gemeente "
|
||||||
|
if not normalized.casefold().startswith(prefix.casefold()):
|
||||||
|
return None
|
||||||
|
municipality = normalized[len(prefix) :].split(" - ", 1)[0].strip()
|
||||||
|
return municipality or None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None:
|
def _cached_dataset(db, project_id: UUID, filename: str) -> Dataset | None:
|
||||||
return (
|
return (
|
||||||
@@ -541,6 +557,8 @@ class BathymetryProfileAcquisitionService:
|
|||||||
scope_geometry = BathymetryProfileAcquisitionService._scope_geometry(
|
scope_geometry = BathymetryProfileAcquisitionService._scope_geometry(
|
||||||
db, project_id, payload.area_id, bbox_values
|
db, project_id, payload.area_id, bbox_values
|
||||||
)
|
)
|
||||||
|
area = db.get(Area, payload.area_id) if payload.area_id else None
|
||||||
|
municipality = BathymetryProfileAcquisitionService._municipality_name(area)
|
||||||
exact_bbox = tuple(float(value) for value in scope_geometry.bounds)
|
exact_bbox = tuple(float(value) for value in scope_geometry.bounds)
|
||||||
request_identity = {
|
request_identity = {
|
||||||
"provider": BathymetryProfileAcquisitionService.PROVIDER,
|
"provider": BathymetryProfileAcquisitionService.PROVIDER,
|
||||||
@@ -568,7 +586,16 @@ class BathymetryProfileAcquisitionService:
|
|||||||
vhag_codes, resolved_settings, opener
|
vhag_codes, resolved_settings, opener
|
||||||
)
|
)
|
||||||
feature_collection, summary = BathymetryProfileAcquisitionService._normalize_features(
|
feature_collection, summary = BathymetryProfileAcquisitionService._normalize_features(
|
||||||
raw_features, scope_geometry, names
|
raw_features,
|
||||||
|
scope_geometry,
|
||||||
|
names,
|
||||||
|
partition_properties={
|
||||||
|
"partition_area_id": str(area.id),
|
||||||
|
"partition_area_name": area.name,
|
||||||
|
**({"municipality": municipality} if municipality else {}),
|
||||||
|
}
|
||||||
|
if area
|
||||||
|
else None,
|
||||||
)
|
)
|
||||||
if summary["profile_count"] == 0:
|
if summary["profile_count"] == 0:
|
||||||
raise AppError(
|
raise AppError(
|
||||||
@@ -585,6 +612,11 @@ class BathymetryProfileAcquisitionService:
|
|||||||
"theme": "bathymetry",
|
"theme": "bathymetry",
|
||||||
"layer_name": "bathymetry_profiles",
|
"layer_name": "bathymetry_profiles",
|
||||||
"coverage_scope": "municipality" if payload.area_id else "bounded_selection",
|
"coverage_scope": "municipality" if payload.area_id else "bounded_selection",
|
||||||
|
"partition_area_id": str(area.id) if area else None,
|
||||||
|
"partition_area_name": area.name if area else None,
|
||||||
|
"municipality": municipality,
|
||||||
|
"partitioned_source_audit": False,
|
||||||
|
"regional_partitions_complete": False,
|
||||||
"bbox_epsg4326": list(exact_bbox),
|
"bbox_epsg4326": list(exact_bbox),
|
||||||
**summary,
|
**summary,
|
||||||
"selection_aggregation": {
|
"selection_aggregation": {
|
||||||
@@ -669,3 +701,160 @@ class BathymetryProfileAcquisitionService:
|
|||||||
)
|
)
|
||||||
persisted = db.get(Dataset, dataset.id)
|
persisted = db.get(Dataset, dataset.id)
|
||||||
return BathymetryProfileAcquisitionService._result(persisted, reused=False)
|
return BathymetryProfileAcquisitionService._result(persisted, reused=False)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def finalize_partitions(
|
||||||
|
db,
|
||||||
|
project_id: UUID,
|
||||||
|
payload: BathymetryPartitionFinalizeRequest,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
if not db.get(Project, project_id):
|
||||||
|
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||||
|
|
||||||
|
expected_area_ids = set(payload.expected_area_ids)
|
||||||
|
dataset_ids = set(payload.dataset_ids)
|
||||||
|
no_profile_area_ids = set(payload.no_profile_area_ids)
|
||||||
|
|
||||||
|
areas: dict[UUID, Area] = {}
|
||||||
|
for area_id in expected_area_ids:
|
||||||
|
area = db.get(Area, area_id)
|
||||||
|
if area is None or area.project_id != project_id:
|
||||||
|
raise AppError(
|
||||||
|
code="BATHYMETRY_PARTITION_AREA_INVALID",
|
||||||
|
message="Every expected partition Area must belong to the project",
|
||||||
|
details={"area_id": str(area_id)},
|
||||||
|
status_code=400,
|
||||||
|
)
|
||||||
|
if BathymetryProfileAcquisitionService._municipality_name(area) is None:
|
||||||
|
raise AppError(
|
||||||
|
code="BATHYMETRY_PARTITION_AREA_INVALID",
|
||||||
|
message="Bathymetry partitions must use persisted municipality Areas",
|
||||||
|
details={"area_id": str(area_id), "area_name": area.name},
|
||||||
|
status_code=400,
|
||||||
|
)
|
||||||
|
areas[area_id] = area
|
||||||
|
|
||||||
|
if not no_profile_area_ids.issubset(expected_area_ids):
|
||||||
|
raise AppError(
|
||||||
|
code="BATHYMETRY_PARTITION_MANIFEST_INVALID",
|
||||||
|
message="No-profile partitions must be part of the expected Area set",
|
||||||
|
status_code=400,
|
||||||
|
)
|
||||||
|
|
||||||
|
datasets: list[Dataset] = []
|
||||||
|
data_area_ids: set[UUID] = set()
|
||||||
|
for dataset_id in payload.dataset_ids:
|
||||||
|
dataset = db.get(Dataset, dataset_id)
|
||||||
|
if (
|
||||||
|
dataset is None
|
||||||
|
or dataset.project_id != project_id
|
||||||
|
or dataset.source_name != BathymetryProfileAcquisitionService.PROVIDER
|
||||||
|
or dataset.status != "ready"
|
||||||
|
or dataset.area_id not in expected_area_ids
|
||||||
|
):
|
||||||
|
raise AppError(
|
||||||
|
code="BATHYMETRY_PARTITION_DATASET_INVALID",
|
||||||
|
message="Every partition Dataset must be a ready VHA profile Dataset scoped to an expected Area",
|
||||||
|
details={"dataset_id": str(dataset_id)},
|
||||||
|
status_code=400,
|
||||||
|
)
|
||||||
|
if dataset.area_id in data_area_ids:
|
||||||
|
raise AppError(
|
||||||
|
code="BATHYMETRY_PARTITION_DATASET_DUPLICATE",
|
||||||
|
message="A complete manifest may reference only one profile Dataset per Area",
|
||||||
|
details={"area_id": str(dataset.area_id)},
|
||||||
|
status_code=400,
|
||||||
|
)
|
||||||
|
data_area_ids.add(dataset.area_id)
|
||||||
|
datasets.append(dataset)
|
||||||
|
|
||||||
|
accounted_area_ids = data_area_ids.union(no_profile_area_ids)
|
||||||
|
if accounted_area_ids != expected_area_ids:
|
||||||
|
raise AppError(
|
||||||
|
code="BATHYMETRY_PARTITION_MANIFEST_INCOMPLETE",
|
||||||
|
message="Every expected Area must have one ready Dataset or an explicit no-profile result",
|
||||||
|
details={
|
||||||
|
"missing_area_ids": sorted(str(value) for value in expected_area_ids - accounted_area_ids),
|
||||||
|
"unexpected_area_ids": sorted(str(value) for value in accounted_area_ids - expected_area_ids),
|
||||||
|
},
|
||||||
|
status_code=400,
|
||||||
|
)
|
||||||
|
|
||||||
|
profile_count = 0
|
||||||
|
document_count = 0
|
||||||
|
structured_depth_count = 0
|
||||||
|
dates_min: list[str] = []
|
||||||
|
dates_max: list[str] = []
|
||||||
|
shared_metadata = {
|
||||||
|
"partition_scope_key": payload.partition_scope_key,
|
||||||
|
"partition_count": len(expected_area_ids),
|
||||||
|
"data_partition_count": len(datasets),
|
||||||
|
"no_profile_partition_count": len(no_profile_area_ids),
|
||||||
|
"partition_manifest_sha256": payload.manifest_sha256,
|
||||||
|
"partition_manifest_observed_at": payload.observed_at.isoformat(),
|
||||||
|
"partitioned_source_audit": True,
|
||||||
|
"regional_partitions_complete": True,
|
||||||
|
}
|
||||||
|
shared_provenance = {
|
||||||
|
"partition_manifest_sha256": payload.manifest_sha256,
|
||||||
|
"partition_manifest_observed_at": payload.observed_at.isoformat(),
|
||||||
|
"partition_scope_key": payload.partition_scope_key,
|
||||||
|
"regional_partitions_complete": True,
|
||||||
|
"no_profile_area_ids": sorted(str(value) for value in no_profile_area_ids),
|
||||||
|
}
|
||||||
|
|
||||||
|
for dataset in datasets:
|
||||||
|
source_metadata = dict(dataset.source_metadata or {})
|
||||||
|
provenance_metadata = dict(dataset.provenance_metadata or {})
|
||||||
|
profile_count += int(source_metadata.get("profile_count") or 0)
|
||||||
|
document_count += int(source_metadata.get("document_count") or 0)
|
||||||
|
structured_depth_count += int(source_metadata.get("structured_depth_count") or 0)
|
||||||
|
if source_metadata.get("measurement_date_min"):
|
||||||
|
dates_min.append(str(source_metadata["measurement_date_min"]))
|
||||||
|
if source_metadata.get("measurement_date_max"):
|
||||||
|
dates_max.append(str(source_metadata["measurement_date_max"]))
|
||||||
|
area = areas[dataset.area_id]
|
||||||
|
source_metadata.update(
|
||||||
|
{
|
||||||
|
**shared_metadata,
|
||||||
|
"coverage_scope": payload.partition_scope_key,
|
||||||
|
"partition_area_id": str(area.id),
|
||||||
|
"partition_area_name": area.name,
|
||||||
|
"municipality": BathymetryProfileAcquisitionService._municipality_name(area),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
provenance_metadata.update(shared_provenance)
|
||||||
|
dataset.source_metadata = source_metadata
|
||||||
|
dataset.provenance_metadata = provenance_metadata
|
||||||
|
|
||||||
|
if dataset_ids:
|
||||||
|
versions = (
|
||||||
|
db.query(DatasetVersion)
|
||||||
|
.filter(DatasetVersion.dataset_id.in_(dataset_ids))
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
for version in versions:
|
||||||
|
version.source_metadata = dict(
|
||||||
|
next(dataset.source_metadata for dataset in datasets if dataset.id == version.dataset_id)
|
||||||
|
)
|
||||||
|
version.provenance_metadata = dict(
|
||||||
|
next(dataset.provenance_metadata for dataset in datasets if dataset.id == version.dataset_id)
|
||||||
|
)
|
||||||
|
|
||||||
|
db.commit()
|
||||||
|
return BathymetryPartitionFinalizationResult(
|
||||||
|
partition_scope_key=payload.partition_scope_key,
|
||||||
|
regional_partitions_complete=True,
|
||||||
|
partition_count=len(expected_area_ids),
|
||||||
|
data_partition_count=len(datasets),
|
||||||
|
no_profile_partition_count=len(no_profile_area_ids),
|
||||||
|
profile_count=profile_count,
|
||||||
|
document_count=document_count,
|
||||||
|
structured_depth_count=structured_depth_count,
|
||||||
|
measurement_date_min=min(dates_min) if dates_min else None,
|
||||||
|
measurement_date_max=max(dates_max) if dates_max else None,
|
||||||
|
dataset_ids=payload.dataset_ids,
|
||||||
|
manifest_sha256=payload.manifest_sha256,
|
||||||
|
observed_at=payload.observed_at,
|
||||||
|
limitation_message=BathymetryProfileAcquisitionService.LIMITATION,
|
||||||
|
).model_dump(mode="json")
|
||||||
|
|||||||
@@ -0,0 +1,253 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
import hashlib
|
||||||
|
import ssl
|
||||||
|
from typing import Any, Callable
|
||||||
|
from urllib.error import HTTPError, URLError
|
||||||
|
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||||
|
from urllib.request import Request, urlopen
|
||||||
|
from xml.etree import ElementTree
|
||||||
|
|
||||||
|
from app.core.config import Settings, get_settings
|
||||||
|
from app.schemas.bathymetry import BathymetrySourceProbeRead
|
||||||
|
|
||||||
|
|
||||||
|
class MdkBathymetryProbeService:
|
||||||
|
SOURCE_KEY = "mdk_bcp_bathymetry"
|
||||||
|
LIMITATION = (
|
||||||
|
"Deze probe leest alleen WCS GetCapabilities met strikte TLS-controle. "
|
||||||
|
"GeoIntel downloadt of activeert geen Noordzee-raster totdat endpoint, coverage-id, CRS, LAT, "
|
||||||
|
"nodata, resolutie, begrenzing en responslimieten live zijn gevalideerd."
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _capabilities_url(configured_url: str) -> str:
|
||||||
|
parsed = urlsplit(configured_url.strip())
|
||||||
|
if parsed.scheme.lower() != "https" or not parsed.hostname:
|
||||||
|
raise ValueError("The MDK WCS probe requires an absolute HTTPS URL")
|
||||||
|
parameters = dict(parse_qsl(parsed.query, keep_blank_values=True))
|
||||||
|
parameters.update({"service": "WCS", "request": "GetCapabilities", "version": "1.0.0"})
|
||||||
|
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(parameters), ""))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _read_capabilities(
|
||||||
|
capabilities_url: str,
|
||||||
|
settings: Settings,
|
||||||
|
opener: Callable[..., Any] | None,
|
||||||
|
) -> tuple[bytes, str]:
|
||||||
|
request = Request(
|
||||||
|
capabilities_url,
|
||||||
|
headers={
|
||||||
|
"Accept": "application/xml,text/xml;q=0.9,*/*;q=0.1",
|
||||||
|
"User-Agent": "GeoIntel/1.0 MDK-bathymetry-readiness-probe",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
with (opener or urlopen)(request, timeout=settings.mdk_bathymetry_probe_timeout_seconds) as response:
|
||||||
|
limit = settings.mdk_bathymetry_probe_max_response_mb * 1024 * 1024
|
||||||
|
content = response.read(limit + 1)
|
||||||
|
if len(content) > limit:
|
||||||
|
raise ValueError("MDK WCS capabilities response exceeded the configured size limit")
|
||||||
|
content_type = str(response.headers.get("Content-Type") or "") if hasattr(response, "headers") else ""
|
||||||
|
return content, content_type
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _local_name(tag: str) -> str:
|
||||||
|
return tag.rsplit("}", 1)[-1].casefold()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _parse_capabilities(content: bytes) -> dict[str, Any]:
|
||||||
|
root = ElementTree.fromstring(content)
|
||||||
|
root_name = MdkBathymetryProbeService._local_name(root.tag)
|
||||||
|
if "capabilities" not in root_name:
|
||||||
|
raise ValueError("MDK endpoint did not return a WCS capabilities document")
|
||||||
|
|
||||||
|
coverage_identifiers: set[str] = set()
|
||||||
|
advertised_formats: set[str] = set()
|
||||||
|
advertised_crs: set[str] = set()
|
||||||
|
for element in root.iter():
|
||||||
|
local_name = MdkBathymetryProbeService._local_name(element.tag)
|
||||||
|
text = (element.text or "").strip()
|
||||||
|
if local_name in {"coverageofferingbrief", "coverageoffering"}:
|
||||||
|
for child in element:
|
||||||
|
if MdkBathymetryProbeService._local_name(child.tag) in {"name", "identifier"}:
|
||||||
|
identifier = (child.text or "").strip()
|
||||||
|
if identifier:
|
||||||
|
coverage_identifiers.add(identifier)
|
||||||
|
break
|
||||||
|
if local_name in {"format", "formats"} and text:
|
||||||
|
advertised_formats.add(text)
|
||||||
|
if local_name in {"requestresponsecrss", "requestcrss", "responsecrss", "nativecrss", "crs"} and text:
|
||||||
|
advertised_crs.add(text)
|
||||||
|
for attribute_value in element.attrib.values():
|
||||||
|
normalized = str(attribute_value).strip()
|
||||||
|
if "EPSG" in normalized.upper() or "CRS:" in normalized.upper():
|
||||||
|
advertised_crs.add(normalized)
|
||||||
|
|
||||||
|
return {
|
||||||
|
"wcs_version": str(root.attrib.get("version") or "") or None,
|
||||||
|
"coverage_identifiers": sorted(coverage_identifiers),
|
||||||
|
"advertised_formats": sorted(advertised_formats),
|
||||||
|
"advertised_crs": sorted(advertised_crs),
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _result(
|
||||||
|
*,
|
||||||
|
settings: Settings,
|
||||||
|
status: str,
|
||||||
|
checked_at: datetime,
|
||||||
|
message: str,
|
||||||
|
capabilities_url: str | None = None,
|
||||||
|
tls_verified: bool = False,
|
||||||
|
capabilities_reachable: bool = False,
|
||||||
|
response_sha256: str | None = None,
|
||||||
|
parsed: dict[str, Any] | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
parsed = parsed or {}
|
||||||
|
return BathymetrySourceProbeRead(
|
||||||
|
source_key=MdkBathymetryProbeService.SOURCE_KEY,
|
||||||
|
status=status,
|
||||||
|
configured_url=settings.mdk_bathymetry_wcs_url,
|
||||||
|
capabilities_url=capabilities_url,
|
||||||
|
tls_verified=tls_verified,
|
||||||
|
capabilities_reachable=capabilities_reachable,
|
||||||
|
acquisition_supported=False,
|
||||||
|
wcs_version=parsed.get("wcs_version"),
|
||||||
|
coverage_identifiers=parsed.get("coverage_identifiers") or [],
|
||||||
|
advertised_formats=parsed.get("advertised_formats") or [],
|
||||||
|
advertised_crs=parsed.get("advertised_crs") or [],
|
||||||
|
response_sha256=response_sha256,
|
||||||
|
checked_at=checked_at,
|
||||||
|
message=message,
|
||||||
|
limitation_message=MdkBathymetryProbeService.LIMITATION,
|
||||||
|
).model_dump(mode="json")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def probe(
|
||||||
|
*,
|
||||||
|
settings: Settings | None = None,
|
||||||
|
opener: Callable[..., Any] | None = None,
|
||||||
|
checked_at: datetime | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
resolved_settings = settings or get_settings()
|
||||||
|
now = checked_at or datetime.now(UTC)
|
||||||
|
if not resolved_settings.mdk_bathymetry_probe_enabled:
|
||||||
|
return MdkBathymetryProbeService._result(
|
||||||
|
settings=resolved_settings,
|
||||||
|
status="disabled",
|
||||||
|
checked_at=now,
|
||||||
|
message="MDK bathymetry readiness probing is disabled.",
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
capabilities_url = MdkBathymetryProbeService._capabilities_url(
|
||||||
|
resolved_settings.mdk_bathymetry_wcs_url
|
||||||
|
)
|
||||||
|
except ValueError as exc:
|
||||||
|
return MdkBathymetryProbeService._result(
|
||||||
|
settings=resolved_settings,
|
||||||
|
status="invalid_configuration",
|
||||||
|
checked_at=now,
|
||||||
|
message=str(exc),
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
content, content_type = MdkBathymetryProbeService._read_capabilities(
|
||||||
|
capabilities_url, resolved_settings, opener
|
||||||
|
)
|
||||||
|
except HTTPError as exc:
|
||||||
|
return MdkBathymetryProbeService._result(
|
||||||
|
settings=resolved_settings,
|
||||||
|
status="endpoint_unavailable",
|
||||||
|
checked_at=now,
|
||||||
|
capabilities_url=capabilities_url,
|
||||||
|
tls_verified=True,
|
||||||
|
message=f"MDK WCS GetCapabilities returned HTTP {exc.code}.",
|
||||||
|
)
|
||||||
|
except ssl.SSLCertVerificationError:
|
||||||
|
return MdkBathymetryProbeService._result(
|
||||||
|
settings=resolved_settings,
|
||||||
|
status="tls_error",
|
||||||
|
checked_at=now,
|
||||||
|
capabilities_url=capabilities_url,
|
||||||
|
message="MDK WCS TLS certificate validation failed; insecure fallback is prohibited.",
|
||||||
|
)
|
||||||
|
except URLError as exc:
|
||||||
|
reason = exc.reason
|
||||||
|
is_tls_error = isinstance(reason, (ssl.SSLError, ssl.CertificateError)) or "certificate" in str(
|
||||||
|
reason
|
||||||
|
).casefold()
|
||||||
|
return MdkBathymetryProbeService._result(
|
||||||
|
settings=resolved_settings,
|
||||||
|
status="tls_error" if is_tls_error else "endpoint_unavailable",
|
||||||
|
checked_at=now,
|
||||||
|
capabilities_url=capabilities_url,
|
||||||
|
message=(
|
||||||
|
"MDK WCS TLS certificate validation failed; insecure fallback is prohibited."
|
||||||
|
if is_tls_error
|
||||||
|
else "MDK WCS GetCapabilities could not be reached."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
except (TimeoutError, OSError) as exc:
|
||||||
|
return MdkBathymetryProbeService._result(
|
||||||
|
settings=resolved_settings,
|
||||||
|
status="endpoint_unavailable",
|
||||||
|
checked_at=now,
|
||||||
|
capabilities_url=capabilities_url,
|
||||||
|
message=f"MDK WCS GetCapabilities could not be reached ({type(exc).__name__}).",
|
||||||
|
)
|
||||||
|
except ValueError as exc:
|
||||||
|
return MdkBathymetryProbeService._result(
|
||||||
|
settings=resolved_settings,
|
||||||
|
status="invalid_capabilities",
|
||||||
|
checked_at=now,
|
||||||
|
capabilities_url=capabilities_url,
|
||||||
|
tls_verified=True,
|
||||||
|
capabilities_reachable=True,
|
||||||
|
message=str(exc),
|
||||||
|
)
|
||||||
|
|
||||||
|
response_sha256 = hashlib.sha256(content).hexdigest()
|
||||||
|
try:
|
||||||
|
parsed = MdkBathymetryProbeService._parse_capabilities(content)
|
||||||
|
except (ElementTree.ParseError, ValueError) as exc:
|
||||||
|
detail = " ".join(str(exc).split())
|
||||||
|
if content_type:
|
||||||
|
detail = f"{detail} Content-Type: {content_type}."
|
||||||
|
return MdkBathymetryProbeService._result(
|
||||||
|
settings=resolved_settings,
|
||||||
|
status="invalid_capabilities",
|
||||||
|
checked_at=now,
|
||||||
|
capabilities_url=capabilities_url,
|
||||||
|
tls_verified=True,
|
||||||
|
capabilities_reachable=True,
|
||||||
|
response_sha256=response_sha256,
|
||||||
|
message=detail,
|
||||||
|
)
|
||||||
|
|
||||||
|
if not parsed["coverage_identifiers"]:
|
||||||
|
return MdkBathymetryProbeService._result(
|
||||||
|
settings=resolved_settings,
|
||||||
|
status="invalid_capabilities",
|
||||||
|
checked_at=now,
|
||||||
|
capabilities_url=capabilities_url,
|
||||||
|
tls_verified=True,
|
||||||
|
capabilities_reachable=True,
|
||||||
|
response_sha256=response_sha256,
|
||||||
|
parsed=parsed,
|
||||||
|
message="MDK WCS capabilities are reachable but advertise no coverage identifier.",
|
||||||
|
)
|
||||||
|
return MdkBathymetryProbeService._result(
|
||||||
|
settings=resolved_settings,
|
||||||
|
status="reachable",
|
||||||
|
checked_at=now,
|
||||||
|
capabilities_url=capabilities_url,
|
||||||
|
tls_verified=True,
|
||||||
|
capabilities_reachable=True,
|
||||||
|
response_sha256=response_sha256,
|
||||||
|
parsed=parsed,
|
||||||
|
message=(
|
||||||
|
"MDK WCS capabilities are reachable with verified TLS. "
|
||||||
|
"Raster acquisition remains disabled pending bounded coverage validation."
|
||||||
|
),
|
||||||
|
)
|
||||||
@@ -282,8 +282,11 @@ def test_bathymetry_acquisition_persists_reference_dataset_through_dataset_servi
|
|||||||
assert captured["reference_layer_name"] == "bathymetry_profiles"
|
assert captured["reference_layer_name"] == "bathymetry_profiles"
|
||||||
assert captured["source_metadata"]["theme"] == "bathymetry"
|
assert captured["source_metadata"]["theme"] == "bathymetry"
|
||||||
assert captured["source_metadata"]["volume_supported"] is False
|
assert captured["source_metadata"]["volume_supported"] is False
|
||||||
assert captured["provenance_metadata"]["water_volume_available"] is False
|
assert captured["source_metadata"]["municipality"] == "Mol"
|
||||||
|
assert captured["source_metadata"]["regional_partitions_complete"] is False
|
||||||
payload = json.loads(captured["content"])
|
payload = json.loads(captured["content"])
|
||||||
|
assert payload["features"][0]["properties"]["municipality"] == "Mol"
|
||||||
|
assert captured["provenance_metadata"]["water_volume_available"] is False
|
||||||
assert len(payload["features"]) == 2
|
assert len(payload["features"]) == 2
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,295 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import ssl
|
||||||
|
import sys
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from urllib.error import URLError
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.core.config import Settings
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.db.session import get_db
|
||||||
|
from app.main import app
|
||||||
|
from app.models import Area, Dataset, DatasetVersion, Project
|
||||||
|
from app.schemas.bathymetry import BathymetryPartitionFinalizeRequest
|
||||||
|
from app.services.bathymetry_profile_acquisition_service import BathymetryProfileAcquisitionService
|
||||||
|
from app.services.mdk_bathymetry_probe_service import MdkBathymetryProbeService
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
SCRIPTS = ROOT / "scripts"
|
||||||
|
if str(SCRIPTS) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SCRIPTS))
|
||||||
|
|
||||||
|
import provision_flanders_geographic_scope as flanders_scope # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
class BinaryResponse:
|
||||||
|
def __init__(self, content: bytes, *, content_type: str = "application/xml"):
|
||||||
|
self.content = content
|
||||||
|
self.headers = {"Content-Type": content_type}
|
||||||
|
|
||||||
|
def __enter__(self):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def __exit__(self, *_args):
|
||||||
|
return False
|
||||||
|
|
||||||
|
def read(self, size=-1):
|
||||||
|
return self.content if size < 0 else self.content[:size]
|
||||||
|
|
||||||
|
|
||||||
|
class JsonResponse:
|
||||||
|
def __init__(self, payload, *, url="https://geo.api.vlaanderen.be/VRBG/items"):
|
||||||
|
self.payload = payload
|
||||||
|
self.url = url
|
||||||
|
|
||||||
|
def raise_for_status(self):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def json(self):
|
||||||
|
return self.payload
|
||||||
|
|
||||||
|
|
||||||
|
class SourceSession:
|
||||||
|
def __init__(self, payload):
|
||||||
|
self.payload = payload
|
||||||
|
|
||||||
|
def get(self, *_args, **_kwargs):
|
||||||
|
return JsonResponse(self.payload)
|
||||||
|
|
||||||
|
|
||||||
|
class VersionQuery:
|
||||||
|
def __init__(self, versions):
|
||||||
|
self.versions = versions
|
||||||
|
|
||||||
|
def filter(self, *_args):
|
||||||
|
return self
|
||||||
|
|
||||||
|
def all(self):
|
||||||
|
return self.versions
|
||||||
|
|
||||||
|
|
||||||
|
class FinalizeSession:
|
||||||
|
def __init__(self, rows, versions=None):
|
||||||
|
self.rows = rows
|
||||||
|
self.versions = versions or []
|
||||||
|
self.commit_count = 0
|
||||||
|
|
||||||
|
def get(self, model, row_id):
|
||||||
|
return self.rows.get((model, row_id))
|
||||||
|
|
||||||
|
def query(self, model):
|
||||||
|
assert model is DatasetVersion
|
||||||
|
return VersionQuery(self.versions)
|
||||||
|
|
||||||
|
def commit(self):
|
||||||
|
self.commit_count += 1
|
||||||
|
|
||||||
|
|
||||||
|
def capabilities_xml() -> bytes:
|
||||||
|
return b"""<?xml version="1.0"?>
|
||||||
|
<WCS_Capabilities version="1.0.0" xmlns="http://www.opengis.net/wcs">
|
||||||
|
<ContentMetadata>
|
||||||
|
<CoverageOfferingBrief>
|
||||||
|
<name>EL.GridCoverage</name>
|
||||||
|
<label>Belgian Continental Shelf bathymetry</label>
|
||||||
|
<lonLatEnvelope srsName="urn:ogc:def:crs:OGC:1.3:CRS84" />
|
||||||
|
</CoverageOfferingBrief>
|
||||||
|
</ContentMetadata>
|
||||||
|
<Capability><Request><GetCoverage><resultFormat><formats>GeoTIFF</formats></resultFormat></GetCoverage></Request></Capability>
|
||||||
|
</WCS_Capabilities>"""
|
||||||
|
|
||||||
|
|
||||||
|
def test_mdk_probe_parses_capabilities_without_enabling_acquisition() -> None:
|
||||||
|
seen = {}
|
||||||
|
|
||||||
|
def opener(request, timeout):
|
||||||
|
seen["url"] = request.full_url
|
||||||
|
seen["timeout"] = timeout
|
||||||
|
return BinaryResponse(capabilities_xml())
|
||||||
|
|
||||||
|
result = MdkBathymetryProbeService.probe(
|
||||||
|
settings=Settings(_env_file=None),
|
||||||
|
opener=opener,
|
||||||
|
checked_at=datetime(2026, 7, 17, tzinfo=UTC),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["status"] == "reachable"
|
||||||
|
assert result["tls_verified"] is True
|
||||||
|
assert result["capabilities_reachable"] is True
|
||||||
|
assert result["acquisition_supported"] is False
|
||||||
|
assert result["coverage_identifiers"] == ["EL.GridCoverage"]
|
||||||
|
assert result["advertised_formats"] == ["GeoTIFF"]
|
||||||
|
assert result["response_sha256"]
|
||||||
|
assert "request=GetCapabilities" in seen["url"]
|
||||||
|
assert seen["timeout"] == 20
|
||||||
|
|
||||||
|
|
||||||
|
def test_mdk_probe_reports_tls_failure_and_never_uses_insecure_fallback() -> None:
|
||||||
|
calls = 0
|
||||||
|
|
||||||
|
def opener(_request, timeout):
|
||||||
|
nonlocal calls
|
||||||
|
assert timeout == 20
|
||||||
|
calls += 1
|
||||||
|
raise URLError(ssl.SSLCertVerificationError("hostname mismatch"))
|
||||||
|
|
||||||
|
result = MdkBathymetryProbeService.probe(
|
||||||
|
settings=Settings(_env_file=None),
|
||||||
|
opener=opener,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert calls == 1
|
||||||
|
assert result["status"] == "tls_error"
|
||||||
|
assert result["tls_verified"] is False
|
||||||
|
assert result["capabilities_reachable"] is False
|
||||||
|
assert "insecure fallback is prohibited" in result["message"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_mdk_readiness_api_uses_canonical_envelope(monkeypatch) -> None:
|
||||||
|
project_id = uuid4()
|
||||||
|
db = SimpleNamespace(get=lambda model, row_id: Project(id=project_id, name="Mol") if model is Project else None)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
MdkBathymetryProbeService,
|
||||||
|
"probe",
|
||||||
|
lambda: {
|
||||||
|
"source_key": "mdk_bcp_bathymetry",
|
||||||
|
"status": "tls_error",
|
||||||
|
"acquisition_supported": False,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
app.dependency_overrides[get_db] = lambda: db
|
||||||
|
try:
|
||||||
|
response = TestClient(app).get(
|
||||||
|
f"/api/v1/projects/{project_id}/datasets/bathymetry/sources/mdk_bcp_bathymetry/readiness"
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
app.dependency_overrides.clear()
|
||||||
|
|
||||||
|
assert response.status_code == 200
|
||||||
|
assert set(response.json()) == {"data"}
|
||||||
|
assert response.json()["data"]["status"] == "tls_error"
|
||||||
|
assert response.json()["data"]["acquisition_supported"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_partition_finalization_requires_complete_area_accounting_and_updates_versions() -> None:
|
||||||
|
project_id = uuid4()
|
||||||
|
area_ids = [uuid4(), uuid4()]
|
||||||
|
dataset_id = uuid4()
|
||||||
|
project = Project(id=project_id, name="Flanders")
|
||||||
|
areas = [
|
||||||
|
Area(id=area_ids[0], project_id=project_id, name="Gemeente Mol - officiële grens"),
|
||||||
|
Area(id=area_ids[1], project_id=project_id, name="Gemeente Geel - officiële grens"),
|
||||||
|
]
|
||||||
|
dataset = Dataset(
|
||||||
|
id=dataset_id,
|
||||||
|
project_id=project_id,
|
||||||
|
area_id=area_ids[0],
|
||||||
|
name="vha.geojson",
|
||||||
|
dataset_type="vector",
|
||||||
|
source="VHA",
|
||||||
|
source_name=BathymetryProfileAcquisitionService.PROVIDER,
|
||||||
|
source_metadata={
|
||||||
|
"profile_count": 3,
|
||||||
|
"document_count": 2,
|
||||||
|
"structured_depth_count": 1,
|
||||||
|
"measurement_date_min": "1990-01-01",
|
||||||
|
"measurement_date_max": "2020-01-01",
|
||||||
|
},
|
||||||
|
provenance_metadata={},
|
||||||
|
status="ready",
|
||||||
|
)
|
||||||
|
version = DatasetVersion(id=uuid4(), dataset_id=dataset_id, version=1)
|
||||||
|
rows = {
|
||||||
|
(Project, project_id): project,
|
||||||
|
(Area, area_ids[0]): areas[0],
|
||||||
|
(Area, area_ids[1]): areas[1],
|
||||||
|
(Dataset, dataset_id): dataset,
|
||||||
|
}
|
||||||
|
db = FinalizeSession(rows, [version])
|
||||||
|
payload = BathymetryPartitionFinalizeRequest(
|
||||||
|
partition_scope_key="flanders",
|
||||||
|
expected_area_ids=area_ids,
|
||||||
|
dataset_ids=[dataset_id],
|
||||||
|
no_profile_area_ids=[area_ids[1]],
|
||||||
|
manifest_sha256="a" * 64,
|
||||||
|
observed_at=datetime(2026, 7, 17, tzinfo=UTC),
|
||||||
|
)
|
||||||
|
|
||||||
|
result = BathymetryProfileAcquisitionService.finalize_partitions(db, project_id, payload)
|
||||||
|
|
||||||
|
assert result["regional_partitions_complete"] is True
|
||||||
|
assert result["partition_count"] == 2
|
||||||
|
assert result["data_partition_count"] == 1
|
||||||
|
assert result["no_profile_partition_count"] == 1
|
||||||
|
assert result["profile_count"] == 3
|
||||||
|
assert dataset.source_metadata["coverage_scope"] == "flanders"
|
||||||
|
assert dataset.source_metadata["municipality"] == "Mol"
|
||||||
|
assert dataset.source_metadata["partitioned_source_audit"] is True
|
||||||
|
assert version.source_metadata == dataset.source_metadata
|
||||||
|
assert version.provenance_metadata == dataset.provenance_metadata
|
||||||
|
assert db.commit_count == 1
|
||||||
|
|
||||||
|
incomplete = payload.model_copy(update={"no_profile_area_ids": []})
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
BathymetryProfileAcquisitionService.finalize_partitions(
|
||||||
|
FinalizeSession(rows, [version]),
|
||||||
|
project_id,
|
||||||
|
incomplete,
|
||||||
|
)
|
||||||
|
assert exc_info.value.code == "BATHYMETRY_PARTITION_MANIFEST_INCOMPLETE"
|
||||||
|
|
||||||
|
|
||||||
|
def test_flanders_scope_discovery_uses_complete_unique_vrbg_inventory() -> None:
|
||||||
|
features = [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"id": f"Refgem.{index:05d}",
|
||||||
|
"properties": {"NISCODE": f"{index:05d}", "NAAM": f"Gemeente {index:03d}"},
|
||||||
|
"geometry": {
|
||||||
|
"type": "Polygon",
|
||||||
|
"coordinates": [[[4.0, 50.5], [4.1, 50.5], [4.1, 50.6], [4.0, 50.5]]],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for index in range(1, 286)
|
||||||
|
]
|
||||||
|
scope, selected, source_url = flanders_scope.discover_flanders_scope(
|
||||||
|
SourceSession({"features": list(reversed(features))}),
|
||||||
|
timeout=30,
|
||||||
|
min_municipalities=270,
|
||||||
|
max_municipalities=300,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert scope.key == "flanders"
|
||||||
|
assert scope.project_name == "Flanders Regional Workbench"
|
||||||
|
assert len(scope.members) == 285
|
||||||
|
assert len(set(scope.nis_codes)) == 285
|
||||||
|
assert [item["properties"]["NISCODE"] for item in selected] == sorted(scope.nis_codes)
|
||||||
|
assert source_url.startswith("https://geo.api.vlaanderen.be/")
|
||||||
|
|
||||||
|
|
||||||
|
def test_expansion_scripts_are_packaged_and_readiness_checked() -> None:
|
||||||
|
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||||
|
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||||
|
for script in (
|
||||||
|
"provision_flanders_geographic_scope.py",
|
||||||
|
"provision_flanders_bathymetry_profiles.py",
|
||||||
|
"probe_mdk_bathymetry.py",
|
||||||
|
):
|
||||||
|
assert f"COPY scripts/{script}" in dockerfile
|
||||||
|
assert f"py_compile scripts/{script}" in readiness
|
||||||
|
|
||||||
|
sources = {
|
||||||
|
item["key"]: item
|
||||||
|
for item in BathymetryProfileAcquisitionService.list_sources()
|
||||||
|
}
|
||||||
|
assert sources["mdk_bcp_bathymetry"]["integration_status"] == "probe_only"
|
||||||
|
assert sources["mdk_bcp_bathymetry"]["acquisition_supported"] is False
|
||||||
|
assert "EL_wcs" in sources["mdk_bcp_bathymetry"]["service_url"]
|
||||||
@@ -89,6 +89,9 @@ COPY scripts/provision_regional_historical_landuse.py /app/scripts/provision_reg
|
|||||||
COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_official_landuse_timeseries.py
|
COPY scripts/provision_official_landuse_timeseries.py /app/scripts/provision_official_landuse_timeseries.py
|
||||||
COPY scripts/provision_waterinfo_station_history.py /app/scripts/provision_waterinfo_station_history.py
|
COPY scripts/provision_waterinfo_station_history.py /app/scripts/provision_waterinfo_station_history.py
|
||||||
COPY scripts/provision_mol_bathymetry_profiles.py /app/scripts/provision_mol_bathymetry_profiles.py
|
COPY scripts/provision_mol_bathymetry_profiles.py /app/scripts/provision_mol_bathymetry_profiles.py
|
||||||
|
COPY scripts/provision_flanders_geographic_scope.py /app/scripts/provision_flanders_geographic_scope.py
|
||||||
|
COPY scripts/provision_flanders_bathymetry_profiles.py /app/scripts/provision_flanders_bathymetry_profiles.py
|
||||||
|
COPY scripts/probe_mdk_bathymetry.py /app/scripts/probe_mdk_bathymetry.py
|
||||||
COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natura2000.py
|
COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natura2000.py
|
||||||
COPY scripts/provision_regional_bwk_natura2000.py /app/scripts/provision_regional_bwk_natura2000.py
|
COPY scripts/provision_regional_bwk_natura2000.py /app/scripts/provision_regional_bwk_natura2000.py
|
||||||
COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py
|
COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py
|
||||||
|
|||||||
@@ -51,6 +51,11 @@
|
|||||||
<Config Name="VMM Flood Hazard WCS URL" Target="FLOOD_HAZARD_WCS_URL" Default="https://geoservice.waterinfo.be/OGRK/wcs" Mode="" Description="Official VMM OGRK WCS endpoint for governed flood-depth scenarios." Type="Variable" Display="advanced" Required="true" Mask="false">https://geoservice.waterinfo.be/OGRK/wcs</Config>
|
<Config Name="VMM Flood Hazard WCS URL" Target="FLOOD_HAZARD_WCS_URL" Default="https://geoservice.waterinfo.be/OGRK/wcs" Mode="" Description="Official VMM OGRK WCS endpoint for governed flood-depth scenarios." Type="Variable" Display="advanced" Required="true" Mask="false">https://geoservice.waterinfo.be/OGRK/wcs</Config>
|
||||||
<Config Name="Flood Hazard Analysis Resolution (m)" Target="FLOOD_HAZARD_RESOLUTION_M" Default="5.0" Mode="" Description="Stored analysis grid resolution; official source values are converted from centimetres to metres." Type="Variable" Display="advanced" Required="true" Mask="false">5.0</Config>
|
<Config Name="Flood Hazard Analysis Resolution (m)" Target="FLOOD_HAZARD_RESOLUTION_M" Default="5.0" Mode="" Description="Stored analysis grid resolution; official source values are converted from centimetres to metres." Type="Variable" Display="advanced" Required="true" Mask="false">5.0</Config>
|
||||||
<Config Name="Flood Hazard Maximum Cells" Target="FLOOD_HAZARD_MAX_PIXELS" Default="12000000" Mode="" Description="Maximum raster cells per flood-hazard acquisition or selection analysis." Type="Variable" Display="advanced" Required="true" Mask="false">12000000</Config>
|
<Config Name="Flood Hazard Maximum Cells" Target="FLOOD_HAZARD_MAX_PIXELS" Default="12000000" Mode="" Description="Maximum raster cells per flood-hazard acquisition or selection analysis." Type="Variable" Display="advanced" Required="true" Mask="false">12000000</Config>
|
||||||
|
<Config Name="VHA Bathymetry Profiles" Target="BATHYMETRY_PROFILES_ENABLED" Default="true" Mode="" Description="Allow explicit bounded acquisition of official VHA cross-section profile points." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
|
||||||
|
<Config Name="VHA Profile Maximum Features" Target="BATHYMETRY_PROFILES_MAX_FEATURES" Default="50000" Mode="" Description="Safety limit per municipality or bounded VHA profile request." Type="Variable" Display="advanced" Required="true" Mask="false">50000</Config>
|
||||||
|
<Config Name="MDK Bathymetry Readiness Probe" Target="MDK_BATHYMETRY_PROBE_ENABLED" Default="true" Mode="" Description="Enable strict-TLS read-only GetCapabilities checks. This never downloads or activates a North Sea raster." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
|
||||||
|
<Config Name="MDK Bathymetry WCS URL" Target="MDK_BATHYMETRY_WCS_URL" Default="https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs" Mode="" Description="Official metadata WCS endpoint. TLS verification is mandatory and cannot be bypassed." Type="Variable" Display="advanced" Required="true" Mask="false">https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs</Config>
|
||||||
|
<Config Name="MDK Probe Timeout Seconds" Target="MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS" Default="20" Mode="" Description="Maximum wait for one read-only MDK GetCapabilities request." Type="Variable" Display="advanced" Required="true" Mask="false">20</Config>
|
||||||
<Config Name="Official Thematic Raster Acquisition" Target="THEMATIC_RASTER_ENABLED" Default="true" Mode="" Description="Allow bounded official Departement Omgeving rasters for space, population, accessibility and services." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
|
<Config Name="Official Thematic Raster Acquisition" Target="THEMATIC_RASTER_ENABLED" Default="true" Mode="" Description="Allow bounded official Departement Omgeving rasters for space, population, accessibility and services." Type="Variable" Display="advanced" Required="true" Mask="false">true</Config>
|
||||||
<Config Name="Thematic Raster WCS URL" Target="THEMATIC_RASTER_WCS_URL" Default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs" Mode="" Description="Official public MercatorNet WCS endpoint. Product identifiers remain server allowlisted." Type="Variable" Display="advanced" Required="true" Mask="false">https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs</Config>
|
<Config Name="Thematic Raster WCS URL" Target="THEMATIC_RASTER_WCS_URL" Default="https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs" Mode="" Description="Official public MercatorNet WCS endpoint. Product identifiers remain server allowlisted." Type="Variable" Display="advanced" Required="true" Mask="false">https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs</Config>
|
||||||
<Config Name="Thematic Raster Maximum Side (m)" Target="THEMATIC_RASTER_MAX_SIDE_M" Default="60000" Mode="" Description="Maximum side length for one allowlisted thematic raster scope; supports the complete Kempen work area while WCS transfers remain tiled." Type="Variable" Display="advanced" Required="true" Mask="false">60000</Config>
|
<Config Name="Thematic Raster Maximum Side (m)" Target="THEMATIC_RASTER_MAX_SIDE_M" Default="60000" Mode="" Description="Maximum side length for one allowlisted thematic raster scope; supports the complete Kempen work area while WCS transfers remain tiled." Type="Variable" Display="advanced" Required="true" Mask="false">60000</Config>
|
||||||
|
|||||||
@@ -59,6 +59,17 @@ FLOOD_HAZARD_MAX_SIDE_M=20000
|
|||||||
FLOOD_HAZARD_MAX_PIXELS=12000000
|
FLOOD_HAZARD_MAX_PIXELS=12000000
|
||||||
FLOOD_HAZARD_TIMEOUT_SECONDS=300
|
FLOOD_HAZARD_TIMEOUT_SECONDS=300
|
||||||
FLOOD_HAZARD_MAX_RESPONSE_MB=160
|
FLOOD_HAZARD_MAX_RESPONSE_MB=160
|
||||||
|
BATHYMETRY_PROFILES_ENABLED=true
|
||||||
|
BATHYMETRY_PROFILES_LAYER_URL=https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0
|
||||||
|
BATHYMETRY_WATERCOURSE_LAYER_URL=https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1
|
||||||
|
BATHYMETRY_PROFILES_PAGE_SIZE=1000
|
||||||
|
BATHYMETRY_PROFILES_MAX_FEATURES=50000
|
||||||
|
BATHYMETRY_PROFILES_TIMEOUT_SECONDS=120
|
||||||
|
BATHYMETRY_PROFILES_MAX_RESPONSE_MB=32
|
||||||
|
MDK_BATHYMETRY_PROBE_ENABLED=true
|
||||||
|
MDK_BATHYMETRY_WCS_URL=https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs
|
||||||
|
MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS=20
|
||||||
|
MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB=4
|
||||||
|
|
||||||
# Allowlisted Departement Omgeving policy rasters. Regional requests are
|
# Allowlisted Departement Omgeving policy rasters. Regional requests are
|
||||||
# transferred as fixed 10 km WCS tiles before exact Area clipping.
|
# transferred as fixed 10 km WCS tiles before exact Area clipping.
|
||||||
|
|||||||
@@ -51,6 +51,17 @@ FLOOD_HAZARD_MAX_SIDE_M="${FLOOD_HAZARD_MAX_SIDE_M:-20000}"
|
|||||||
FLOOD_HAZARD_MAX_PIXELS="${FLOOD_HAZARD_MAX_PIXELS:-12000000}"
|
FLOOD_HAZARD_MAX_PIXELS="${FLOOD_HAZARD_MAX_PIXELS:-12000000}"
|
||||||
FLOOD_HAZARD_TIMEOUT_SECONDS="${FLOOD_HAZARD_TIMEOUT_SECONDS:-300}"
|
FLOOD_HAZARD_TIMEOUT_SECONDS="${FLOOD_HAZARD_TIMEOUT_SECONDS:-300}"
|
||||||
FLOOD_HAZARD_MAX_RESPONSE_MB="${FLOOD_HAZARD_MAX_RESPONSE_MB:-160}"
|
FLOOD_HAZARD_MAX_RESPONSE_MB="${FLOOD_HAZARD_MAX_RESPONSE_MB:-160}"
|
||||||
|
BATHYMETRY_PROFILES_ENABLED="${BATHYMETRY_PROFILES_ENABLED:-true}"
|
||||||
|
BATHYMETRY_PROFILES_LAYER_URL="${BATHYMETRY_PROFILES_LAYER_URL:-https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/0}"
|
||||||
|
BATHYMETRY_WATERCOURSE_LAYER_URL="${BATHYMETRY_WATERCOURSE_LAYER_URL:-https://vha.waterinfo.be/arcgis/rest/services/digitale_atlas/MapServer/1}"
|
||||||
|
BATHYMETRY_PROFILES_PAGE_SIZE="${BATHYMETRY_PROFILES_PAGE_SIZE:-1000}"
|
||||||
|
BATHYMETRY_PROFILES_MAX_FEATURES="${BATHYMETRY_PROFILES_MAX_FEATURES:-50000}"
|
||||||
|
BATHYMETRY_PROFILES_TIMEOUT_SECONDS="${BATHYMETRY_PROFILES_TIMEOUT_SECONDS:-120}"
|
||||||
|
BATHYMETRY_PROFILES_MAX_RESPONSE_MB="${BATHYMETRY_PROFILES_MAX_RESPONSE_MB:-32}"
|
||||||
|
MDK_BATHYMETRY_PROBE_ENABLED="${MDK_BATHYMETRY_PROBE_ENABLED:-true}"
|
||||||
|
MDK_BATHYMETRY_WCS_URL="${MDK_BATHYMETRY_WCS_URL:-https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs}"
|
||||||
|
MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS="${MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS:-20}"
|
||||||
|
MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB="${MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB:-4}"
|
||||||
THEMATIC_RASTER_ENABLED="${THEMATIC_RASTER_ENABLED:-true}"
|
THEMATIC_RASTER_ENABLED="${THEMATIC_RASTER_ENABLED:-true}"
|
||||||
THEMATIC_RASTER_WCS_URL="${THEMATIC_RASTER_WCS_URL:-https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs}"
|
THEMATIC_RASTER_WCS_URL="${THEMATIC_RASTER_WCS_URL:-https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs}"
|
||||||
THEMATIC_RASTER_MIN_SIDE_M="${THEMATIC_RASTER_MIN_SIDE_M:-100}"
|
THEMATIC_RASTER_MIN_SIDE_M="${THEMATIC_RASTER_MIN_SIDE_M:-100}"
|
||||||
@@ -158,6 +169,17 @@ docker run -d \
|
|||||||
-e FLOOD_HAZARD_MAX_PIXELS="$FLOOD_HAZARD_MAX_PIXELS" \
|
-e FLOOD_HAZARD_MAX_PIXELS="$FLOOD_HAZARD_MAX_PIXELS" \
|
||||||
-e FLOOD_HAZARD_TIMEOUT_SECONDS="$FLOOD_HAZARD_TIMEOUT_SECONDS" \
|
-e FLOOD_HAZARD_TIMEOUT_SECONDS="$FLOOD_HAZARD_TIMEOUT_SECONDS" \
|
||||||
-e FLOOD_HAZARD_MAX_RESPONSE_MB="$FLOOD_HAZARD_MAX_RESPONSE_MB" \
|
-e FLOOD_HAZARD_MAX_RESPONSE_MB="$FLOOD_HAZARD_MAX_RESPONSE_MB" \
|
||||||
|
-e BATHYMETRY_PROFILES_ENABLED="$BATHYMETRY_PROFILES_ENABLED" \
|
||||||
|
-e BATHYMETRY_PROFILES_LAYER_URL="$BATHYMETRY_PROFILES_LAYER_URL" \
|
||||||
|
-e BATHYMETRY_WATERCOURSE_LAYER_URL="$BATHYMETRY_WATERCOURSE_LAYER_URL" \
|
||||||
|
-e BATHYMETRY_PROFILES_PAGE_SIZE="$BATHYMETRY_PROFILES_PAGE_SIZE" \
|
||||||
|
-e BATHYMETRY_PROFILES_MAX_FEATURES="$BATHYMETRY_PROFILES_MAX_FEATURES" \
|
||||||
|
-e BATHYMETRY_PROFILES_TIMEOUT_SECONDS="$BATHYMETRY_PROFILES_TIMEOUT_SECONDS" \
|
||||||
|
-e BATHYMETRY_PROFILES_MAX_RESPONSE_MB="$BATHYMETRY_PROFILES_MAX_RESPONSE_MB" \
|
||||||
|
-e MDK_BATHYMETRY_PROBE_ENABLED="$MDK_BATHYMETRY_PROBE_ENABLED" \
|
||||||
|
-e MDK_BATHYMETRY_WCS_URL="$MDK_BATHYMETRY_WCS_URL" \
|
||||||
|
-e MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS="$MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS" \
|
||||||
|
-e MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB="$MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB" \
|
||||||
-e THEMATIC_RASTER_ENABLED="$THEMATIC_RASTER_ENABLED" \
|
-e THEMATIC_RASTER_ENABLED="$THEMATIC_RASTER_ENABLED" \
|
||||||
-e THEMATIC_RASTER_WCS_URL="$THEMATIC_RASTER_WCS_URL" \
|
-e THEMATIC_RASTER_WCS_URL="$THEMATIC_RASTER_WCS_URL" \
|
||||||
-e THEMATIC_RASTER_MIN_SIDE_M="$THEMATIC_RASTER_MIN_SIDE_M" \
|
-e THEMATIC_RASTER_MIN_SIDE_M="$THEMATIC_RASTER_MIN_SIDE_M" \
|
||||||
|
|||||||
@@ -61,6 +61,10 @@ services:
|
|||||||
BATHYMETRY_PROFILES_MAX_FEATURES: ${BATHYMETRY_PROFILES_MAX_FEATURES:-50000}
|
BATHYMETRY_PROFILES_MAX_FEATURES: ${BATHYMETRY_PROFILES_MAX_FEATURES:-50000}
|
||||||
BATHYMETRY_PROFILES_TIMEOUT_SECONDS: ${BATHYMETRY_PROFILES_TIMEOUT_SECONDS:-120}
|
BATHYMETRY_PROFILES_TIMEOUT_SECONDS: ${BATHYMETRY_PROFILES_TIMEOUT_SECONDS:-120}
|
||||||
BATHYMETRY_PROFILES_MAX_RESPONSE_MB: ${BATHYMETRY_PROFILES_MAX_RESPONSE_MB:-32}
|
BATHYMETRY_PROFILES_MAX_RESPONSE_MB: ${BATHYMETRY_PROFILES_MAX_RESPONSE_MB:-32}
|
||||||
|
MDK_BATHYMETRY_PROBE_ENABLED: ${MDK_BATHYMETRY_PROBE_ENABLED:-true}
|
||||||
|
MDK_BATHYMETRY_WCS_URL: ${MDK_BATHYMETRY_WCS_URL:-https://bathy.agentschapmdk.be/spatialfusionserver/services/ows/wcs/EL_wcs}
|
||||||
|
MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS: ${MDK_BATHYMETRY_PROBE_TIMEOUT_SECONDS:-20}
|
||||||
|
MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB: ${MDK_BATHYMETRY_PROBE_MAX_RESPONSE_MB:-4}
|
||||||
THEMATIC_RASTER_ENABLED: ${THEMATIC_RASTER_ENABLED:-true}
|
THEMATIC_RASTER_ENABLED: ${THEMATIC_RASTER_ENABLED:-true}
|
||||||
THEMATIC_RASTER_WCS_URL: ${THEMATIC_RASTER_WCS_URL:-https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs}
|
THEMATIC_RASTER_WCS_URL: ${THEMATIC_RASTER_WCS_URL:-https://www.mercator.vlaanderen.be/raadpleegdienstenmercatorpubliek/wcs}
|
||||||
THEMATIC_RASTER_MIN_SIDE_M: ${THEMATIC_RASTER_MIN_SIDE_M:-100}
|
THEMATIC_RASTER_MIN_SIDE_M: ${THEMATIC_RASTER_MIN_SIDE_M:-100}
|
||||||
|
|||||||
+29
-4
@@ -2061,10 +2061,20 @@ sets an explicit Ollama context window and returns
|
|||||||
### GET `/api/v1/projects/{project_id}/datasets/bathymetry/sources`
|
### GET `/api/v1/projects/{project_id}/datasets/bathymetry/sources`
|
||||||
|
|
||||||
Returns the governed bathymetry source registry in the canonical envelope.
|
Returns the governed bathymetry source registry in the canonical envelope.
|
||||||
VHA inland profiles are `operational`. MDK Belgian Continental Shelf and SPW
|
VHA inland profiles are `operational`. MDK Belgian Continental Shelf is
|
||||||
Walloon bathymetry remain `available_not_integrated`; they cannot be acquired
|
`probe_only`; SPW Walloon bathymetry remains `available_not_integrated`.
|
||||||
through this contract until their raster/download and vertical-datum flows
|
Neither source can be acquired until its raster/download and vertical-datum
|
||||||
pass live validation.
|
flow passes live validation.
|
||||||
|
|
||||||
|
### GET `/api/v1/projects/{project_id}/datasets/bathymetry/sources/mdk_bcp_bathymetry/readiness`
|
||||||
|
|
||||||
|
Runs one bounded, read-only WCS 1.0.0 `GetCapabilities` request with mandatory
|
||||||
|
system TLS verification and a configured response-size limit. Status is one
|
||||||
|
of `disabled`, `invalid_configuration`, `tls_error`,
|
||||||
|
`endpoint_unavailable`, `invalid_capabilities` or `reachable`. A reachable
|
||||||
|
response lists coverage identifiers, advertised formats and CRS values, but
|
||||||
|
always returns `acquisition_supported=false`. There is no insecure TLS
|
||||||
|
fallback and no `GetCoverage` request.
|
||||||
|
|
||||||
### POST `/api/v1/projects/{project_id}/datasets/bathymetry/profiles/acquire`
|
### POST `/api/v1/projects/{project_id}/datasets/bathymetry/profiles/acquire`
|
||||||
|
|
||||||
@@ -2089,3 +2099,18 @@ The acquired Dataset uses
|
|||||||
`reference_layer_name=bathymetry_profiles`. Existing vector content and
|
`reference_layer_name=bathymetry_profiles`. Existing vector content and
|
||||||
selection endpoints provide GeoJSON and metrics. The contract does not expose
|
selection endpoints provide GeoJSON and metrics. The contract does not expose
|
||||||
a continuous bathymetric surface, current water depth or water volume.
|
a continuous bathymetric surface, current water depth or water volume.
|
||||||
|
|
||||||
|
### POST `/api/v1/projects/{project_id}/datasets/bathymetry/profiles/partitions/finalize`
|
||||||
|
|
||||||
|
Finalizes a complete municipality-partition manifest. The request supplies a
|
||||||
|
scope key, every expected municipality Area id, one ready VHA Dataset id for
|
||||||
|
each non-empty partition, explicit Area ids with zero source profiles, a
|
||||||
|
SHA-256 manifest identity and observation time. The backend verifies project,
|
||||||
|
Area, provider, Dataset and one-Dataset-per-Area consistency. Only a complete
|
||||||
|
accounting sets `partitioned_source_audit=true` and
|
||||||
|
`regional_partitions_complete=true`; partial operator runs remain hidden at
|
||||||
|
regional scope.
|
||||||
|
|
||||||
|
Future provider output continues to use DatasetService and, for vectors,
|
||||||
|
VectorFeatureService. Arbitrary service URLs, browser-side fetches, insecure
|
||||||
|
TLS bypasses and startup downloads remain forbidden.
|
||||||
|
|||||||
@@ -120,12 +120,21 @@ bed evolution.
|
|||||||
|
|
||||||
## Implementation order
|
## Implementation order
|
||||||
|
|
||||||
1. Operate and validate the Mol VHA profile Dataset and map flow.
|
1. Operate and validate the Mol VHA profile Dataset and map flow. **Done.**
|
||||||
2. Add VHA municipal partition orchestration for Flanders.
|
2. Add VHA municipal partition orchestration for Flanders. **Implemented;
|
||||||
|
complete live provisioning remains an explicit operator run.**
|
||||||
3. Implement a bounded MDK WCS probe, then acquisition behind live evidence.
|
3. Implement a bounded MDK WCS probe, then acquisition behind live evidence.
|
||||||
|
**Probe implemented. Acquisition blocked because the live endpoint fails
|
||||||
|
strict hostname validation and does not expose usable capabilities.**
|
||||||
4. Implement SPW download staging and vertical-datum metadata validation.
|
4. Implement SPW download staging and vertical-datum metadata validation.
|
||||||
5. Add maritime boundaries as separate authoritative scope layers.
|
5. Add maritime boundaries as separate authoritative scope layers.
|
||||||
6. Add cross-source vertical-datum transformation only with authoritative
|
6. Add cross-source vertical-datum transformation only with authoritative
|
||||||
grids/parameters and uncertainty tests.
|
grids/parameters and uncertainty tests.
|
||||||
7. Add volume only after a compatible measured or modeled water-surface source
|
7. Add volume only after a compatible measured or modeled water-surface source
|
||||||
is part of the same analysis contract.
|
is part of the same analysis contract.
|
||||||
|
|
||||||
|
The Flanders scope is land-only and is dynamically derived from the complete
|
||||||
|
current VRBG municipality collection (285 members at the 2026-07-17
|
||||||
|
validation). Belgium is not represented by expanding that polygon. Wallonia,
|
||||||
|
Brussels, the territorial sea and the Belgian EEZ/continental shelf require
|
||||||
|
their own authoritative adapters and legal scope labels.
|
||||||
|
|||||||
@@ -10054,3 +10054,41 @@ Validation:
|
|||||||
historical survey range `1877-2020` and loaded all 828 persisted objects.
|
historical survey range `1877-2020` and loaded all 828 persisted objects.
|
||||||
The browser console contained no warnings or errors and the 1265x720
|
The browser console contained no warnings or errors and the 1265x720
|
||||||
viewport had no horizontal document overflow.
|
viewport had no horizontal document overflow.
|
||||||
|
|
||||||
|
## Sprint 236 - Flemish bathymetry partitions and safe North Sea probe (2026-07-17)
|
||||||
|
|
||||||
|
Implemented:
|
||||||
|
- Added a dynamic Flanders scope operator that discovers the complete current
|
||||||
|
official VRBG RefGem municipality inventory instead of freezing a fragile
|
||||||
|
hand-maintained list. It validates unique NIS codes/names, polygonal source
|
||||||
|
features and a 270..300 safety range before creating artifacts or API state.
|
||||||
|
- Added an atomic, resumable VHA municipality coordinator. Every partition is
|
||||||
|
exact-Area clipped and persisted through the existing acquisition endpoint;
|
||||||
|
failures, explicit zero-profile results and ready Dataset ids remain in the
|
||||||
|
checksum-bound coverage manifest.
|
||||||
|
- Added server-side partition finalization. Regional activation requires exact
|
||||||
|
accounting for every expected municipality, one ready VHA Dataset per
|
||||||
|
non-empty Area and explicit no-profile Area ids. Dataset and DatasetVersion
|
||||||
|
metadata retain the manifest identity and completeness flags.
|
||||||
|
- Added a strict-TLS, response-bounded MDK WCS GetCapabilities probe and a
|
||||||
|
concise Sources-workspace control. It never performs GetCoverage, never
|
||||||
|
disables certificate verification and always leaves acquisition disabled.
|
||||||
|
- Packaged the Flanders scope, VHA coordinator and MDK probe operators in the
|
||||||
|
all-in-one image and exposed their bounded runtime settings in Compose and
|
||||||
|
the Unraid deployment configuration.
|
||||||
|
|
||||||
|
Validation:
|
||||||
|
- Official VRBG fetch-only validation returned 285 unique municipalities, an
|
||||||
|
exact union of 13,625.73 km2 and WGS84 bounds
|
||||||
|
`[2.54132923, 50.68749237, 5.9111094, 51.50511313]`.
|
||||||
|
- The live MDK metadata endpoint currently fails strict hostname validation:
|
||||||
|
`bathy.agentschapmdk.be` presents a certificate for `*.l27powered.eu`.
|
||||||
|
Diagnostic requests to both the previously configured and current metadata
|
||||||
|
paths returned HTTP 404 after an explicitly external, non-application
|
||||||
|
insecure inspection. GeoIntel itself reports `tls_error` and provides no
|
||||||
|
bypass.
|
||||||
|
- The complete readiness gate passed 920 backend tests, backend compilation,
|
||||||
|
documentation/contract audits, 115 documented API routes, Alembic head
|
||||||
|
`202607160001`, frontend TypeScript typecheck and the production Vite build.
|
||||||
|
- Live Docker deployment and controlled Flanders provisioning are recorded
|
||||||
|
after the repository commit below.
|
||||||
|
|||||||
@@ -2,6 +2,14 @@
|
|||||||
|
|
||||||
Database: PostgreSQL + PostGIS.
|
Database: PostgreSQL + PostGIS.
|
||||||
|
|
||||||
|
Bathymetry regional completeness requires no new table. Every VHA municipality
|
||||||
|
partition remains an ordinary `datasets` row plus `vector_features`. After a
|
||||||
|
complete server-validated manifest, Dataset and DatasetVersion metadata retain
|
||||||
|
`partition_scope_key`, partition counts, the manifest SHA-256,
|
||||||
|
`partitioned_source_audit=true` and
|
||||||
|
`regional_partitions_complete=true`. Empty source partitions are recorded in
|
||||||
|
the manifest provenance and never represented by fabricated features.
|
||||||
|
|
||||||
## Rules
|
## Rules
|
||||||
|
|
||||||
- Store geometries in PostGIS with explicit SRID.
|
- Store geometries in PostGIS with explicit SRID.
|
||||||
|
|||||||
+15
-1
@@ -722,7 +722,11 @@ are not filled by fabricated OCR output.
|
|||||||
The following sources are audited but not yet operational:
|
The following sources are audited but not yet operational:
|
||||||
|
|
||||||
- MDK Dieptemodel Belgisch Continentaal Plat/Noordzee: 20 x 20 m continuous
|
- MDK Dieptemodel Belgisch Continentaal Plat/Noordzee: 20 x 20 m continuous
|
||||||
raster in LAT, exposed through WCS/WMTS.
|
raster in LAT, exposed through WCS/WMTS. GeoIntel now has a strict-TLS,
|
||||||
|
read-only GetCapabilities probe. On 2026-07-17 the official metadata
|
||||||
|
endpoint presented a certificate for another hostname and returned no
|
||||||
|
usable capabilities path; status therefore remains `tls_error` and raster
|
||||||
|
acquisition is disabled.
|
||||||
- SPW bathymetry of navigable waterways and reservoir lakes: 0.5 m bed
|
- SPW bathymetry of navigable waterways and reservoir lakes: 0.5 m bed
|
||||||
elevation and XYZ data in mDNG.
|
elevation and XYZ data in mDNG.
|
||||||
- Port of Antwerp-Bruges periodic soundings: catalog candidate pending a
|
- Port of Antwerp-Bruges periodic soundings: catalog candidate pending a
|
||||||
@@ -732,3 +736,13 @@ See `docs/BATHYMETRY_EXPANSION_ROADMAP.md`. TAW, LAT and mDNG remain separate
|
|||||||
until an authoritative vertical transformation is implemented and tested.
|
until an authoritative vertical transformation is implemented and tested.
|
||||||
The territorial sea, EEZ and continental shelf are separate scope layers and
|
The territorial sea, EEZ and continental shelf are separate scope layers and
|
||||||
must be labelled according to their legal meaning.
|
must be labelled according to their legal meaning.
|
||||||
|
|
||||||
|
The official VRBG RefGem collection currently exposes 285 Flemish
|
||||||
|
municipalities. `scripts/provision_flanders_geographic_scope.py` validates the
|
||||||
|
complete inventory, produces checksum-bound land-boundary artifacts and
|
||||||
|
creates one persisted municipality Area per source feature. The separate
|
||||||
|
`scripts/provision_flanders_bathymetry_profiles.py` coordinator acquires VHA
|
||||||
|
profiles per Area, writes an atomic resumable manifest and requests regional
|
||||||
|
activation only after all 285 partitions are accounted for. A municipality
|
||||||
|
with zero source points is retained as an explicit no-profile partition, not
|
||||||
|
silently omitted.
|
||||||
|
|||||||
@@ -222,7 +222,30 @@ Raster / gemodelleerd overstromingsgevaar.
|
|||||||
|
|
||||||
### Prioriteit
|
### Prioriteit
|
||||||
|
|
||||||
P5 overstromingsscenario's operationeel voor Mol; bathymetrie blijft open.
|
P5 overstromingsscenario's operationeel. VHA-dwarsprofielen zijn als
|
||||||
|
historische puntmetingen operationeel voor Mol en partitioneerbaar voor alle
|
||||||
|
officiële Vlaamse gemeente-Areas. Ze blijven strikt gescheiden van dit
|
||||||
|
scenario-raster en leveren geen gebiedsdekkende bathymetrie of volume.
|
||||||
|
|
||||||
|
## Bathymetry partition metadata
|
||||||
|
|
||||||
|
Een VHA-profiel-Dataset behoudt `partition_area_id`,
|
||||||
|
`partition_area_name`, `municipality`, meetdatumbereik en exacte aantallen.
|
||||||
|
Regionale Vlaamse dekking is alleen geldig wanneer de server een volledig
|
||||||
|
manifest voor de actuele officiële gemeente-inventaris heeft gevalideerd.
|
||||||
|
Daarna bevatten Dataset en DatasetVersion:
|
||||||
|
|
||||||
|
- `partition_scope_key`
|
||||||
|
- `partition_count`, `data_partition_count` en `no_profile_partition_count`
|
||||||
|
- `partition_manifest_sha256`
|
||||||
|
- `partitioned_source_audit=true`
|
||||||
|
- `regional_partitions_complete=true`
|
||||||
|
|
||||||
|
Een no-profile partition is bronbewijs dat de begrensde VHA-query nul punten
|
||||||
|
opleverde. Er worden nooit lege of kunstmatige profielobjecten aangemaakt.
|
||||||
|
|
||||||
|
MDK Noordzee blijft `probe_only`: WCS-capabilities kunnen veilig worden
|
||||||
|
gecontroleerd, maar er bestaat nog geen toegestane rasteracquisitie.
|
||||||
|
|
||||||
## LAS/LAZ LiDAR
|
## LAS/LAZ LiDAR
|
||||||
|
|
||||||
|
|||||||
+4
-3
@@ -751,11 +751,12 @@ This file now starts with the current implementation status. Older preparation/b
|
|||||||
creates a new immutable Dataset with official `YYYY.NN` source version.
|
creates a new immutable Dataset with official `YYYY.NN` source version.
|
||||||
# Bathymetry follow-up
|
# Bathymetry follow-up
|
||||||
|
|
||||||
- [ ] Add governed municipality partition orchestration for all of Flanders
|
- [x] Add governed municipality partition orchestration for all of Flanders
|
||||||
after the Mol VHA operator passes live acceptance.
|
after the Mol VHA operator passes live acceptance.
|
||||||
- [ ] Add read-only MDK WCS capability/TLS probe and maritime scope metadata.
|
- [x] Add read-only MDK WCS capability/TLS probe and maritime scope metadata.
|
||||||
- [ ] Add bounded MDK GeoTIFF acquisition only after live CRS, LAT, nodata and
|
- [ ] Add bounded MDK GeoTIFF acquisition only after live CRS, LAT, nodata and
|
||||||
response-limit validation.
|
response-limit validation. Current official endpoint is blocked by TLS
|
||||||
|
hostname mismatch and unavailable capabilities; never bypass verification.
|
||||||
- [ ] Add SPW staged-download adapter with mDNG metadata and survey-epoch
|
- [ ] Add SPW staged-download adapter with mDNG metadata and survey-epoch
|
||||||
coverage validation.
|
coverage validation.
|
||||||
- [ ] Add authoritative territorial-sea, EEZ and continental-shelf boundary
|
- [ ] Add authoritative territorial-sea, EEZ and continental-shelf boundary
|
||||||
|
|||||||
+4
-2
@@ -646,5 +646,7 @@ profile-document link.
|
|||||||
|
|
||||||
The UI always labels the points as historical cross-sections. It does not
|
The UI always labels the points as historical cross-sections. It does not
|
||||||
present them as a continuous bed raster and does not calculate water volume.
|
present them as a continuous bed raster and does not calculate water volume.
|
||||||
The Sources workspace lists the future MDK North Sea and SPW Walloon
|
Municipality partitions remain hidden for a regional Area until their backend
|
||||||
bathymetry integrations as planned until backend acquisition is operational.
|
manifest reports complete coverage. The Sources workspace lists MDK North Sea
|
||||||
|
as read-only probe-only and SPW Walloon bathymetry as planned until bounded
|
||||||
|
raster/download acquisition is operational.
|
||||||
|
|||||||
@@ -806,7 +806,7 @@ function App(): JSX.Element {
|
|||||||
|
|
||||||
{activeWorkspace === 'data' ? (
|
{activeWorkspace === 'data' ? (
|
||||||
<div className="workspace-grid workspace-grid-data">
|
<div className="workspace-grid workspace-grid-data">
|
||||||
<SourceCatalogPanel datasets={datasets} />
|
<SourceCatalogPanel datasets={datasets} projectId={selectedProjectId ?? undefined} />
|
||||||
<ProjectPanel
|
<ProjectPanel
|
||||||
projects={projects}
|
projects={projects}
|
||||||
selectedProjectId={selectedProjectId}
|
selectedProjectId={selectedProjectId}
|
||||||
|
|||||||
@@ -1,4 +1,6 @@
|
|||||||
import type { DatasetCreateResponse } from '../../types'
|
import { useState } from 'react'
|
||||||
|
import { datasetsApi } from '../../services/api'
|
||||||
|
import type { BathymetrySourceProbeRead, DatasetCreateResponse } from '../../types'
|
||||||
import { getDatasetSourceDisplayName } from '../../lib/datasetDisplay'
|
import { getDatasetSourceDisplayName } from '../../lib/datasetDisplay'
|
||||||
import {
|
import {
|
||||||
OFFICIAL_SOURCE_PORTFOLIO,
|
OFFICIAL_SOURCE_PORTFOLIO,
|
||||||
@@ -10,6 +12,7 @@ import {
|
|||||||
|
|
||||||
interface SourceCatalogPanelProps {
|
interface SourceCatalogPanelProps {
|
||||||
datasets: DatasetCreateResponse[]
|
datasets: DatasetCreateResponse[]
|
||||||
|
projectId?: string
|
||||||
}
|
}
|
||||||
|
|
||||||
const THEME_LABELS: Record<string, string> = {
|
const THEME_LABELS: Record<string, string> = {
|
||||||
@@ -63,7 +66,19 @@ function timelineSummary(
|
|||||||
].filter(Boolean).join(' · ')
|
].filter(Boolean).join(' · ')
|
||||||
}
|
}
|
||||||
|
|
||||||
export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.Element {
|
function mdkStatusLabel(status: BathymetrySourceProbeRead['status']): string {
|
||||||
|
if (status === 'reachable') return 'Service bereikbaar'
|
||||||
|
if (status === 'tls_error') return 'Geblokkeerd: certificaat'
|
||||||
|
if (status === 'endpoint_unavailable') return 'Service niet bereikbaar'
|
||||||
|
if (status === 'invalid_capabilities') return 'Ongeldige service-informatie'
|
||||||
|
if (status === 'disabled') return 'Controle uitgeschakeld'
|
||||||
|
return 'Configuratie controleren'
|
||||||
|
}
|
||||||
|
|
||||||
|
export function SourceCatalogPanel({ datasets, projectId }: SourceCatalogPanelProps): JSX.Element {
|
||||||
|
const [mdkProbe, setMdkProbe] = useState<BathymetrySourceProbeRead | null>(null)
|
||||||
|
const [mdkProbeLoading, setMdkProbeLoading] = useState(false)
|
||||||
|
const [mdkProbeError, setMdkProbeError] = useState<string | null>(null)
|
||||||
const ready = datasets.filter((dataset) => dataset.status === 'ready')
|
const ready = datasets.filter((dataset) => dataset.status === 'ready')
|
||||||
const waterinfoDatasets = ready.filter((dataset) => dataset.source_name === 'waterinfo')
|
const waterinfoDatasets = ready.filter((dataset) => dataset.source_name === 'waterinfo')
|
||||||
const historicalOrthophotos = ready.filter(
|
const historicalOrthophotos = ready.filter(
|
||||||
@@ -135,6 +150,19 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
const checkMdkReadiness = async (): Promise<void> => {
|
||||||
|
if (!projectId || mdkProbeLoading) return
|
||||||
|
setMdkProbeLoading(true)
|
||||||
|
setMdkProbeError(null)
|
||||||
|
try {
|
||||||
|
setMdkProbe(await datasetsApi.probeMdkBathymetry(projectId))
|
||||||
|
} catch (error) {
|
||||||
|
setMdkProbeError(error instanceof Error ? error.message : 'De Noordzee-bron kon niet worden gecontroleerd.')
|
||||||
|
} finally {
|
||||||
|
setMdkProbeLoading(false)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<section className="workspace-panel source-catalog-panel" aria-label="Beschikbare databronnen">
|
<section className="workspace-panel source-catalog-panel" aria-label="Beschikbare databronnen">
|
||||||
<div className="panel-title-row">
|
<div className="panel-title-row">
|
||||||
@@ -264,6 +292,29 @@ export function SourceCatalogPanel({ datasets }: SourceCatalogPanelProps): JSX.E
|
|||||||
</details>
|
</details>
|
||||||
) : null}
|
) : null}
|
||||||
|
|
||||||
|
<details className="source-loaded-disclosure">
|
||||||
|
<summary>Maritieme bronstatus</summary>
|
||||||
|
<div className="source-catalog-loaded">
|
||||||
|
<article>
|
||||||
|
<strong>MDK-dieptemodel Belgische Noordzee</strong>
|
||||||
|
<span>{mdkProbe ? mdkStatusLabel(mdkProbe.status) : 'Nog niet live gecontroleerd'}</span>
|
||||||
|
<p>
|
||||||
|
{mdkProbe?.message ??
|
||||||
|
'Controleert uitsluitend de officiële service-informatie met geldige TLS. Er wordt geen raster gedownload.'}
|
||||||
|
</p>
|
||||||
|
{mdkProbeError ? <small className="error-text">{mdkProbeError}</small> : null}
|
||||||
|
<button
|
||||||
|
className="secondary-button"
|
||||||
|
type="button"
|
||||||
|
disabled={!projectId || mdkProbeLoading}
|
||||||
|
onClick={() => void checkMdkReadiness()}
|
||||||
|
>
|
||||||
|
{mdkProbeLoading ? 'Controleren...' : 'Controleer Noordzee-bron'}
|
||||||
|
</button>
|
||||||
|
</article>
|
||||||
|
</div>
|
||||||
|
</details>
|
||||||
|
|
||||||
<details className="source-opportunity-list">
|
<details className="source-opportunity-list">
|
||||||
<summary>Officiële bronnen die hierna kunnen worden ingeladen</summary>
|
<summary>Officiële bronnen die hierna kunnen worden ingeladen</summary>
|
||||||
<div className="source-opportunity-domains">
|
<div className="source-opportunity-domains">
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ import type {
|
|||||||
FloodHazardProductRead,
|
FloodHazardProductRead,
|
||||||
FloodHazardSelectionResponse,
|
FloodHazardSelectionResponse,
|
||||||
BathymetryProfileAcquireRequest,
|
BathymetryProfileAcquireRequest,
|
||||||
|
BathymetrySourceProbeRead,
|
||||||
BathymetrySourceRead,
|
BathymetrySourceRead,
|
||||||
DhmvProductRead,
|
DhmvProductRead,
|
||||||
TerrainSelectionResponse,
|
TerrainSelectionResponse,
|
||||||
@@ -173,6 +174,10 @@ export const datasetsApi = {
|
|||||||
apiGet<{ items: BathymetrySourceRead[]; total: number }>(
|
apiGet<{ items: BathymetrySourceRead[]; total: number }>(
|
||||||
`/api/v1/projects/${projectId}/datasets/bathymetry/sources`,
|
`/api/v1/projects/${projectId}/datasets/bathymetry/sources`,
|
||||||
),
|
),
|
||||||
|
probeMdkBathymetry: (projectId: string): Promise<BathymetrySourceProbeRead> =>
|
||||||
|
apiGet<BathymetrySourceProbeRead>(
|
||||||
|
`/api/v1/projects/${projectId}/datasets/bathymetry/sources/mdk_bcp_bathymetry/readiness`,
|
||||||
|
),
|
||||||
acquireBathymetryProfiles: (projectId: string, payload: BathymetryProfileAcquireRequest): Promise<JobRead> =>
|
acquireBathymetryProfiles: (projectId: string, payload: BathymetryProfileAcquireRequest): Promise<JobRead> =>
|
||||||
apiPost<JobRead>(`/api/v1/projects/${projectId}/datasets/bathymetry/profiles/acquire`, payload),
|
apiPost<JobRead>(`/api/v1/projects/${projectId}/datasets/bathymetry/profiles/acquire`, payload),
|
||||||
acquireThematicRaster: (projectId: string, payload: ThematicRasterAcquireRequest): Promise<JobRead> =>
|
acquireThematicRaster: (projectId: string, payload: ThematicRasterAcquireRequest): Promise<JobRead> =>
|
||||||
|
|||||||
+25
-1
@@ -459,7 +459,7 @@ export interface BathymetrySourceRead {
|
|||||||
vertical_reference: string
|
vertical_reference: string
|
||||||
horizontal_crs: string
|
horizontal_crs: string
|
||||||
native_resolution?: string | null
|
native_resolution?: string | null
|
||||||
integration_status: 'operational' | 'available_not_integrated' | 'catalog_only'
|
integration_status: 'operational' | 'probe_only' | 'available_not_integrated' | 'catalog_only'
|
||||||
acquisition_supported: boolean
|
acquisition_supported: boolean
|
||||||
configured: boolean
|
configured: boolean
|
||||||
service_url?: string | null
|
service_url?: string | null
|
||||||
@@ -469,6 +469,30 @@ export interface BathymetrySourceRead {
|
|||||||
limitation_message: string
|
limitation_message: string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface BathymetrySourceProbeRead {
|
||||||
|
source_key: 'mdk_bcp_bathymetry'
|
||||||
|
status:
|
||||||
|
| 'disabled'
|
||||||
|
| 'invalid_configuration'
|
||||||
|
| 'tls_error'
|
||||||
|
| 'endpoint_unavailable'
|
||||||
|
| 'invalid_capabilities'
|
||||||
|
| 'reachable'
|
||||||
|
configured_url: string
|
||||||
|
capabilities_url?: string | null
|
||||||
|
tls_verified: boolean
|
||||||
|
capabilities_reachable: boolean
|
||||||
|
acquisition_supported: false
|
||||||
|
wcs_version?: string | null
|
||||||
|
coverage_identifiers: string[]
|
||||||
|
advertised_formats: string[]
|
||||||
|
advertised_crs: string[]
|
||||||
|
response_sha256?: string | null
|
||||||
|
checked_at: string
|
||||||
|
message: string
|
||||||
|
limitation_message: string
|
||||||
|
}
|
||||||
|
|
||||||
export interface ThematicRasterAcquireRequest {
|
export interface ThematicRasterAcquireRequest {
|
||||||
bbox: VectorSelectionBBox
|
bbox: VectorSelectionBBox
|
||||||
area_id?: string | null
|
area_id?: string | null
|
||||||
|
|||||||
@@ -1909,3 +1909,32 @@ python scripts/provision_mol_bathymetry_profiles.py \
|
|||||||
|
|
||||||
The output is a point dataset with historical evidence. It is not a continuous
|
The output is a point dataset with historical evidence. It is not a continuous
|
||||||
water-bottom raster and cannot calculate current water volume.
|
water-bottom raster and cannot calculate current water volume.
|
||||||
|
|
||||||
|
## Flanders VHA bathymetry partitions
|
||||||
|
|
||||||
|
Create the complete official Flemish land scope from the current VRBG
|
||||||
|
municipality collection:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec geointel python /app/scripts/provision_flanders_geographic_scope.py
|
||||||
|
```
|
||||||
|
|
||||||
|
Acquire VHA profile partitions with atomic resume evidence:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec geointel python /app/scripts/provision_flanders_bathymetry_profiles.py
|
||||||
|
```
|
||||||
|
|
||||||
|
Use `--members Mol Geel` or `--max-partitions 5` only for a partial operational
|
||||||
|
check. Partial runs do not activate regional coverage. The complete run
|
||||||
|
finalizes only when every official municipality has either one ready Dataset
|
||||||
|
or an explicit zero-profile source result.
|
||||||
|
|
||||||
|
Probe the official-metadata MDK WCS safely:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker exec geointel python /app/scripts/probe_mdk_bathymetry.py
|
||||||
|
```
|
||||||
|
|
||||||
|
This performs only `GetCapabilities`, keeps strict TLS verification enabled
|
||||||
|
and returns exit code `2` for an honest non-ready source.
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
"""Run the safe MDK WCS readiness probe through the canonical GeoIntel API."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||||
|
DEFAULT_PROJECT_NAME = "Kempen Regional Workbench"
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(description="Probe MDK bathymetry WCS readiness without downloading coverage.")
|
||||||
|
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
|
||||||
|
parser.add_argument("--project-name", default=DEFAULT_PROJECT_NAME)
|
||||||
|
parser.add_argument("--timeout", type=int, default=60)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def unwrap(response: requests.Response):
|
||||||
|
response.raise_for_status()
|
||||||
|
payload = response.json()
|
||||||
|
if not isinstance(payload, dict) or "data" not in payload:
|
||||||
|
raise RuntimeError(f"Non-canonical API response from {response.url}")
|
||||||
|
return payload["data"]
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
args = parse_args()
|
||||||
|
base_url = args.base_url.rstrip("/")
|
||||||
|
session = requests.Session()
|
||||||
|
projects = unwrap(
|
||||||
|
session.get(
|
||||||
|
f"{base_url}/api/v1/projects",
|
||||||
|
params={"name": args.project_name, "limit": 1},
|
||||||
|
timeout=args.timeout,
|
||||||
|
)
|
||||||
|
)["items"]
|
||||||
|
if not projects:
|
||||||
|
raise RuntimeError(f"Project {args.project_name!r} was not found")
|
||||||
|
result = unwrap(
|
||||||
|
session.get(
|
||||||
|
f"{base_url}/api/v1/projects/{projects[0]['id']}/datasets/"
|
||||||
|
"bathymetry/sources/mdk_bcp_bathymetry/readiness",
|
||||||
|
timeout=args.timeout,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||||
|
return 0 if result["status"] == "reachable" else 2
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,400 @@
|
|||||||
|
"""Provision VHA cross-section profiles for every persisted Flemish municipality.
|
||||||
|
|
||||||
|
The operator is resumable and calls only canonical GeoIntel API endpoints.
|
||||||
|
Regional completeness is finalized server-side only when every municipality
|
||||||
|
has either one ready profile Dataset or an explicit no-profile source result.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Iterable
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||||
|
DEFAULT_PROJECT_NAME = "Flanders Regional Workbench"
|
||||||
|
DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/bathymetry/vha-flanders/manifest.json")
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(description="Provision partitioned VHA profiles for Flanders.")
|
||||||
|
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
|
||||||
|
parser.add_argument("--project-name", default=DEFAULT_PROJECT_NAME)
|
||||||
|
parser.add_argument(
|
||||||
|
"--manifest-path",
|
||||||
|
type=Path,
|
||||||
|
default=Path(os.environ.get("GEOINTEL_VHA_FLANDERS_MANIFEST", DEFAULT_MANIFEST_PATH)),
|
||||||
|
)
|
||||||
|
parser.add_argument("--timeout", type=int, default=900)
|
||||||
|
parser.add_argument("--force", action="store_true")
|
||||||
|
parser.add_argument("--continue-on-error", action="store_true")
|
||||||
|
parser.add_argument("--members", nargs="*", default=[])
|
||||||
|
parser.add_argument("--max-partitions", type=int, default=0)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def unwrap(response: requests.Response) -> Any:
|
||||||
|
try:
|
||||||
|
payload = response.json()
|
||||||
|
except ValueError as exc:
|
||||||
|
raise RuntimeError(f"GeoIntel returned non-JSON ({response.status_code}): {response.text[:300]}") from exc
|
||||||
|
if not response.ok:
|
||||||
|
error_code = payload.get("error") if isinstance(payload, dict) else None
|
||||||
|
message = payload.get("message") if isinstance(payload, dict) else None
|
||||||
|
raise RuntimeError(f"{error_code or response.status_code}: {message or response.text[:300]}")
|
||||||
|
if not isinstance(payload, dict) or "data" not in payload:
|
||||||
|
raise RuntimeError(f"Non-canonical API response from {response.url}")
|
||||||
|
return payload["data"]
|
||||||
|
|
||||||
|
|
||||||
|
def list_paginated(session: requests.Session, url: str, *, timeout: int) -> list[dict[str, Any]]:
|
||||||
|
items: list[dict[str, Any]] = []
|
||||||
|
offset = 0
|
||||||
|
total: int | None = None
|
||||||
|
while True:
|
||||||
|
page = unwrap(session.get(url, params={"limit": 200, "offset": offset}, timeout=timeout))
|
||||||
|
page_items = list(page.get("items") or [])
|
||||||
|
items.extend(page_items)
|
||||||
|
total = int(page.get("total") or 0) if total is None else total
|
||||||
|
if not page_items or len(items) >= total:
|
||||||
|
break
|
||||||
|
offset += len(page_items)
|
||||||
|
if total is not None and len(items) != total:
|
||||||
|
raise RuntimeError(f"Paginated API returned {len(items)} of {total} records for {url}")
|
||||||
|
return items
|
||||||
|
|
||||||
|
|
||||||
|
def coordinates(geometry: dict[str, Any]) -> Iterable[tuple[float, float]]:
|
||||||
|
def walk(value: Any):
|
||||||
|
if isinstance(value, list) and len(value) >= 2 and all(isinstance(item, (int, float)) for item in value[:2]):
|
||||||
|
yield float(value[0]), float(value[1])
|
||||||
|
return
|
||||||
|
if isinstance(value, list):
|
||||||
|
for child in value:
|
||||||
|
yield from walk(child)
|
||||||
|
|
||||||
|
yield from walk(geometry.get("coordinates", []))
|
||||||
|
|
||||||
|
|
||||||
|
def geometry_bbox(geometry: dict[str, Any]) -> dict[str, float | str]:
|
||||||
|
points = list(coordinates(geometry))
|
||||||
|
if not points:
|
||||||
|
raise RuntimeError("Persisted municipality Area contains no coordinates")
|
||||||
|
xs = [point[0] for point in points]
|
||||||
|
ys = [point[1] for point in points]
|
||||||
|
return {
|
||||||
|
"min_x": min(xs),
|
||||||
|
"min_y": min(ys),
|
||||||
|
"max_x": max(xs),
|
||||||
|
"max_y": max(ys),
|
||||||
|
"crs": "EPSG:4326",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def area_identity(area: dict[str, Any]) -> str:
|
||||||
|
canonical = {
|
||||||
|
"id": area.get("id"),
|
||||||
|
"name": area.get("name"),
|
||||||
|
"geometry": area.get("geometry"),
|
||||||
|
}
|
||||||
|
return hashlib.sha256(
|
||||||
|
json.dumps(canonical, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def write_manifest(path: Path, payload: dict[str, Any]) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
temporary = path.with_suffix(path.suffix + ".partial")
|
||||||
|
temporary.write_text(
|
||||||
|
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
temporary.replace(path)
|
||||||
|
|
||||||
|
|
||||||
|
def load_manifest(path: Path) -> dict[str, Any]:
|
||||||
|
if not path.is_file():
|
||||||
|
return {}
|
||||||
|
try:
|
||||||
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, json.JSONDecodeError):
|
||||||
|
return {}
|
||||||
|
return payload if isinstance(payload, dict) else {}
|
||||||
|
|
||||||
|
|
||||||
|
def source_error(response: requests.Response) -> tuple[str | None, str]:
|
||||||
|
try:
|
||||||
|
payload = response.json()
|
||||||
|
except ValueError:
|
||||||
|
return None, response.text[:300]
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
return None, response.text[:300]
|
||||||
|
return (
|
||||||
|
str(payload.get("error")) if payload.get("error") else None,
|
||||||
|
str(payload.get("message") or response.text[:300]),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def acquire_partition(
|
||||||
|
session: requests.Session,
|
||||||
|
*,
|
||||||
|
base_url: str,
|
||||||
|
project_id: str,
|
||||||
|
area: dict[str, Any],
|
||||||
|
timeout: int,
|
||||||
|
force: bool,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
bbox = geometry_bbox(area["geometry"])
|
||||||
|
response = session.post(
|
||||||
|
f"{base_url}/api/v1/projects/{project_id}/datasets/bathymetry/profiles/acquire",
|
||||||
|
json={"bbox": bbox, "area_id": area["id"], "force_refresh": force},
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
if response.status_code == 404:
|
||||||
|
code, message = source_error(response)
|
||||||
|
if code == "BATHYMETRY_NO_PROFILES":
|
||||||
|
return {
|
||||||
|
"status": "no_profiles",
|
||||||
|
"area_id": area["id"],
|
||||||
|
"area_name": area["name"],
|
||||||
|
"area_identity_sha256": area_identity(area),
|
||||||
|
"bbox": bbox,
|
||||||
|
"message": message,
|
||||||
|
"completed_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
}
|
||||||
|
data = unwrap(response)
|
||||||
|
if data.get("status") != "success" or not data.get("output_dataset_id"):
|
||||||
|
raise RuntimeError(f"VHA acquisition failed for {area['name']}: {data.get('error_message') or data}")
|
||||||
|
result = data.get("result_json") or {}
|
||||||
|
return {
|
||||||
|
"status": "complete",
|
||||||
|
"area_id": area["id"],
|
||||||
|
"area_name": area["name"],
|
||||||
|
"area_identity_sha256": area_identity(area),
|
||||||
|
"bbox": bbox,
|
||||||
|
"dataset_id": data["output_dataset_id"],
|
||||||
|
"reused": bool(result.get("reused")),
|
||||||
|
"profile_count": int(result.get("profile_count") or 0),
|
||||||
|
"document_count": int(result.get("document_count") or 0),
|
||||||
|
"structured_depth_count": int(result.get("structured_depth_count") or 0),
|
||||||
|
"measurement_date_min": result.get("measurement_date_min"),
|
||||||
|
"measurement_date_max": result.get("measurement_date_max"),
|
||||||
|
"completed_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def manifest_identity(
|
||||||
|
*,
|
||||||
|
project_id: str,
|
||||||
|
partitions: list[dict[str, Any]],
|
||||||
|
) -> tuple[dict[str, Any], str]:
|
||||||
|
identity = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"source": "vmm_vha_bathymetry_profiles",
|
||||||
|
"partition_scope_key": "flanders",
|
||||||
|
"project_id": project_id,
|
||||||
|
"partitions": [
|
||||||
|
{
|
||||||
|
key: partition.get(key)
|
||||||
|
for key in (
|
||||||
|
"area_id",
|
||||||
|
"area_name",
|
||||||
|
"area_identity_sha256",
|
||||||
|
"status",
|
||||||
|
"dataset_id",
|
||||||
|
"profile_count",
|
||||||
|
"document_count",
|
||||||
|
"structured_depth_count",
|
||||||
|
"measurement_date_min",
|
||||||
|
"measurement_date_max",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
for partition in sorted(partitions, key=lambda item: str(item["area_id"]))
|
||||||
|
],
|
||||||
|
}
|
||||||
|
digest = hashlib.sha256(
|
||||||
|
json.dumps(identity, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||||
|
).hexdigest()
|
||||||
|
return identity, digest
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
args = parse_args()
|
||||||
|
if args.max_partitions < 0:
|
||||||
|
print(json.dumps({"status": "error", "message": "--max-partitions cannot be negative"}), file=sys.stderr)
|
||||||
|
return 2
|
||||||
|
base_url = args.base_url.rstrip("/")
|
||||||
|
session = requests.Session()
|
||||||
|
session.headers.update({"User-Agent": "GeoIntel-VHA-Flanders-Operator/1.0"})
|
||||||
|
manifest = load_manifest(args.manifest_path)
|
||||||
|
|
||||||
|
try:
|
||||||
|
projects = list_paginated(session, f"{base_url}/api/v1/projects", timeout=60)
|
||||||
|
project = next((item for item in projects if item.get("name") == args.project_name), None)
|
||||||
|
if project is None:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Project {args.project_name!r} was not found; run provision_flanders_geographic_scope.py first"
|
||||||
|
)
|
||||||
|
all_areas = list_paginated(
|
||||||
|
session,
|
||||||
|
f"{base_url}/api/v1/projects/{project['id']}/areas",
|
||||||
|
timeout=120,
|
||||||
|
)
|
||||||
|
municipalities = sorted(
|
||||||
|
(area for area in all_areas if str(area.get("name") or "").casefold().startswith("gemeente ")),
|
||||||
|
key=lambda item: str(item["name"]).casefold(),
|
||||||
|
)
|
||||||
|
if not 270 <= len(municipalities) <= 300:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Flanders workspace exposes {len(municipalities)} municipality Areas; expected 270..300"
|
||||||
|
)
|
||||||
|
requested_names = {name.casefold() for name in args.members}
|
||||||
|
selected = [
|
||||||
|
area
|
||||||
|
for area in municipalities
|
||||||
|
if not requested_names
|
||||||
|
or str(area["name"]).removeprefix("Gemeente ").split(" - ", 1)[0].casefold() in requested_names
|
||||||
|
]
|
||||||
|
if requested_names:
|
||||||
|
found = {
|
||||||
|
str(area["name"]).removeprefix("Gemeente ").split(" - ", 1)[0].casefold()
|
||||||
|
for area in selected
|
||||||
|
}
|
||||||
|
missing = sorted(requested_names - found)
|
||||||
|
if missing:
|
||||||
|
raise RuntimeError(f"Unknown Flanders municipality selections: {', '.join(missing)}")
|
||||||
|
if args.max_partitions:
|
||||||
|
selected = selected[: args.max_partitions]
|
||||||
|
|
||||||
|
prior_by_area = {
|
||||||
|
str(item.get("area_id")): item
|
||||||
|
for item in (manifest.get("partitions") or [])
|
||||||
|
if isinstance(item, dict) and item.get("area_id")
|
||||||
|
}
|
||||||
|
current: list[dict[str, Any]] = []
|
||||||
|
failures: list[dict[str, Any]] = []
|
||||||
|
observed_at = datetime.now(timezone.utc).isoformat()
|
||||||
|
working_manifest = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"status": "running",
|
||||||
|
"source": "vmm_vha_bathymetry_profiles",
|
||||||
|
"partition_scope_key": "flanders",
|
||||||
|
"project_id": project["id"],
|
||||||
|
"project_name": project["name"],
|
||||||
|
"municipality_inventory_count": len(municipalities),
|
||||||
|
"selected_partition_count": len(selected),
|
||||||
|
"observed_at": observed_at,
|
||||||
|
"partitions": current,
|
||||||
|
}
|
||||||
|
for index, area in enumerate(selected, start=1):
|
||||||
|
identity = area_identity(area)
|
||||||
|
prior = prior_by_area.get(str(area["id"]))
|
||||||
|
if (
|
||||||
|
not args.force
|
||||||
|
and prior
|
||||||
|
and prior.get("area_identity_sha256") == identity
|
||||||
|
and prior.get("status") in {"complete", "no_profiles"}
|
||||||
|
):
|
||||||
|
result = prior
|
||||||
|
else:
|
||||||
|
try:
|
||||||
|
result = acquire_partition(
|
||||||
|
session,
|
||||||
|
base_url=base_url,
|
||||||
|
project_id=str(project["id"]),
|
||||||
|
area=area,
|
||||||
|
timeout=args.timeout,
|
||||||
|
force=args.force,
|
||||||
|
)
|
||||||
|
except (RuntimeError, requests.RequestException) as exc:
|
||||||
|
result = {
|
||||||
|
"status": "failed",
|
||||||
|
"area_id": area["id"],
|
||||||
|
"area_name": area["name"],
|
||||||
|
"area_identity_sha256": identity,
|
||||||
|
"message": str(exc),
|
||||||
|
"failed_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
}
|
||||||
|
failures.append(result)
|
||||||
|
current.append(result)
|
||||||
|
working_manifest["completed_partition_count"] = index
|
||||||
|
working_manifest["partitions"] = current
|
||||||
|
write_manifest(args.manifest_path, working_manifest)
|
||||||
|
if result["status"] == "failed" and not args.continue_on_error:
|
||||||
|
raise RuntimeError(f"Partition failed for {area['name']}: {result['message']}")
|
||||||
|
|
||||||
|
full_inventory_selected = len(selected) == len(municipalities) and not requested_names and not args.max_partitions
|
||||||
|
complete = not failures and all(item["status"] in {"complete", "no_profiles"} for item in current)
|
||||||
|
identity_payload, identity_sha = manifest_identity(
|
||||||
|
project_id=str(project["id"]),
|
||||||
|
partitions=current,
|
||||||
|
)
|
||||||
|
finalization = None
|
||||||
|
if complete and full_inventory_selected:
|
||||||
|
dataset_ids = [item["dataset_id"] for item in current if item["status"] == "complete"]
|
||||||
|
no_profile_area_ids = [item["area_id"] for item in current if item["status"] == "no_profiles"]
|
||||||
|
finalization = unwrap(
|
||||||
|
session.post(
|
||||||
|
f"{base_url}/api/v1/projects/{project['id']}/datasets/bathymetry/profiles/partitions/finalize",
|
||||||
|
json={
|
||||||
|
"partition_scope_key": "flanders",
|
||||||
|
"expected_area_ids": [area["id"] for area in municipalities],
|
||||||
|
"dataset_ids": dataset_ids,
|
||||||
|
"no_profile_area_ids": no_profile_area_ids,
|
||||||
|
"manifest_sha256": identity_sha,
|
||||||
|
"observed_at": observed_at,
|
||||||
|
},
|
||||||
|
timeout=args.timeout,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
working_manifest.update(
|
||||||
|
{
|
||||||
|
"status": "complete" if complete else "failed",
|
||||||
|
"completed_at": datetime.now(timezone.utc).isoformat(),
|
||||||
|
"coverage_manifest": identity_payload,
|
||||||
|
"coverage_manifest_sha256": identity_sha,
|
||||||
|
"regional_finalized": finalization is not None,
|
||||||
|
"finalization": finalization,
|
||||||
|
"failed_partition_count": len(failures),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
write_manifest(args.manifest_path, working_manifest)
|
||||||
|
except (OSError, RuntimeError, ValueError, KeyError, requests.RequestException) as exc:
|
||||||
|
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"status": working_manifest["status"],
|
||||||
|
"project_id": project["id"],
|
||||||
|
"municipality_inventory_count": len(municipalities),
|
||||||
|
"processed_partition_count": len(current),
|
||||||
|
"data_partition_count": sum(item["status"] == "complete" for item in current),
|
||||||
|
"no_profile_partition_count": sum(item["status"] == "no_profiles" for item in current),
|
||||||
|
"failed_partition_count": len(failures),
|
||||||
|
"profile_count": sum(int(item.get("profile_count") or 0) for item in current),
|
||||||
|
"document_count": sum(int(item.get("document_count") or 0) for item in current),
|
||||||
|
"regional_finalized": finalization is not None,
|
||||||
|
"manifest_path": str(args.manifest_path),
|
||||||
|
"manifest_sha256": identity_sha,
|
||||||
|
"finalization": finalization,
|
||||||
|
},
|
||||||
|
ensure_ascii=False,
|
||||||
|
indent=2,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return 0 if working_manifest["status"] == "complete" else 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,220 @@
|
|||||||
|
"""Provision all current official Flemish municipality Areas.
|
||||||
|
|
||||||
|
The municipality inventory is discovered from the official VRBG RefGem
|
||||||
|
collection at runtime. The script uses the existing geographic-scope artifact
|
||||||
|
and API persistence flow; it never writes directly to PostGIS.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
|
from geographic_scopes import GeographicScope, ScopeMember
|
||||||
|
from provision_geographic_scope import (
|
||||||
|
VRBG_ATTRIBUTION,
|
||||||
|
VRBG_ITEMS_URL,
|
||||||
|
build_scope_payloads,
|
||||||
|
build_source_session,
|
||||||
|
provision_scope,
|
||||||
|
sha256_file,
|
||||||
|
write_json_atomic,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||||
|
DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-data/geographic-scopes")
|
||||||
|
MIN_EXPECTED_MUNICIPALITIES = 270
|
||||||
|
MAX_EXPECTED_MUNICIPALITIES = 300
|
||||||
|
|
||||||
|
|
||||||
|
def parse_args() -> argparse.Namespace:
|
||||||
|
parser = argparse.ArgumentParser(description="Provision the current official Flanders municipality scope.")
|
||||||
|
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
|
||||||
|
parser.add_argument(
|
||||||
|
"--output-root",
|
||||||
|
type=Path,
|
||||||
|
default=Path(os.environ.get("GEOINTEL_SCOPE_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)),
|
||||||
|
)
|
||||||
|
parser.add_argument("--request-timeout", type=int, default=180)
|
||||||
|
parser.add_argument("--import-timeout", type=int, default=3600)
|
||||||
|
parser.add_argument("--force", action="store_true")
|
||||||
|
parser.add_argument("--fetch-only", action="store_true")
|
||||||
|
parser.add_argument("--min-municipalities", type=int, default=MIN_EXPECTED_MUNICIPALITIES)
|
||||||
|
parser.add_argument("--max-municipalities", type=int, default=MAX_EXPECTED_MUNICIPALITIES)
|
||||||
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def discover_flanders_scope(
|
||||||
|
session: requests.Session,
|
||||||
|
*,
|
||||||
|
timeout: int,
|
||||||
|
min_municipalities: int,
|
||||||
|
max_municipalities: int,
|
||||||
|
) -> tuple[GeographicScope, list[dict[str, Any]], str]:
|
||||||
|
if min_municipalities <= 0 or max_municipalities < min_municipalities:
|
||||||
|
raise ValueError("Municipality count safety limits are invalid")
|
||||||
|
response = session.get(
|
||||||
|
VRBG_ITEMS_URL,
|
||||||
|
params={"f": "application/geo+json", "limit": "1000"},
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
payload = response.json()
|
||||||
|
features = payload.get("features")
|
||||||
|
if not isinstance(features, list):
|
||||||
|
raise RuntimeError("Official VRBG response does not contain a feature list")
|
||||||
|
if not min_municipalities <= len(features) <= max_municipalities:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Official VRBG returned {len(features)} municipalities; expected "
|
||||||
|
f"{min_municipalities}..{max_municipalities}. Refusing an incomplete or broadened scope."
|
||||||
|
)
|
||||||
|
|
||||||
|
by_code: dict[str, dict[str, Any]] = {}
|
||||||
|
names: set[str] = set()
|
||||||
|
for feature in features:
|
||||||
|
if not isinstance(feature, dict) or not isinstance(feature.get("geometry"), dict):
|
||||||
|
raise RuntimeError("Official VRBG contains a malformed municipality feature")
|
||||||
|
properties = feature.get("properties")
|
||||||
|
if not isinstance(properties, dict):
|
||||||
|
raise RuntimeError("Official VRBG municipality is missing properties")
|
||||||
|
nis_code = str(properties.get("NISCODE") or "").strip()
|
||||||
|
name = str(properties.get("NAAM") or "").strip()
|
||||||
|
if len(nis_code) != 5 or not nis_code.isdigit() or not name:
|
||||||
|
raise RuntimeError(f"Official VRBG municipality identity is invalid: {nis_code!r} / {name!r}")
|
||||||
|
if nis_code in by_code or name.casefold() in names:
|
||||||
|
raise RuntimeError(f"Official VRBG contains a duplicate municipality: {nis_code} / {name}")
|
||||||
|
by_code[nis_code] = feature
|
||||||
|
names.add(name.casefold())
|
||||||
|
|
||||||
|
ordered = [by_code[code] for code in sorted(by_code)]
|
||||||
|
members = tuple(
|
||||||
|
ScopeMember(
|
||||||
|
name=str(feature["properties"]["NAAM"]).strip(),
|
||||||
|
nis_code=str(feature["properties"]["NISCODE"]).strip(),
|
||||||
|
)
|
||||||
|
for feature in ordered
|
||||||
|
)
|
||||||
|
scope = GeographicScope(
|
||||||
|
key="flanders",
|
||||||
|
display_name=f"Vlaanderen ({len(members)} gemeenten)",
|
||||||
|
project_name="Flanders Regional Workbench",
|
||||||
|
project_region="Vlaanderen, België",
|
||||||
|
area_name="Vlaanderen - officiële operationele grens",
|
||||||
|
authority_name="Digitaal Vlaanderen VRBG",
|
||||||
|
authority_url=VRBG_ITEMS_URL,
|
||||||
|
scope_type="region",
|
||||||
|
limitation_message=(
|
||||||
|
"Officiële unie van de actuele VRBG-gemeentegrenzen. De operationele regio omvat het Vlaamse "
|
||||||
|
"landgebied; territoriale zee, EEZ en continentaal plat zijn afzonderlijke maritieme scopes."
|
||||||
|
),
|
||||||
|
members=members,
|
||||||
|
)
|
||||||
|
return scope, ordered, response.url
|
||||||
|
|
||||||
|
|
||||||
|
def prepare_artifacts(
|
||||||
|
args: argparse.Namespace,
|
||||||
|
scope: GeographicScope,
|
||||||
|
features: list[dict[str, Any]],
|
||||||
|
source_url: str,
|
||||||
|
) -> tuple[Path, Path, dict[str, Any]]:
|
||||||
|
output_dir = args.output_root / scope.key
|
||||||
|
manifest_path = output_dir / "flanders_scope_manifest.json"
|
||||||
|
generated_at = datetime.now(timezone.utc).isoformat()
|
||||||
|
snapshot_date = generated_at[:10]
|
||||||
|
boundary_path = output_dir / f"flanders_boundary_{snapshot_date}.geojson"
|
||||||
|
members_path = output_dir / f"flanders_municipalities_{snapshot_date}.geojson"
|
||||||
|
member_codes = list(scope.nis_codes)
|
||||||
|
|
||||||
|
if not args.force and manifest_path.is_file():
|
||||||
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
cached_boundary = output_dir / str(manifest.get("boundary_filename") or "")
|
||||||
|
cached_members = output_dir / str(manifest.get("municipalities_filename") or "")
|
||||||
|
if (
|
||||||
|
manifest.get("status") == "complete"
|
||||||
|
and manifest.get("member_nis_codes") == member_codes
|
||||||
|
and cached_boundary.is_file()
|
||||||
|
and cached_members.is_file()
|
||||||
|
and sha256_file(cached_boundary) == manifest.get("boundary_sha256")
|
||||||
|
and sha256_file(cached_members) == manifest.get("municipalities_sha256")
|
||||||
|
):
|
||||||
|
return cached_boundary, cached_members, manifest
|
||||||
|
|
||||||
|
boundary_payload, members_payload, summary = build_scope_payloads(
|
||||||
|
scope,
|
||||||
|
features,
|
||||||
|
source_url=source_url,
|
||||||
|
generated_at=generated_at,
|
||||||
|
)
|
||||||
|
write_json_atomic(boundary_path, boundary_payload)
|
||||||
|
write_json_atomic(members_path, members_payload)
|
||||||
|
manifest = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"status": "complete",
|
||||||
|
"generated_at": generated_at,
|
||||||
|
"observed_at": f"{snapshot_date}T00:00:00Z",
|
||||||
|
"boundary_filename": boundary_path.name,
|
||||||
|
"boundary_sha256": sha256_file(boundary_path),
|
||||||
|
"municipalities_filename": members_path.name,
|
||||||
|
"municipalities_sha256": sha256_file(members_path),
|
||||||
|
"vrbg_source_url": source_url,
|
||||||
|
"vrbg_attribution": VRBG_ATTRIBUTION,
|
||||||
|
"scope_authority_name": scope.authority_name,
|
||||||
|
"scope_authority_url": scope.authority_url,
|
||||||
|
"scope_limitation": scope.limitation_message,
|
||||||
|
**summary,
|
||||||
|
}
|
||||||
|
write_json_atomic(manifest_path, manifest, pretty=True)
|
||||||
|
return boundary_path, members_path, manifest
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
args = parse_args()
|
||||||
|
args.operator_tool = "provision_flanders_geographic_scope.py"
|
||||||
|
try:
|
||||||
|
with build_source_session() as session:
|
||||||
|
scope, features, source_url = discover_flanders_scope(
|
||||||
|
session,
|
||||||
|
timeout=args.request_timeout,
|
||||||
|
min_municipalities=args.min_municipalities,
|
||||||
|
max_municipalities=args.max_municipalities,
|
||||||
|
)
|
||||||
|
boundary_path, members_path, manifest = prepare_artifacts(args, scope, features, source_url)
|
||||||
|
workspace = None if args.fetch_only else provision_scope(
|
||||||
|
args, scope, boundary_path, members_path, manifest
|
||||||
|
)
|
||||||
|
except (OSError, RuntimeError, ValueError, KeyError, requests.RequestException) as exc:
|
||||||
|
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
|
||||||
|
return 1
|
||||||
|
|
||||||
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"status": "ok",
|
||||||
|
"mode": "fetch_only" if args.fetch_only else "provisioned",
|
||||||
|
"scope": scope.key,
|
||||||
|
"member_count": manifest["member_count"],
|
||||||
|
"area_km2": manifest["area_km2"],
|
||||||
|
"wgs84_bbox": manifest["wgs84_bbox"],
|
||||||
|
"boundary_path": str(boundary_path),
|
||||||
|
"municipalities_path": str(members_path),
|
||||||
|
"workspace": workspace,
|
||||||
|
"limitation": scope.limitation_message,
|
||||||
|
},
|
||||||
|
ensure_ascii=False,
|
||||||
|
indent=2,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -478,7 +478,7 @@ def provision_scope(
|
|||||||
"attribution": VRBG_ATTRIBUTION,
|
"attribution": VRBG_ATTRIBUTION,
|
||||||
}
|
}
|
||||||
common_provenance = {
|
common_provenance = {
|
||||||
"operator_tool": "provision_geographic_scope.py",
|
"operator_tool": getattr(args, "operator_tool", "provision_geographic_scope.py"),
|
||||||
"operator_explicit_fetch": True,
|
"operator_explicit_fetch": True,
|
||||||
"manifest_path": str(args.output_root / scope.key / f"{scope.key.replace('-', '_')}_scope_manifest.json"),
|
"manifest_path": str(args.output_root / scope.key / f"{scope.key.replace('-', '_')}_scope_manifest.json"),
|
||||||
"source_url": manifest["vrbg_source_url"],
|
"source_url": manifest["vrbg_source_url"],
|
||||||
|
|||||||
@@ -51,6 +51,9 @@ ${PYTHON_BIN} -m py_compile scripts/provision_mol_historical_landuse.py
|
|||||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_historical_landuse.py
|
${PYTHON_BIN} -m py_compile scripts/provision_regional_historical_landuse.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/provision_official_landuse_timeseries.py
|
${PYTHON_BIN} -m py_compile scripts/provision_official_landuse_timeseries.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/provision_waterinfo_station_history.py
|
${PYTHON_BIN} -m py_compile scripts/provision_waterinfo_station_history.py
|
||||||
|
${PYTHON_BIN} -m py_compile scripts/provision_flanders_geographic_scope.py
|
||||||
|
${PYTHON_BIN} -m py_compile scripts/provision_flanders_bathymetry_profiles.py
|
||||||
|
${PYTHON_BIN} -m py_compile scripts/probe_mdk_bathymetry.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py
|
${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_bwk_natura2000.py
|
${PYTHON_BIN} -m py_compile scripts/provision_regional_bwk_natura2000.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py
|
${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py
|
||||||
|
|||||||
Reference in New Issue
Block a user