Add regional flood hazard provisioning
This commit is contained in:
@@ -7,6 +7,22 @@
|
||||
|
||||
# Changelog
|
||||
|
||||
## Sprint 211 Regional VMM flood-hazard provisioning support (2026-07-16)
|
||||
|
||||
- Added `provision_regional_flood_hazards.py`, an explicit operator for the
|
||||
approved 28-municipality Kempen scope that provisions official VMM
|
||||
flood-depth scenarios per municipality Area through the existing canonical
|
||||
flood-hazard API.
|
||||
- Kept persistence inside the existing Dataset/DatasetVersion/Job raster flow;
|
||||
no direct PostGIS writes, no browser-side provider fetches, no startup fetches
|
||||
and no new schema/API contract were introduced.
|
||||
- Added dry-run, member/product subset, resume/reuse and failure-reporting
|
||||
controls so operators can validate Mol or selected municipalities before the
|
||||
full 336 municipality/scenario matrix.
|
||||
- Updated Unraid packaging, readiness checks, tests and docs. The semantic
|
||||
limitation remains explicit: VMM water depth is modeled scenario depth, not
|
||||
permanent water volume, current water level or bathymetry.
|
||||
|
||||
## Sprint 210 Regional BWK/Natura 2000 expansion (2026-07-16)
|
||||
|
||||
- Added an explicit 28-municipality operator for the official INBO BWK/Natura
|
||||
|
||||
@@ -1246,6 +1246,37 @@ Run all scenarios for Mol after the regional workspace and Mol Area exist:
|
||||
docker exec geointel python /app/scripts/provision_mol_flood_hazards.py
|
||||
```
|
||||
|
||||
Provision the same official VMM scenario set for every persisted municipality
|
||||
Area in the approved Kempen regional workspace:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_regional_flood_hazards.py \
|
||||
--scope kempen-transport-region
|
||||
```
|
||||
|
||||
Inspect the planned municipality/scenario matrix without writing data:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_regional_flood_hazards.py \
|
||||
--scope kempen-transport-region --dry-run
|
||||
```
|
||||
|
||||
Useful bounded runs:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_regional_flood_hazards.py \
|
||||
--members Mol,Geel --products pluviaal_current_t100
|
||||
docker exec geointel python /app/scripts/provision_regional_flood_hazards.py \
|
||||
--members 13025 --products pluviaal_current_t10,pluviaal_current_t100
|
||||
```
|
||||
|
||||
The regional operator uses the canonical API only. It requires the geographic
|
||||
scope Areas to exist first, persists one ordinary raster Dataset per
|
||||
municipality/scenario and reuses existing Datasets unless `--force` is supplied.
|
||||
The full Kempen scope with all products means 28 municipalities times 12
|
||||
scenario rasters. This is intentionally explicit operator work, not startup
|
||||
work and not a browser-side provider fetch.
|
||||
|
||||
Use `--products pluviaal_current_t100`, `--resolution-m 5` or `--force` for an
|
||||
explicit subset/refresh. `POST .../raster/flood-hazard/select` returns mapped
|
||||
inundated hectares, selection share and local modeled maximum-depth statistics.
|
||||
|
||||
@@ -0,0 +1,167 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
|
||||
|
||||
def load_operator():
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"test_provision_regional_flood_hazards",
|
||||
SCRIPTS / "provision_regional_flood_hazards.py",
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
assert spec and spec.loader
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
def __init__(self, payload: dict[str, Any]):
|
||||
self.payload = payload
|
||||
self.url = "http://test.local"
|
||||
|
||||
def raise_for_status(self) -> None:
|
||||
return None
|
||||
|
||||
def json(self) -> dict[str, Any]:
|
||||
return self.payload
|
||||
|
||||
|
||||
class FakeSession:
|
||||
def __init__(self, module):
|
||||
self.module = module
|
||||
self.posts: list[tuple[str, dict[str, Any]]] = []
|
||||
self.headers: dict[str, str] = {}
|
||||
|
||||
def get(self, url: str, **_kwargs):
|
||||
if url.endswith("/api/v1/projects"):
|
||||
return FakeResponse({"data": {"items": [{"id": "project-1", "name": "Kempen Regional Workbench"}]}})
|
||||
if url.endswith("/areas"):
|
||||
return FakeResponse(
|
||||
{
|
||||
"data": {
|
||||
"items": [
|
||||
{
|
||||
"id": "area-mol",
|
||||
"name": "Gemeente Mol - officiele grens",
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [[
|
||||
[5.0, 51.0],
|
||||
[5.1, 51.0],
|
||||
[5.1, 51.1],
|
||||
[5.0, 51.1],
|
||||
[5.0, 51.0],
|
||||
]],
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
)
|
||||
if url.endswith("/datasets/flood-hazard/products"):
|
||||
return FakeResponse({"data": {"items": [{"key": key} for key in self.module.PRODUCTS]}})
|
||||
raise AssertionError(url)
|
||||
|
||||
def post(self, url: str, json: dict[str, Any], **_kwargs):
|
||||
self.posts.append((url, json))
|
||||
if url.endswith("/datasets/flood-hazard/acquire"):
|
||||
return FakeResponse(
|
||||
{
|
||||
"data": {
|
||||
"id": "job-1",
|
||||
"status": "success",
|
||||
"output_dataset_id": "dataset-1",
|
||||
"result_json": {"reused": True},
|
||||
}
|
||||
}
|
||||
)
|
||||
if url.endswith("/raster/flood-hazard/select"):
|
||||
return FakeResponse(
|
||||
{
|
||||
"data": {
|
||||
"resolution_m": 5.0,
|
||||
"selected_cell_count": 100,
|
||||
"inundated_cell_count": 10,
|
||||
"unsupported_metrics": [
|
||||
"bathymetry_depth_m",
|
||||
"permanent_water_volume_m3",
|
||||
"concurrent_flood_volume_m3",
|
||||
],
|
||||
"summary": {"metrics": [{"metric_key": "modelled_inundated_area_ha", "metric_value": 0.25}]},
|
||||
}
|
||||
}
|
||||
)
|
||||
raise AssertionError(url)
|
||||
|
||||
|
||||
def test_regional_flood_operator_resolves_products_and_members() -> None:
|
||||
module = load_operator()
|
||||
|
||||
products = module.requested_products("pluviaal_current_t100,fluviaal_future_2050_t1000", set(module.PRODUCTS))
|
||||
members = module.requested_members("Mol,13008", module.KEMPEN_TRANSPORT_REGION_SCOPE.members)
|
||||
|
||||
assert products == ["pluviaal_current_t100", "fluviaal_future_2050_t1000"]
|
||||
assert [member.nis_code for member in members] == ["13025", "13008"]
|
||||
|
||||
|
||||
def test_regional_flood_operator_rejects_unknown_scope_inputs() -> None:
|
||||
module = load_operator()
|
||||
|
||||
with pytest.raises(RuntimeError, match="Unsupported flood-hazard"):
|
||||
module.requested_products("custom", set(module.PRODUCTS))
|
||||
with pytest.raises(RuntimeError, match="Unknown scope members"):
|
||||
module.requested_members("Atlantis", module.KEMPEN_TRANSPORT_REGION_SCOPE.members)
|
||||
|
||||
|
||||
def test_regional_flood_operator_dry_run_uses_canonical_registry(monkeypatch, capsys) -> None:
|
||||
module = load_operator()
|
||||
fake_session = FakeSession(module)
|
||||
monkeypatch.setattr(module.requests, "Session", lambda: fake_session)
|
||||
|
||||
result = module.main(["--members", "Mol", "--products", "pluviaal_current_t100", "--dry-run"])
|
||||
output = capsys.readouterr().out
|
||||
|
||||
assert result == 0
|
||||
assert '"status": "dry_run"' in output
|
||||
assert '"planned_acquisitions": 1' in output
|
||||
assert fake_session.posts == []
|
||||
|
||||
|
||||
def test_regional_flood_operator_calls_acquisition_and_selection(monkeypatch, capsys) -> None:
|
||||
module = load_operator()
|
||||
fake_session = FakeSession(module)
|
||||
monkeypatch.setattr(module.requests, "Session", lambda: fake_session)
|
||||
|
||||
result = module.main(["--members", "Mol", "--products", "pluviaal_current_t100"])
|
||||
output = capsys.readouterr().out
|
||||
|
||||
assert result == 0
|
||||
assert '"completed_count": 1' in output
|
||||
assert len(fake_session.posts) == 2
|
||||
acquisition_payload = fake_session.posts[0][1]
|
||||
assert fake_session.posts[0][0].endswith("/datasets/flood-hazard/acquire")
|
||||
assert acquisition_payload["area_id"] == "area-mol"
|
||||
assert acquisition_payload["product_key"] == "pluviaal_current_t100"
|
||||
assert acquisition_payload["bbox"]["crs"] == "EPSG:4326"
|
||||
|
||||
|
||||
def test_regional_flood_operator_is_packaged() -> None:
|
||||
readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
docs = (ROOT / "scripts" / "README.md").read_text(encoding="utf-8")
|
||||
|
||||
assert "py_compile scripts/provision_regional_flood_hazards.py" in readiness
|
||||
assert "COPY scripts/provision_regional_flood_hazards.py" in dockerfile
|
||||
assert "provision_regional_flood_hazards.py" in docs
|
||||
@@ -76,6 +76,7 @@ COPY scripts/provision_mol_municipality_workspace.py /app/scripts/provision_mol_
|
||||
COPY scripts/provision_mol_context_layers.py /app/scripts/provision_mol_context_layers.py
|
||||
COPY scripts/provision_mol_dhmv.py /app/scripts/provision_mol_dhmv.py
|
||||
COPY scripts/provision_mol_flood_hazards.py /app/scripts/provision_mol_flood_hazards.py
|
||||
COPY scripts/provision_regional_flood_hazards.py /app/scripts/provision_regional_flood_hazards.py
|
||||
COPY scripts/provision_mol_population_history.py /app/scripts/provision_mol_population_history.py
|
||||
COPY scripts/provision_mol_historical_landuse.py /app/scripts/provision_mol_historical_landuse.py
|
||||
COPY scripts/provision_regional_historical_landuse.py /app/scripts/provision_regional_historical_landuse.py
|
||||
|
||||
@@ -8940,3 +8940,40 @@ Boundaries and next step:
|
||||
governed municipality partitions and measured VMM scenario rasters before
|
||||
attempting a full-resolution regional DHMV expansion; no water volume or
|
||||
inland bathymetry may be inferred from flood depth.
|
||||
|
||||
## Sprint 211 - Regional VMM flood-hazard provisioning support (2026-07-16)
|
||||
|
||||
Implemented:
|
||||
- Added `scripts/provision_regional_flood_hazards.py`, an explicit operator
|
||||
that resolves the approved geographic scope, validates the fixed twelve-item
|
||||
VMM flood-hazard registry and acquires scenarios per persisted municipality
|
||||
Area through the existing canonical API endpoints.
|
||||
- Kept all raster persistence inside the existing `FloodHazardAcquisitionService`
|
||||
and Dataset/DatasetVersion/Job flow. The operator performs no direct WCS
|
||||
requests, no direct PostGIS writes and no browser/startup provider fetching.
|
||||
- Added `--dry-run`, `--members`, `--products`, `--force` and
|
||||
`--stop-on-error` controls so Mol, selected municipalities or the complete
|
||||
28-member scope can be run safely and resumed.
|
||||
- Added packaging/readiness support so the operator is compiled and available
|
||||
in the all-in-one Unraid image.
|
||||
- Documented that regional flood coverage remains municipality-partitioned
|
||||
because one monolithic Kempen raster would exceed practical WCS/pixel limits.
|
||||
|
||||
Validation:
|
||||
- Focused tests cover product/member resolution, dry-run planning, canonical
|
||||
acquisition/selection calls and release packaging.
|
||||
|
||||
Known limitations:
|
||||
- This pass adds the operational regional provisioner and validation contract.
|
||||
A complete live run of all 336 municipality/scenario acquisitions is long
|
||||
operator work and should be launched deliberately on Tower after reviewing
|
||||
the dry-run matrix.
|
||||
- VMM flood depth remains modelled local maximum scenario depth. GeoIntel still
|
||||
must not expose permanent waterbody volume, current water level or
|
||||
bathymetry from this source.
|
||||
|
||||
Next:
|
||||
- Run the regional VMM operator first for Mol/Geel with `pluviaal_current_t100`
|
||||
on Tower, then expand to the full twelve-scenario municipality matrix if the
|
||||
provider remains stable. After that, prioritize regional DHMV DTM/DSM with
|
||||
the same partitioned operator discipline.
|
||||
|
||||
@@ -383,6 +383,34 @@ De officiële audit vond geen publieke, gebiedsdekkende inland-bathymetrie voor
|
||||
Mol. Kust- en Noordzeeproducten vallen buiten de ruimtelijke scope. Waterinfo
|
||||
stations blijven puntmetingen en GRB-watergeometrie blijft tweedimensionaal.
|
||||
|
||||
## Regionale VMM overstromingsscenario's
|
||||
|
||||
`scripts/provision_regional_flood_hazards.py` breidt de beheerde
|
||||
VMM-overstromingsflow uit naar de goedgekeurde `kempen-transport-region`
|
||||
scope. De operator gebruikt de 28 persistente gemeente-Areas die door
|
||||
`provision_geographic_scope.py` zijn aangemaakt en roept per gemeente en per
|
||||
scenario uitsluitend de bestaande canonical API aan:
|
||||
|
||||
- `POST /api/v1/projects/{project_id}/datasets/flood-hazard/acquire`
|
||||
- `POST /api/v1/projects/{project_id}/datasets/{dataset_id}/raster/flood-hazard/select`
|
||||
|
||||
Deze gemeentepartities zijn bewust. Een volledig regionaal raster in een
|
||||
aanvraag zou de publieke WCS- en pixelgrenzen onnodig belasten. In de UI wordt
|
||||
een VMM-dataset alleen als overstromingslaag getoond voor het actieve
|
||||
werkgebied waaraan die dataset gekoppeld is. Zo blijft Mol bij Mol, Geel bij
|
||||
Geel, enzovoort.
|
||||
|
||||
De regionale operator ondersteunt `--dry-run`, `--members` en `--products`.
|
||||
Een volledige scope met alle twaalf scenario's plant 336 gecontroleerde
|
||||
acquisities. Herhaalruns gebruiken bestaande checksummed Datasets via de
|
||||
backend-cache zolang de requestidentiteit niet verandert.
|
||||
|
||||
Ook regionaal blijft de semantiek onveranderd: VMM-waterdiepte is een
|
||||
gemodelleerde maximale lokale diepte per kans- en klimaatscenario. GeoIntel kan
|
||||
oppervlakte, gemiddelde/P90/maximale diepte en de diepte-oppervlakte-integraal
|
||||
berekenen. Dat is geen gelijktijdig waterbergingsvolume, geen actuele
|
||||
waterstand en geen bathymetrie.
|
||||
|
||||
## Gebouwenregister
|
||||
|
||||
The governed operator `scripts/provision_buildings_addresses_register.py`
|
||||
|
||||
@@ -118,6 +118,15 @@ metadata records the 1 m native product, 5 m default analysis grid, EPSG:31370,
|
||||
derived browser views and are never authoritative. No raster binary is stored
|
||||
in PostgreSQL and no DHMV file is treated as water depth or volume.
|
||||
|
||||
VMM flood-hazard scenario outputs follow the same raster Dataset policy. The
|
||||
bounded acquisition service stores one normalized, compressed, Area-clipped
|
||||
GeoTIFF per scenario/request identity. `provision_regional_flood_hazards.py`
|
||||
does not create a parallel storage layout: every municipality/scenario result
|
||||
is an ordinary Dataset and DatasetVersion with WCS request hashes, tile request
|
||||
URLs, response/coverage/normalized checksums, EPSG:31370 bounds, scenario
|
||||
metadata and explicit unsupported-volume flags. Repeat runs reuse matching
|
||||
ready Datasets through the acquisition service cache.
|
||||
|
||||
BWK/Natura 2000 evidence lives under
|
||||
`storage/operator-evidence/bwk-natura2000-2025/mol/`. The `raw/` directory
|
||||
contains immutable WFS pages; the adjacent manifest records their URLs,
|
||||
|
||||
@@ -26,6 +26,7 @@
|
||||
- [x] Add bounded historical orthophoto acquisition for official 1971-2025 products with a map overlay and no current-GRB QA on old imagery.
|
||||
- [x] Add BWK/Natura 2000 through an explicit provider/operator contract.
|
||||
- [x] Expand BWK/Natura 2000 state 2025 from Mol to all 28 approved Kempen municipalities with partitioned source evidence.
|
||||
- [x] Add a regional VMM flood-hazard operator that provisions official scenario rasters per municipality Area through the canonical API.
|
||||
- [x] Add annual agricultural-use parcels through an explicit provider/operator contract.
|
||||
- [x] Extend the official 1778/1873/1969 historical buildings, water and roads series from Mol to the approved regional scope with partitioned source audits.
|
||||
- [x] Connect a drawn rectangle to bounded official orthophoto acquisition, local configured-YOLO detection and persisted GRB QA.
|
||||
@@ -72,6 +73,7 @@ pass live Mol validation before regional expansion.
|
||||
- [x] Integrate the separate public VMM fluvial/pluvial flood-hazard depth scenarios without presenting them as bathymetry or current water state.
|
||||
- [x] Persist scenario identity, source centimetres, normalized metres, WCS checksums, exact Area clipping and positive-depth coverage.
|
||||
- [x] Expose mapped inundation area, depth statistics and a clearly named maximum-depth area integral with a prohibition on calling it concurrent volume.
|
||||
- [x] Extend the VMM scenario workflow to the approved regional scope using 28 municipality partitions instead of one unsafe monolithic raster.
|
||||
- [ ] Define waterbody linkage, surface elevation, bottom elevation and uncertainty propagation before adding any volume metric.
|
||||
- [ ] Validate coverage gaps and prohibit extrapolation outside measured/profiled waterbodies.
|
||||
- [ ] Add independent GIS review and golden-volume fixtures before exposing the result to users or Ollama.
|
||||
|
||||
@@ -518,6 +518,14 @@ pattern and a rectangle returns scenario-bound hectare/depth metrics. Raster
|
||||
GeoJSON export stays disabled. The UI never labels the maximum-depth area
|
||||
integral as current, permanent or concurrent water volume.
|
||||
|
||||
Regional VMM provisioning creates one scenario raster per municipality Area.
|
||||
The explorer therefore shows only the flood scenarios whose `area_id` matches
|
||||
the active work area. This avoids presenting a Mol scenario while the map is
|
||||
focused on another municipality. The region-wide Area remains the navigation
|
||||
context; municipality Areas are the analysis scope for flood rasters because
|
||||
the public WCS and raster cell limits make one monolithic Kempen raster
|
||||
operationally unsafe.
|
||||
|
||||
## Useful repository scripts
|
||||
|
||||
- `bash scripts/frontend_install.sh`
|
||||
|
||||
@@ -1579,6 +1579,40 @@ docker exec geointel python /app/scripts/provision_mol_flood_hazards.py --produc
|
||||
docker exec geointel python /app/scripts/provision_mol_flood_hazards.py --resolution-m 5 --force
|
||||
```
|
||||
|
||||
## Regional VMM flood-hazard scenarios
|
||||
|
||||
Provision governed VMM flood-depth scenarios for every persisted municipality
|
||||
Area in an approved scope:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_regional_flood_hazards.py \
|
||||
--scope kempen-transport-region --dry-run
|
||||
|
||||
docker exec geointel python /app/scripts/provision_regional_flood_hazards.py \
|
||||
--scope kempen-transport-region
|
||||
```
|
||||
|
||||
The command requires `provision_geographic_scope.py --scope
|
||||
kempen-transport-region` to have created the regional project and member
|
||||
Areas. It uses only canonical API calls, validates the backend twelve-product
|
||||
registry and runs a full-Area selection smoke after each acquisition. Existing
|
||||
scenario Datasets are reused unless `--force` is supplied.
|
||||
|
||||
Useful bounded runs while validating source availability:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_regional_flood_hazards.py \
|
||||
--members Mol --products pluviaal_current_t100
|
||||
|
||||
docker exec geointel python /app/scripts/provision_regional_flood_hazards.py \
|
||||
--members Mol,Geel --products pluviaal_current_t10,pluviaal_current_t100
|
||||
```
|
||||
|
||||
A complete Kempen run plans 28 municipalities times 12 scenario rasters. It can
|
||||
take a long time because every VMM WCS tile is bounded, rate-limited and
|
||||
validated. This is expected operator work; the app never fetches these rasters
|
||||
on page load or map click.
|
||||
|
||||
## Tower deployment
|
||||
|
||||
Push the local branch to Gitea, then rebuild the Unraid/Tower Docker runtime:
|
||||
|
||||
@@ -0,0 +1,322 @@
|
||||
"""Provision governed VMM flood-depth scenarios for an approved GeoIntel scope.
|
||||
|
||||
The command coordinates canonical API calls only. It does not fetch rasters
|
||||
directly, does not write database rows directly and does not run implicitly at
|
||||
application startup.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterable, Sequence
|
||||
|
||||
import requests
|
||||
|
||||
from geographic_scopes import GEOGRAPHIC_SCOPES, KEMPEN_TRANSPORT_REGION_SCOPE, ScopeMember
|
||||
|
||||
|
||||
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||
DEFAULT_SCOPE = KEMPEN_TRANSPORT_REGION_SCOPE.key
|
||||
PRODUCTS = tuple(
|
||||
f"{mechanism}_{climate}_t{period}"
|
||||
for mechanism in ("pluviaal", "fluviaal")
|
||||
for climate in ("current", "future_2050")
|
||||
for period in (10, 100, 1000)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolvedArea:
|
||||
id: str
|
||||
name: str
|
||||
member_name: str
|
||||
nis_code: str
|
||||
geometry: dict[str, Any]
|
||||
|
||||
|
||||
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Provision official VMM flood-depth scenarios for an approved scope.")
|
||||
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
|
||||
parser.add_argument("--scope", choices=sorted(GEOGRAPHIC_SCOPES), default=DEFAULT_SCOPE)
|
||||
parser.add_argument("--products", default=",".join(PRODUCTS), help="Comma-separated governed product keys.")
|
||||
parser.add_argument(
|
||||
"--members",
|
||||
default="",
|
||||
help="Optional comma-separated municipality names or NIS codes. Empty means every member in the selected scope.",
|
||||
)
|
||||
parser.add_argument("--resolution-m", type=float, default=5.0)
|
||||
parser.add_argument("--timeout", type=int, default=1800)
|
||||
parser.add_argument("--force", action="store_true")
|
||||
parser.add_argument("--stop-on-error", action="store_true")
|
||||
parser.add_argument("--dry-run", action="store_true", help="Resolve scope/products only; do not call acquisition endpoints.")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def unwrap(response: requests.Response) -> Any:
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
if not isinstance(payload, dict) or "data" not in payload:
|
||||
raise RuntimeError(f"Non-canonical API response from {response.url}")
|
||||
return payload["data"]
|
||||
|
||||
|
||||
def coordinates(geometry: dict[str, Any]) -> Iterable[tuple[float, float]]:
|
||||
def walk(value: Any):
|
||||
if isinstance(value, list) and len(value) >= 2 and all(isinstance(item, (int, float)) for item in value[:2]):
|
||||
yield float(value[0]), float(value[1])
|
||||
return
|
||||
if isinstance(value, list):
|
||||
for child in value:
|
||||
yield from walk(child)
|
||||
|
||||
yield from walk(geometry.get("coordinates", []))
|
||||
|
||||
|
||||
def geometry_bbox(geometry: dict[str, Any]) -> dict[str, float | str]:
|
||||
points = list(coordinates(geometry))
|
||||
if not points:
|
||||
raise RuntimeError("Persisted Area geometry contains no coordinates")
|
||||
xs = [point[0] for point in points]
|
||||
ys = [point[1] for point in points]
|
||||
return {"min_x": min(xs), "min_y": min(ys), "max_x": max(xs), "max_y": max(ys), "crs": "EPSG:4326"}
|
||||
|
||||
|
||||
def requested_products(raw_products: str, registry_keys: set[str]) -> list[str]:
|
||||
products = [item.strip().lower() for item in raw_products.split(",") if item.strip()]
|
||||
invalid = sorted(set(products) - registry_keys)
|
||||
if invalid:
|
||||
raise RuntimeError(f"Unsupported flood-hazard product keys: {', '.join(invalid)}")
|
||||
if not products:
|
||||
raise RuntimeError("At least one flood-hazard product key is required")
|
||||
return products
|
||||
|
||||
|
||||
def requested_members(raw_members: str, members: tuple[ScopeMember, ...]) -> list[ScopeMember]:
|
||||
if not raw_members.strip():
|
||||
return list(members)
|
||||
index: dict[str, ScopeMember] = {}
|
||||
for member in members:
|
||||
index[member.name.casefold()] = member
|
||||
index[member.nis_code] = member
|
||||
selected: list[ScopeMember] = []
|
||||
unknown: list[str] = []
|
||||
for token in [item.strip() for item in raw_members.split(",") if item.strip()]:
|
||||
member = index.get(token.casefold()) or index.get(token)
|
||||
if member is None:
|
||||
unknown.append(token)
|
||||
elif member not in selected:
|
||||
selected.append(member)
|
||||
if unknown:
|
||||
raise RuntimeError(f"Unknown scope members: {', '.join(unknown)}")
|
||||
if not selected:
|
||||
raise RuntimeError("At least one scope member is required")
|
||||
return selected
|
||||
|
||||
|
||||
def resolve_project(session: requests.Session, base_url: str, project_name: str) -> dict[str, Any]:
|
||||
projects = unwrap(session.get(f"{base_url}/api/v1/projects", params={"limit": 200, "offset": 0}, timeout=60))["items"]
|
||||
project = next((item for item in projects if item["name"] == project_name), None)
|
||||
if project is None:
|
||||
raise RuntimeError(f"Project {project_name!r} was not found. Run provision_geographic_scope.py first.")
|
||||
return project
|
||||
|
||||
|
||||
def resolve_areas(
|
||||
session: requests.Session,
|
||||
base_url: str,
|
||||
project_id: str,
|
||||
members: Sequence[ScopeMember],
|
||||
) -> list[ResolvedArea]:
|
||||
areas = unwrap(
|
||||
session.get(
|
||||
f"{base_url}/api/v1/projects/{project_id}/areas",
|
||||
params={"limit": 500, "offset": 0},
|
||||
timeout=60,
|
||||
)
|
||||
)["items"]
|
||||
resolved: list[ResolvedArea] = []
|
||||
missing: list[str] = []
|
||||
for member in members:
|
||||
name_token = member.name.casefold()
|
||||
nis_token = member.nis_code
|
||||
area = next(
|
||||
(
|
||||
item
|
||||
for item in areas
|
||||
if name_token in str(item.get("name", "")).casefold()
|
||||
or nis_token in str(item.get("name", ""))
|
||||
or nis_token in str(item.get("source_metadata", ""))
|
||||
),
|
||||
None,
|
||||
)
|
||||
if area is None:
|
||||
missing.append(f"{member.name} ({member.nis_code})")
|
||||
continue
|
||||
if not area.get("geometry"):
|
||||
raise RuntimeError(f"Area {area.get('name')!r} has no geometry in the canonical API response")
|
||||
resolved.append(
|
||||
ResolvedArea(
|
||||
id=area["id"],
|
||||
name=area["name"],
|
||||
member_name=member.name,
|
||||
nis_code=member.nis_code,
|
||||
geometry=area["geometry"],
|
||||
)
|
||||
)
|
||||
if missing:
|
||||
raise RuntimeError(
|
||||
"Persisted municipality Areas are missing: "
|
||||
+ ", ".join(missing)
|
||||
+ ". Run provision_geographic_scope.py for the selected scope first."
|
||||
)
|
||||
return resolved
|
||||
|
||||
|
||||
def validate_registry(session: requests.Session, base_url: str, project_id: str) -> set[str]:
|
||||
registry = unwrap(session.get(f"{base_url}/api/v1/projects/{project_id}/datasets/flood-hazard/products", timeout=60))["items"]
|
||||
registry_keys = {item["key"] for item in registry}
|
||||
if registry_keys != set(PRODUCTS):
|
||||
raise RuntimeError("Backend flood-hazard registry does not expose the governed twelve-product set")
|
||||
return registry_keys
|
||||
|
||||
|
||||
def acquire_one(
|
||||
session: requests.Session,
|
||||
base_url: str,
|
||||
project_id: str,
|
||||
area: ResolvedArea,
|
||||
product_key: str,
|
||||
resolution_m: float,
|
||||
timeout: int,
|
||||
force: bool,
|
||||
) -> dict[str, Any]:
|
||||
bbox = geometry_bbox(area.geometry)
|
||||
job = unwrap(
|
||||
session.post(
|
||||
f"{base_url}/api/v1/projects/{project_id}/datasets/flood-hazard/acquire",
|
||||
json={
|
||||
"bbox": bbox,
|
||||
"area_id": area.id,
|
||||
"product_key": product_key,
|
||||
"resolution_m": resolution_m,
|
||||
"force_refresh": force,
|
||||
},
|
||||
timeout=timeout,
|
||||
)
|
||||
)
|
||||
if job.get("status") != "success" or not job.get("output_dataset_id"):
|
||||
raise RuntimeError(f"Flood-hazard acquisition failed for {area.member_name} {product_key}: {job.get('error_message') or job}")
|
||||
analysis = unwrap(
|
||||
session.post(
|
||||
f"{base_url}/api/v1/projects/{project_id}/datasets/{job['output_dataset_id']}/raster/flood-hazard/select",
|
||||
json={"bbox": bbox, "area_id": area.id},
|
||||
timeout=timeout,
|
||||
)
|
||||
)
|
||||
unsupported = set(analysis.get("unsupported_metrics", []))
|
||||
required_unsupported = {"bathymetry_depth_m", "permanent_water_volume_m3", "concurrent_flood_volume_m3"}
|
||||
if required_unsupported - unsupported:
|
||||
raise RuntimeError("Flood-hazard contract must keep bathymetry and definitive water volumes unavailable")
|
||||
return {
|
||||
"member_name": area.member_name,
|
||||
"nis_code": area.nis_code,
|
||||
"area_id": area.id,
|
||||
"area_name": area.name,
|
||||
"product_key": product_key,
|
||||
"dataset_id": job["output_dataset_id"],
|
||||
"reused": bool((job.get("result_json") or {}).get("reused")),
|
||||
"resolution_m": analysis["resolution_m"],
|
||||
"selected_cell_count": analysis["selected_cell_count"],
|
||||
"inundated_cell_count": analysis["inundated_cell_count"],
|
||||
"metrics": analysis["summary"]["metrics"],
|
||||
}
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
args = parse_args(argv)
|
||||
scope = GEOGRAPHIC_SCOPES[args.scope]
|
||||
base_url = args.base_url.rstrip("/")
|
||||
selected_members = requested_members(args.members, scope.members)
|
||||
|
||||
session = requests.Session()
|
||||
session.headers.update({"User-Agent": "GeoIntel-Regional-VMM-Flood-Hazard-Operator/1.0"})
|
||||
project = resolve_project(session, base_url, scope.project_name)
|
||||
registry_keys = validate_registry(session, base_url, project["id"])
|
||||
products = requested_products(args.products, registry_keys)
|
||||
areas = resolve_areas(session, base_url, project["id"], selected_members)
|
||||
|
||||
if args.dry_run:
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "dry_run",
|
||||
"scope": scope.key,
|
||||
"project_id": project["id"],
|
||||
"member_count": len(areas),
|
||||
"product_count": len(products),
|
||||
"planned_acquisitions": len(areas) * len(products),
|
||||
"members": [{"name": area.member_name, "nis_code": area.nis_code, "area_id": area.id} for area in areas],
|
||||
"products": products,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
results: list[dict[str, Any]] = []
|
||||
failures: list[dict[str, Any]] = []
|
||||
for area in areas:
|
||||
for product_key in products:
|
||||
try:
|
||||
results.append(
|
||||
acquire_one(
|
||||
session,
|
||||
base_url,
|
||||
project["id"],
|
||||
area,
|
||||
product_key,
|
||||
args.resolution_m,
|
||||
args.timeout,
|
||||
args.force,
|
||||
)
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - operator summary should retain every failed member/product.
|
||||
failure = {
|
||||
"member_name": area.member_name,
|
||||
"nis_code": area.nis_code,
|
||||
"area_id": area.id,
|
||||
"product_key": product_key,
|
||||
"error": str(exc),
|
||||
}
|
||||
failures.append(failure)
|
||||
print(json.dumps({"status": "failed_item", **failure}, ensure_ascii=False), file=sys.stderr)
|
||||
if args.stop_on_error:
|
||||
raise
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "ok" if not failures else "partial",
|
||||
"scope": scope.key,
|
||||
"project_id": project["id"],
|
||||
"member_count": len(areas),
|
||||
"product_count": len(products),
|
||||
"completed_count": len(results),
|
||||
"failure_count": len(failures),
|
||||
"products": results,
|
||||
"failures": failures,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0 if not failures else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -55,6 +55,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_buildings_addresses_register.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_dhmv.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_flood_hazards.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_flood_hazards.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_timeseries.py
|
||||
${PYTHON_BIN} -m py_compile scripts/geographic_scopes.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_geographic_scope.py
|
||||
|
||||
Reference in New Issue
Block a user