338 lines
12 KiB
Python
338 lines
12 KiB
Python
"""Provision governed DHMV II rasters for an approved GeoIntel scope.
|
|
|
|
The command coordinates canonical API calls only. It does not fetch rasters
|
|
directly, write database rows directly or run implicitly at startup.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
from dataclasses import dataclass
|
|
from typing import Any, Iterable, Sequence
|
|
|
|
import requests
|
|
|
|
from geographic_scopes import GEOGRAPHIC_SCOPES, KEMPEN_TRANSPORT_REGION_SCOPE, ScopeMember
|
|
|
|
|
|
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
|
DEFAULT_SCOPE = KEMPEN_TRANSPORT_REGION_SCOPE.key
|
|
PRODUCTS = ("dtm_1m", "dsm_1m")
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ResolvedArea:
|
|
id: str
|
|
name: str
|
|
member_name: str
|
|
nis_code: str
|
|
geometry: dict[str, Any]
|
|
|
|
|
|
def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(description="Provision official DHMV II rasters for an approved scope.")
|
|
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
|
|
parser.add_argument("--scope", choices=sorted(GEOGRAPHIC_SCOPES), default=DEFAULT_SCOPE)
|
|
parser.add_argument("--products", default=",".join(PRODUCTS), help="Comma-separated governed product keys.")
|
|
parser.add_argument(
|
|
"--members",
|
|
default="",
|
|
help="Optional comma-separated municipality names or NIS codes. Empty means every scope member.",
|
|
)
|
|
parser.add_argument("--resolution-m", type=float, default=5.0)
|
|
parser.add_argument("--timeout", type=int, default=1800)
|
|
parser.add_argument("--force", action="store_true")
|
|
parser.add_argument("--stop-on-error", action="store_true")
|
|
parser.add_argument("--dry-run", action="store_true", help="Resolve scope/products only; do not acquire data.")
|
|
return parser.parse_args(argv)
|
|
|
|
|
|
def unwrap(response: requests.Response) -> Any:
|
|
response.raise_for_status()
|
|
payload = response.json()
|
|
if not isinstance(payload, dict) or "data" not in payload:
|
|
raise RuntimeError(f"Non-canonical API response from {response.url}")
|
|
return payload["data"]
|
|
|
|
|
|
def coordinates(geometry: dict[str, Any]) -> Iterable[tuple[float, float]]:
|
|
def walk(value: Any):
|
|
if isinstance(value, list) and len(value) >= 2 and all(isinstance(item, (int, float)) for item in value[:2]):
|
|
yield float(value[0]), float(value[1])
|
|
return
|
|
if isinstance(value, list):
|
|
for child in value:
|
|
yield from walk(child)
|
|
|
|
yield from walk(geometry.get("coordinates", []))
|
|
|
|
|
|
def geometry_bbox(geometry: dict[str, Any]) -> dict[str, float | str]:
|
|
points = list(coordinates(geometry))
|
|
if not points:
|
|
raise RuntimeError("Persisted Area geometry contains no coordinates")
|
|
xs = [point[0] for point in points]
|
|
ys = [point[1] for point in points]
|
|
return {"min_x": min(xs), "min_y": min(ys), "max_x": max(xs), "max_y": max(ys), "crs": "EPSG:4326"}
|
|
|
|
|
|
def requested_products(raw_products: str, registry_keys: set[str]) -> list[str]:
|
|
products = [item.strip().lower() for item in raw_products.split(",") if item.strip()]
|
|
invalid = sorted(set(products) - registry_keys)
|
|
if invalid:
|
|
raise RuntimeError(f"Unsupported DHMV product keys: {', '.join(invalid)}")
|
|
if not products:
|
|
raise RuntimeError("At least one DHMV product key is required")
|
|
return products
|
|
|
|
|
|
def requested_members(raw_members: str, members: tuple[ScopeMember, ...]) -> list[ScopeMember]:
|
|
if not raw_members.strip():
|
|
return list(members)
|
|
index: dict[str, ScopeMember] = {}
|
|
for member in members:
|
|
index[member.name.casefold()] = member
|
|
index[member.nis_code] = member
|
|
selected: list[ScopeMember] = []
|
|
unknown: list[str] = []
|
|
for token in [item.strip() for item in raw_members.split(",") if item.strip()]:
|
|
member = index.get(token.casefold()) or index.get(token)
|
|
if member is None:
|
|
unknown.append(token)
|
|
elif member not in selected:
|
|
selected.append(member)
|
|
if unknown:
|
|
raise RuntimeError(f"Unknown scope members: {', '.join(unknown)}")
|
|
if not selected:
|
|
raise RuntimeError("At least one scope member is required")
|
|
return selected
|
|
|
|
|
|
def resolve_project(session: requests.Session, base_url: str, project_name: str) -> dict[str, Any]:
|
|
projects = unwrap(session.get(f"{base_url}/api/v1/projects", params={"limit": 200, "offset": 0}, timeout=60))["items"]
|
|
project = next((item for item in projects if item["name"] == project_name), None)
|
|
if project is None:
|
|
raise RuntimeError(f"Project {project_name!r} was not found. Run provision_geographic_scope.py first.")
|
|
return project
|
|
|
|
|
|
def resolve_areas(
|
|
session: requests.Session,
|
|
base_url: str,
|
|
project_id: str,
|
|
members: Sequence[ScopeMember],
|
|
) -> list[ResolvedArea]:
|
|
areas: list[dict[str, Any]] = []
|
|
limit = 200
|
|
offset = 0
|
|
while True:
|
|
page = unwrap(
|
|
session.get(
|
|
f"{base_url}/api/v1/projects/{project_id}/areas",
|
|
params={"limit": limit, "offset": offset},
|
|
timeout=60,
|
|
)
|
|
)
|
|
page_items = list(page["items"])
|
|
areas.extend(page_items)
|
|
total = int(page.get("total", len(areas)))
|
|
if len(areas) >= total or len(page_items) < limit:
|
|
break
|
|
offset += limit
|
|
|
|
resolved: list[ResolvedArea] = []
|
|
missing: list[str] = []
|
|
for member in members:
|
|
area = next(
|
|
(
|
|
item
|
|
for item in areas
|
|
if member.name.casefold() in str(item.get("name", "")).casefold()
|
|
or member.nis_code in str(item.get("name", ""))
|
|
or member.nis_code in str(item.get("source_metadata", ""))
|
|
),
|
|
None,
|
|
)
|
|
if area is None:
|
|
missing.append(f"{member.name} ({member.nis_code})")
|
|
continue
|
|
if not area.get("geometry"):
|
|
raise RuntimeError(f"Area {area.get('name')!r} has no geometry in the canonical API response")
|
|
resolved.append(
|
|
ResolvedArea(
|
|
id=area["id"],
|
|
name=area["name"],
|
|
member_name=member.name,
|
|
nis_code=member.nis_code,
|
|
geometry=area["geometry"],
|
|
)
|
|
)
|
|
if missing:
|
|
raise RuntimeError(
|
|
"Persisted municipality Areas are missing: "
|
|
+ ", ".join(missing)
|
|
+ ". Run provision_geographic_scope.py for the selected scope first."
|
|
)
|
|
return resolved
|
|
|
|
|
|
def validate_registry(session: requests.Session, base_url: str, project_id: str) -> set[str]:
|
|
registry = unwrap(session.get(f"{base_url}/api/v1/projects/{project_id}/datasets/dhmv/products", timeout=60))["items"]
|
|
registry_keys = {item["key"] for item in registry}
|
|
if registry_keys != set(PRODUCTS):
|
|
raise RuntimeError("Backend DHMV registry does not expose the governed DTM/DSM product set")
|
|
return registry_keys
|
|
|
|
|
|
def acquire_one(
|
|
session: requests.Session,
|
|
base_url: str,
|
|
project_id: str,
|
|
area: ResolvedArea,
|
|
product_key: str,
|
|
resolution_m: float,
|
|
timeout: int,
|
|
force: bool,
|
|
) -> dict[str, Any]:
|
|
bbox = geometry_bbox(area.geometry)
|
|
job = unwrap(
|
|
session.post(
|
|
f"{base_url}/api/v1/projects/{project_id}/datasets/dhmv/acquire",
|
|
json={
|
|
"bbox": bbox,
|
|
"area_id": area.id,
|
|
"product_key": product_key,
|
|
"resolution_m": resolution_m,
|
|
"force_refresh": force,
|
|
},
|
|
timeout=timeout,
|
|
)
|
|
)
|
|
if job.get("status") != "success" or not job.get("output_dataset_id"):
|
|
raise RuntimeError(f"DHMV acquisition failed for {area.member_name} {product_key}: {job.get('error_message') or job}")
|
|
analysis = unwrap(
|
|
session.post(
|
|
f"{base_url}/api/v1/projects/{project_id}/datasets/{job['output_dataset_id']}/raster/terrain/select",
|
|
json={"bbox": bbox, "area_id": area.id},
|
|
timeout=timeout,
|
|
)
|
|
)
|
|
if sorted(analysis.get("unsupported_metrics", [])) != ["water_depth_m", "water_volume_m3"]:
|
|
raise RuntimeError("DHMV terrain contract must explicitly keep water depth and volume unavailable")
|
|
return {
|
|
"member_name": area.member_name,
|
|
"nis_code": area.nis_code,
|
|
"area_id": area.id,
|
|
"area_name": area.name,
|
|
"product_key": product_key,
|
|
"dataset_id": job["output_dataset_id"],
|
|
"reused": bool((job.get("result_json") or {}).get("reused")),
|
|
"resolution_m": analysis["resolution_m"],
|
|
"sample_count": analysis["sample_count"],
|
|
"coverage_ratio": analysis["coverage_ratio"],
|
|
"metrics": analysis["summary"]["metrics"],
|
|
}
|
|
|
|
|
|
def main(argv: Sequence[str] | None = None) -> int:
|
|
args = parse_args(argv)
|
|
scope = GEOGRAPHIC_SCOPES[args.scope]
|
|
base_url = args.base_url.rstrip("/")
|
|
selected_members = requested_members(args.members, scope.members)
|
|
|
|
session = requests.Session()
|
|
session.headers.update({"User-Agent": "GeoIntel-Regional-DHMV-Operator/1.0"})
|
|
project = resolve_project(session, base_url, scope.project_name)
|
|
registry_keys = validate_registry(session, base_url, project["id"])
|
|
products = requested_products(args.products, registry_keys)
|
|
areas = resolve_areas(session, base_url, project["id"], selected_members)
|
|
|
|
if args.dry_run:
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": "dry_run",
|
|
"scope": scope.key,
|
|
"project_id": project["id"],
|
|
"member_count": len(areas),
|
|
"product_count": len(products),
|
|
"planned_acquisitions": len(areas) * len(products),
|
|
"members": [{"name": area.member_name, "nis_code": area.nis_code, "area_id": area.id} for area in areas],
|
|
"products": products,
|
|
},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
)
|
|
return 0
|
|
|
|
results: list[dict[str, Any]] = []
|
|
failures: list[dict[str, Any]] = []
|
|
for area in areas:
|
|
for product_key in products:
|
|
try:
|
|
result = acquire_one(
|
|
session,
|
|
base_url,
|
|
project["id"],
|
|
area,
|
|
product_key,
|
|
args.resolution_m,
|
|
args.timeout,
|
|
args.force,
|
|
)
|
|
results.append(result)
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": "completed_item",
|
|
"completed": len(results),
|
|
"planned": len(areas) * len(products),
|
|
"member_name": area.member_name,
|
|
"product_key": product_key,
|
|
"dataset_id": result["dataset_id"],
|
|
"reused": result["reused"],
|
|
},
|
|
ensure_ascii=False,
|
|
),
|
|
flush=True,
|
|
)
|
|
except Exception as exc: # noqa: BLE001 - retain every failed member/product in the operator summary.
|
|
failure = {
|
|
"member_name": area.member_name,
|
|
"nis_code": area.nis_code,
|
|
"area_id": area.id,
|
|
"product_key": product_key,
|
|
"error": str(exc),
|
|
}
|
|
failures.append(failure)
|
|
print(json.dumps({"status": "failed_item", **failure}, ensure_ascii=False), file=sys.stderr, flush=True)
|
|
if args.stop_on_error:
|
|
raise
|
|
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"status": "ok" if not failures else "partial",
|
|
"scope": scope.key,
|
|
"project_id": project["id"],
|
|
"member_count": len(areas),
|
|
"product_count": len(products),
|
|
"completed_count": len(results),
|
|
"failure_count": len(failures),
|
|
"products": results,
|
|
"failures": failures,
|
|
},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
)
|
|
return 0 if not failures else 2
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|