Add Flemish bathymetry partition workflow
This commit is contained in:
@@ -1909,3 +1909,32 @@ python scripts/provision_mol_bathymetry_profiles.py \
|
||||
|
||||
The output is a point dataset with historical evidence. It is not a continuous
|
||||
water-bottom raster and cannot calculate current water volume.
|
||||
|
||||
## Flanders VHA bathymetry partitions
|
||||
|
||||
Create the complete official Flemish land scope from the current VRBG
|
||||
municipality collection:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_flanders_geographic_scope.py
|
||||
```
|
||||
|
||||
Acquire VHA profile partitions with atomic resume evidence:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/provision_flanders_bathymetry_profiles.py
|
||||
```
|
||||
|
||||
Use `--members Mol Geel` or `--max-partitions 5` only for a partial operational
|
||||
check. Partial runs do not activate regional coverage. The complete run
|
||||
finalizes only when every official municipality has either one ready Dataset
|
||||
or an explicit zero-profile source result.
|
||||
|
||||
Probe the official-metadata MDK WCS safely:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/probe_mdk_bathymetry.py
|
||||
```
|
||||
|
||||
This performs only `GetCapabilities`, keeps strict TLS verification enabled
|
||||
and returns exit code `2` for an honest non-ready source.
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Run the safe MDK WCS readiness probe through the canonical GeoIntel API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||
DEFAULT_PROJECT_NAME = "Kempen Regional Workbench"
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Probe MDK bathymetry WCS readiness without downloading coverage.")
|
||||
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
|
||||
parser.add_argument("--project-name", default=DEFAULT_PROJECT_NAME)
|
||||
parser.add_argument("--timeout", type=int, default=60)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def unwrap(response: requests.Response):
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
if not isinstance(payload, dict) or "data" not in payload:
|
||||
raise RuntimeError(f"Non-canonical API response from {response.url}")
|
||||
return payload["data"]
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
base_url = args.base_url.rstrip("/")
|
||||
session = requests.Session()
|
||||
projects = unwrap(
|
||||
session.get(
|
||||
f"{base_url}/api/v1/projects",
|
||||
params={"name": args.project_name, "limit": 1},
|
||||
timeout=args.timeout,
|
||||
)
|
||||
)["items"]
|
||||
if not projects:
|
||||
raise RuntimeError(f"Project {args.project_name!r} was not found")
|
||||
result = unwrap(
|
||||
session.get(
|
||||
f"{base_url}/api/v1/projects/{projects[0]['id']}/datasets/"
|
||||
"bathymetry/sources/mdk_bcp_bathymetry/readiness",
|
||||
timeout=args.timeout,
|
||||
)
|
||||
)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
return 0 if result["status"] == "reachable" else 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,400 @@
|
||||
"""Provision VHA cross-section profiles for every persisted Flemish municipality.
|
||||
|
||||
The operator is resumable and calls only canonical GeoIntel API endpoints.
|
||||
Regional completeness is finalized server-side only when every municipality
|
||||
has either one ready profile Dataset or an explicit no-profile source result.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||
DEFAULT_PROJECT_NAME = "Flanders Regional Workbench"
|
||||
DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/bathymetry/vha-flanders/manifest.json")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Provision partitioned VHA profiles for Flanders.")
|
||||
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
|
||||
parser.add_argument("--project-name", default=DEFAULT_PROJECT_NAME)
|
||||
parser.add_argument(
|
||||
"--manifest-path",
|
||||
type=Path,
|
||||
default=Path(os.environ.get("GEOINTEL_VHA_FLANDERS_MANIFEST", DEFAULT_MANIFEST_PATH)),
|
||||
)
|
||||
parser.add_argument("--timeout", type=int, default=900)
|
||||
parser.add_argument("--force", action="store_true")
|
||||
parser.add_argument("--continue-on-error", action="store_true")
|
||||
parser.add_argument("--members", nargs="*", default=[])
|
||||
parser.add_argument("--max-partitions", type=int, default=0)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def unwrap(response: requests.Response) -> Any:
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError as exc:
|
||||
raise RuntimeError(f"GeoIntel returned non-JSON ({response.status_code}): {response.text[:300]}") from exc
|
||||
if not response.ok:
|
||||
error_code = payload.get("error") if isinstance(payload, dict) else None
|
||||
message = payload.get("message") if isinstance(payload, dict) else None
|
||||
raise RuntimeError(f"{error_code or response.status_code}: {message or response.text[:300]}")
|
||||
if not isinstance(payload, dict) or "data" not in payload:
|
||||
raise RuntimeError(f"Non-canonical API response from {response.url}")
|
||||
return payload["data"]
|
||||
|
||||
|
||||
def list_paginated(session: requests.Session, url: str, *, timeout: int) -> list[dict[str, Any]]:
|
||||
items: list[dict[str, Any]] = []
|
||||
offset = 0
|
||||
total: int | None = None
|
||||
while True:
|
||||
page = unwrap(session.get(url, params={"limit": 200, "offset": offset}, timeout=timeout))
|
||||
page_items = list(page.get("items") or [])
|
||||
items.extend(page_items)
|
||||
total = int(page.get("total") or 0) if total is None else total
|
||||
if not page_items or len(items) >= total:
|
||||
break
|
||||
offset += len(page_items)
|
||||
if total is not None and len(items) != total:
|
||||
raise RuntimeError(f"Paginated API returned {len(items)} of {total} records for {url}")
|
||||
return items
|
||||
|
||||
|
||||
def coordinates(geometry: dict[str, Any]) -> Iterable[tuple[float, float]]:
|
||||
def walk(value: Any):
|
||||
if isinstance(value, list) and len(value) >= 2 and all(isinstance(item, (int, float)) for item in value[:2]):
|
||||
yield float(value[0]), float(value[1])
|
||||
return
|
||||
if isinstance(value, list):
|
||||
for child in value:
|
||||
yield from walk(child)
|
||||
|
||||
yield from walk(geometry.get("coordinates", []))
|
||||
|
||||
|
||||
def geometry_bbox(geometry: dict[str, Any]) -> dict[str, float | str]:
|
||||
points = list(coordinates(geometry))
|
||||
if not points:
|
||||
raise RuntimeError("Persisted municipality Area contains no coordinates")
|
||||
xs = [point[0] for point in points]
|
||||
ys = [point[1] for point in points]
|
||||
return {
|
||||
"min_x": min(xs),
|
||||
"min_y": min(ys),
|
||||
"max_x": max(xs),
|
||||
"max_y": max(ys),
|
||||
"crs": "EPSG:4326",
|
||||
}
|
||||
|
||||
|
||||
def area_identity(area: dict[str, Any]) -> str:
|
||||
canonical = {
|
||||
"id": area.get("id"),
|
||||
"name": area.get("name"),
|
||||
"geometry": area.get("geometry"),
|
||||
}
|
||||
return hashlib.sha256(
|
||||
json.dumps(canonical, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||
).hexdigest()
|
||||
|
||||
|
||||
def write_manifest(path: Path, payload: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary = path.with_suffix(path.suffix + ".partial")
|
||||
temporary.write_text(
|
||||
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def load_manifest(path: Path) -> dict[str, Any]:
|
||||
if not path.is_file():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def source_error(response: requests.Response) -> tuple[str | None, str]:
|
||||
try:
|
||||
payload = response.json()
|
||||
except ValueError:
|
||||
return None, response.text[:300]
|
||||
if not isinstance(payload, dict):
|
||||
return None, response.text[:300]
|
||||
return (
|
||||
str(payload.get("error")) if payload.get("error") else None,
|
||||
str(payload.get("message") or response.text[:300]),
|
||||
)
|
||||
|
||||
|
||||
def acquire_partition(
|
||||
session: requests.Session,
|
||||
*,
|
||||
base_url: str,
|
||||
project_id: str,
|
||||
area: dict[str, Any],
|
||||
timeout: int,
|
||||
force: bool,
|
||||
) -> dict[str, Any]:
|
||||
bbox = geometry_bbox(area["geometry"])
|
||||
response = session.post(
|
||||
f"{base_url}/api/v1/projects/{project_id}/datasets/bathymetry/profiles/acquire",
|
||||
json={"bbox": bbox, "area_id": area["id"], "force_refresh": force},
|
||||
timeout=timeout,
|
||||
)
|
||||
if response.status_code == 404:
|
||||
code, message = source_error(response)
|
||||
if code == "BATHYMETRY_NO_PROFILES":
|
||||
return {
|
||||
"status": "no_profiles",
|
||||
"area_id": area["id"],
|
||||
"area_name": area["name"],
|
||||
"area_identity_sha256": area_identity(area),
|
||||
"bbox": bbox,
|
||||
"message": message,
|
||||
"completed_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
data = unwrap(response)
|
||||
if data.get("status") != "success" or not data.get("output_dataset_id"):
|
||||
raise RuntimeError(f"VHA acquisition failed for {area['name']}: {data.get('error_message') or data}")
|
||||
result = data.get("result_json") or {}
|
||||
return {
|
||||
"status": "complete",
|
||||
"area_id": area["id"],
|
||||
"area_name": area["name"],
|
||||
"area_identity_sha256": area_identity(area),
|
||||
"bbox": bbox,
|
||||
"dataset_id": data["output_dataset_id"],
|
||||
"reused": bool(result.get("reused")),
|
||||
"profile_count": int(result.get("profile_count") or 0),
|
||||
"document_count": int(result.get("document_count") or 0),
|
||||
"structured_depth_count": int(result.get("structured_depth_count") or 0),
|
||||
"measurement_date_min": result.get("measurement_date_min"),
|
||||
"measurement_date_max": result.get("measurement_date_max"),
|
||||
"completed_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
|
||||
|
||||
def manifest_identity(
|
||||
*,
|
||||
project_id: str,
|
||||
partitions: list[dict[str, Any]],
|
||||
) -> tuple[dict[str, Any], str]:
|
||||
identity = {
|
||||
"schema_version": 1,
|
||||
"source": "vmm_vha_bathymetry_profiles",
|
||||
"partition_scope_key": "flanders",
|
||||
"project_id": project_id,
|
||||
"partitions": [
|
||||
{
|
||||
key: partition.get(key)
|
||||
for key in (
|
||||
"area_id",
|
||||
"area_name",
|
||||
"area_identity_sha256",
|
||||
"status",
|
||||
"dataset_id",
|
||||
"profile_count",
|
||||
"document_count",
|
||||
"structured_depth_count",
|
||||
"measurement_date_min",
|
||||
"measurement_date_max",
|
||||
)
|
||||
}
|
||||
for partition in sorted(partitions, key=lambda item: str(item["area_id"]))
|
||||
],
|
||||
}
|
||||
digest = hashlib.sha256(
|
||||
json.dumps(identity, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")
|
||||
).hexdigest()
|
||||
return identity, digest
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
if args.max_partitions < 0:
|
||||
print(json.dumps({"status": "error", "message": "--max-partitions cannot be negative"}), file=sys.stderr)
|
||||
return 2
|
||||
base_url = args.base_url.rstrip("/")
|
||||
session = requests.Session()
|
||||
session.headers.update({"User-Agent": "GeoIntel-VHA-Flanders-Operator/1.0"})
|
||||
manifest = load_manifest(args.manifest_path)
|
||||
|
||||
try:
|
||||
projects = list_paginated(session, f"{base_url}/api/v1/projects", timeout=60)
|
||||
project = next((item for item in projects if item.get("name") == args.project_name), None)
|
||||
if project is None:
|
||||
raise RuntimeError(
|
||||
f"Project {args.project_name!r} was not found; run provision_flanders_geographic_scope.py first"
|
||||
)
|
||||
all_areas = list_paginated(
|
||||
session,
|
||||
f"{base_url}/api/v1/projects/{project['id']}/areas",
|
||||
timeout=120,
|
||||
)
|
||||
municipalities = sorted(
|
||||
(area for area in all_areas if str(area.get("name") or "").casefold().startswith("gemeente ")),
|
||||
key=lambda item: str(item["name"]).casefold(),
|
||||
)
|
||||
if not 270 <= len(municipalities) <= 300:
|
||||
raise RuntimeError(
|
||||
f"Flanders workspace exposes {len(municipalities)} municipality Areas; expected 270..300"
|
||||
)
|
||||
requested_names = {name.casefold() for name in args.members}
|
||||
selected = [
|
||||
area
|
||||
for area in municipalities
|
||||
if not requested_names
|
||||
or str(area["name"]).removeprefix("Gemeente ").split(" - ", 1)[0].casefold() in requested_names
|
||||
]
|
||||
if requested_names:
|
||||
found = {
|
||||
str(area["name"]).removeprefix("Gemeente ").split(" - ", 1)[0].casefold()
|
||||
for area in selected
|
||||
}
|
||||
missing = sorted(requested_names - found)
|
||||
if missing:
|
||||
raise RuntimeError(f"Unknown Flanders municipality selections: {', '.join(missing)}")
|
||||
if args.max_partitions:
|
||||
selected = selected[: args.max_partitions]
|
||||
|
||||
prior_by_area = {
|
||||
str(item.get("area_id")): item
|
||||
for item in (manifest.get("partitions") or [])
|
||||
if isinstance(item, dict) and item.get("area_id")
|
||||
}
|
||||
current: list[dict[str, Any]] = []
|
||||
failures: list[dict[str, Any]] = []
|
||||
observed_at = datetime.now(timezone.utc).isoformat()
|
||||
working_manifest = {
|
||||
"schema_version": 1,
|
||||
"status": "running",
|
||||
"source": "vmm_vha_bathymetry_profiles",
|
||||
"partition_scope_key": "flanders",
|
||||
"project_id": project["id"],
|
||||
"project_name": project["name"],
|
||||
"municipality_inventory_count": len(municipalities),
|
||||
"selected_partition_count": len(selected),
|
||||
"observed_at": observed_at,
|
||||
"partitions": current,
|
||||
}
|
||||
for index, area in enumerate(selected, start=1):
|
||||
identity = area_identity(area)
|
||||
prior = prior_by_area.get(str(area["id"]))
|
||||
if (
|
||||
not args.force
|
||||
and prior
|
||||
and prior.get("area_identity_sha256") == identity
|
||||
and prior.get("status") in {"complete", "no_profiles"}
|
||||
):
|
||||
result = prior
|
||||
else:
|
||||
try:
|
||||
result = acquire_partition(
|
||||
session,
|
||||
base_url=base_url,
|
||||
project_id=str(project["id"]),
|
||||
area=area,
|
||||
timeout=args.timeout,
|
||||
force=args.force,
|
||||
)
|
||||
except (RuntimeError, requests.RequestException) as exc:
|
||||
result = {
|
||||
"status": "failed",
|
||||
"area_id": area["id"],
|
||||
"area_name": area["name"],
|
||||
"area_identity_sha256": identity,
|
||||
"message": str(exc),
|
||||
"failed_at": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
failures.append(result)
|
||||
current.append(result)
|
||||
working_manifest["completed_partition_count"] = index
|
||||
working_manifest["partitions"] = current
|
||||
write_manifest(args.manifest_path, working_manifest)
|
||||
if result["status"] == "failed" and not args.continue_on_error:
|
||||
raise RuntimeError(f"Partition failed for {area['name']}: {result['message']}")
|
||||
|
||||
full_inventory_selected = len(selected) == len(municipalities) and not requested_names and not args.max_partitions
|
||||
complete = not failures and all(item["status"] in {"complete", "no_profiles"} for item in current)
|
||||
identity_payload, identity_sha = manifest_identity(
|
||||
project_id=str(project["id"]),
|
||||
partitions=current,
|
||||
)
|
||||
finalization = None
|
||||
if complete and full_inventory_selected:
|
||||
dataset_ids = [item["dataset_id"] for item in current if item["status"] == "complete"]
|
||||
no_profile_area_ids = [item["area_id"] for item in current if item["status"] == "no_profiles"]
|
||||
finalization = unwrap(
|
||||
session.post(
|
||||
f"{base_url}/api/v1/projects/{project['id']}/datasets/bathymetry/profiles/partitions/finalize",
|
||||
json={
|
||||
"partition_scope_key": "flanders",
|
||||
"expected_area_ids": [area["id"] for area in municipalities],
|
||||
"dataset_ids": dataset_ids,
|
||||
"no_profile_area_ids": no_profile_area_ids,
|
||||
"manifest_sha256": identity_sha,
|
||||
"observed_at": observed_at,
|
||||
},
|
||||
timeout=args.timeout,
|
||||
)
|
||||
)
|
||||
|
||||
working_manifest.update(
|
||||
{
|
||||
"status": "complete" if complete else "failed",
|
||||
"completed_at": datetime.now(timezone.utc).isoformat(),
|
||||
"coverage_manifest": identity_payload,
|
||||
"coverage_manifest_sha256": identity_sha,
|
||||
"regional_finalized": finalization is not None,
|
||||
"finalization": finalization,
|
||||
"failed_partition_count": len(failures),
|
||||
}
|
||||
)
|
||||
write_manifest(args.manifest_path, working_manifest)
|
||||
except (OSError, RuntimeError, ValueError, KeyError, requests.RequestException) as exc:
|
||||
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": working_manifest["status"],
|
||||
"project_id": project["id"],
|
||||
"municipality_inventory_count": len(municipalities),
|
||||
"processed_partition_count": len(current),
|
||||
"data_partition_count": sum(item["status"] == "complete" for item in current),
|
||||
"no_profile_partition_count": sum(item["status"] == "no_profiles" for item in current),
|
||||
"failed_partition_count": len(failures),
|
||||
"profile_count": sum(int(item.get("profile_count") or 0) for item in current),
|
||||
"document_count": sum(int(item.get("document_count") or 0) for item in current),
|
||||
"regional_finalized": finalization is not None,
|
||||
"manifest_path": str(args.manifest_path),
|
||||
"manifest_sha256": identity_sha,
|
||||
"finalization": finalization,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0 if working_manifest["status"] == "complete" else 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,220 @@
|
||||
"""Provision all current official Flemish municipality Areas.
|
||||
|
||||
The municipality inventory is discovered from the official VRBG RefGem
|
||||
collection at runtime. The script uses the existing geographic-scope artifact
|
||||
and API persistence flow; it never writes directly to PostGIS.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from geographic_scopes import GeographicScope, ScopeMember
|
||||
from provision_geographic_scope import (
|
||||
VRBG_ATTRIBUTION,
|
||||
VRBG_ITEMS_URL,
|
||||
build_scope_payloads,
|
||||
build_source_session,
|
||||
provision_scope,
|
||||
sha256_file,
|
||||
write_json_atomic,
|
||||
)
|
||||
|
||||
|
||||
DEFAULT_API_URL = "http://127.0.0.1:8000"
|
||||
DEFAULT_OUTPUT_ROOT = Path("/app/storage/operator-data/geographic-scopes")
|
||||
MIN_EXPECTED_MUNICIPALITIES = 270
|
||||
MAX_EXPECTED_MUNICIPALITIES = 300
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Provision the current official Flanders municipality scope.")
|
||||
parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL))
|
||||
parser.add_argument(
|
||||
"--output-root",
|
||||
type=Path,
|
||||
default=Path(os.environ.get("GEOINTEL_SCOPE_OUTPUT_ROOT", DEFAULT_OUTPUT_ROOT)),
|
||||
)
|
||||
parser.add_argument("--request-timeout", type=int, default=180)
|
||||
parser.add_argument("--import-timeout", type=int, default=3600)
|
||||
parser.add_argument("--force", action="store_true")
|
||||
parser.add_argument("--fetch-only", action="store_true")
|
||||
parser.add_argument("--min-municipalities", type=int, default=MIN_EXPECTED_MUNICIPALITIES)
|
||||
parser.add_argument("--max-municipalities", type=int, default=MAX_EXPECTED_MUNICIPALITIES)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def discover_flanders_scope(
|
||||
session: requests.Session,
|
||||
*,
|
||||
timeout: int,
|
||||
min_municipalities: int,
|
||||
max_municipalities: int,
|
||||
) -> tuple[GeographicScope, list[dict[str, Any]], str]:
|
||||
if min_municipalities <= 0 or max_municipalities < min_municipalities:
|
||||
raise ValueError("Municipality count safety limits are invalid")
|
||||
response = session.get(
|
||||
VRBG_ITEMS_URL,
|
||||
params={"f": "application/geo+json", "limit": "1000"},
|
||||
timeout=timeout,
|
||||
)
|
||||
response.raise_for_status()
|
||||
payload = response.json()
|
||||
features = payload.get("features")
|
||||
if not isinstance(features, list):
|
||||
raise RuntimeError("Official VRBG response does not contain a feature list")
|
||||
if not min_municipalities <= len(features) <= max_municipalities:
|
||||
raise RuntimeError(
|
||||
f"Official VRBG returned {len(features)} municipalities; expected "
|
||||
f"{min_municipalities}..{max_municipalities}. Refusing an incomplete or broadened scope."
|
||||
)
|
||||
|
||||
by_code: dict[str, dict[str, Any]] = {}
|
||||
names: set[str] = set()
|
||||
for feature in features:
|
||||
if not isinstance(feature, dict) or not isinstance(feature.get("geometry"), dict):
|
||||
raise RuntimeError("Official VRBG contains a malformed municipality feature")
|
||||
properties = feature.get("properties")
|
||||
if not isinstance(properties, dict):
|
||||
raise RuntimeError("Official VRBG municipality is missing properties")
|
||||
nis_code = str(properties.get("NISCODE") or "").strip()
|
||||
name = str(properties.get("NAAM") or "").strip()
|
||||
if len(nis_code) != 5 or not nis_code.isdigit() or not name:
|
||||
raise RuntimeError(f"Official VRBG municipality identity is invalid: {nis_code!r} / {name!r}")
|
||||
if nis_code in by_code or name.casefold() in names:
|
||||
raise RuntimeError(f"Official VRBG contains a duplicate municipality: {nis_code} / {name}")
|
||||
by_code[nis_code] = feature
|
||||
names.add(name.casefold())
|
||||
|
||||
ordered = [by_code[code] for code in sorted(by_code)]
|
||||
members = tuple(
|
||||
ScopeMember(
|
||||
name=str(feature["properties"]["NAAM"]).strip(),
|
||||
nis_code=str(feature["properties"]["NISCODE"]).strip(),
|
||||
)
|
||||
for feature in ordered
|
||||
)
|
||||
scope = GeographicScope(
|
||||
key="flanders",
|
||||
display_name=f"Vlaanderen ({len(members)} gemeenten)",
|
||||
project_name="Flanders Regional Workbench",
|
||||
project_region="Vlaanderen, België",
|
||||
area_name="Vlaanderen - officiële operationele grens",
|
||||
authority_name="Digitaal Vlaanderen VRBG",
|
||||
authority_url=VRBG_ITEMS_URL,
|
||||
scope_type="region",
|
||||
limitation_message=(
|
||||
"Officiële unie van de actuele VRBG-gemeentegrenzen. De operationele regio omvat het Vlaamse "
|
||||
"landgebied; territoriale zee, EEZ en continentaal plat zijn afzonderlijke maritieme scopes."
|
||||
),
|
||||
members=members,
|
||||
)
|
||||
return scope, ordered, response.url
|
||||
|
||||
|
||||
def prepare_artifacts(
|
||||
args: argparse.Namespace,
|
||||
scope: GeographicScope,
|
||||
features: list[dict[str, Any]],
|
||||
source_url: str,
|
||||
) -> tuple[Path, Path, dict[str, Any]]:
|
||||
output_dir = args.output_root / scope.key
|
||||
manifest_path = output_dir / "flanders_scope_manifest.json"
|
||||
generated_at = datetime.now(timezone.utc).isoformat()
|
||||
snapshot_date = generated_at[:10]
|
||||
boundary_path = output_dir / f"flanders_boundary_{snapshot_date}.geojson"
|
||||
members_path = output_dir / f"flanders_municipalities_{snapshot_date}.geojson"
|
||||
member_codes = list(scope.nis_codes)
|
||||
|
||||
if not args.force and manifest_path.is_file():
|
||||
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||
cached_boundary = output_dir / str(manifest.get("boundary_filename") or "")
|
||||
cached_members = output_dir / str(manifest.get("municipalities_filename") or "")
|
||||
if (
|
||||
manifest.get("status") == "complete"
|
||||
and manifest.get("member_nis_codes") == member_codes
|
||||
and cached_boundary.is_file()
|
||||
and cached_members.is_file()
|
||||
and sha256_file(cached_boundary) == manifest.get("boundary_sha256")
|
||||
and sha256_file(cached_members) == manifest.get("municipalities_sha256")
|
||||
):
|
||||
return cached_boundary, cached_members, manifest
|
||||
|
||||
boundary_payload, members_payload, summary = build_scope_payloads(
|
||||
scope,
|
||||
features,
|
||||
source_url=source_url,
|
||||
generated_at=generated_at,
|
||||
)
|
||||
write_json_atomic(boundary_path, boundary_payload)
|
||||
write_json_atomic(members_path, members_payload)
|
||||
manifest = {
|
||||
"schema_version": 1,
|
||||
"status": "complete",
|
||||
"generated_at": generated_at,
|
||||
"observed_at": f"{snapshot_date}T00:00:00Z",
|
||||
"boundary_filename": boundary_path.name,
|
||||
"boundary_sha256": sha256_file(boundary_path),
|
||||
"municipalities_filename": members_path.name,
|
||||
"municipalities_sha256": sha256_file(members_path),
|
||||
"vrbg_source_url": source_url,
|
||||
"vrbg_attribution": VRBG_ATTRIBUTION,
|
||||
"scope_authority_name": scope.authority_name,
|
||||
"scope_authority_url": scope.authority_url,
|
||||
"scope_limitation": scope.limitation_message,
|
||||
**summary,
|
||||
}
|
||||
write_json_atomic(manifest_path, manifest, pretty=True)
|
||||
return boundary_path, members_path, manifest
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
args.operator_tool = "provision_flanders_geographic_scope.py"
|
||||
try:
|
||||
with build_source_session() as session:
|
||||
scope, features, source_url = discover_flanders_scope(
|
||||
session,
|
||||
timeout=args.request_timeout,
|
||||
min_municipalities=args.min_municipalities,
|
||||
max_municipalities=args.max_municipalities,
|
||||
)
|
||||
boundary_path, members_path, manifest = prepare_artifacts(args, scope, features, source_url)
|
||||
workspace = None if args.fetch_only else provision_scope(
|
||||
args, scope, boundary_path, members_path, manifest
|
||||
)
|
||||
except (OSError, RuntimeError, ValueError, KeyError, requests.RequestException) as exc:
|
||||
print(json.dumps({"status": "error", "message": str(exc)}, ensure_ascii=False), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "ok",
|
||||
"mode": "fetch_only" if args.fetch_only else "provisioned",
|
||||
"scope": scope.key,
|
||||
"member_count": manifest["member_count"],
|
||||
"area_km2": manifest["area_km2"],
|
||||
"wgs84_bbox": manifest["wgs84_bbox"],
|
||||
"boundary_path": str(boundary_path),
|
||||
"municipalities_path": str(members_path),
|
||||
"workspace": workspace,
|
||||
"limitation": scope.limitation_message,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -478,7 +478,7 @@ def provision_scope(
|
||||
"attribution": VRBG_ATTRIBUTION,
|
||||
}
|
||||
common_provenance = {
|
||||
"operator_tool": "provision_geographic_scope.py",
|
||||
"operator_tool": getattr(args, "operator_tool", "provision_geographic_scope.py"),
|
||||
"operator_explicit_fetch": True,
|
||||
"manifest_path": str(args.output_root / scope.key / f"{scope.key.replace('-', '_')}_scope_manifest.json"),
|
||||
"source_url": manifest["vrbg_source_url"],
|
||||
|
||||
@@ -51,6 +51,9 @@ ${PYTHON_BIN} -m py_compile scripts/provision_mol_historical_landuse.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_historical_landuse.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_official_landuse_timeseries.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_waterinfo_station_history.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_flanders_geographic_scope.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_flanders_bathymetry_profiles.py
|
||||
${PYTHON_BIN} -m py_compile scripts/probe_mdk_bathymetry.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_bwk_natura2000.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py
|
||||
|
||||
Reference in New Issue
Block a user