From e620689d8cfc72cf1731f6242a0b621416d804b6 Mon Sep 17 00:00:00 2001 From: Codex Date: Thu, 16 Jul 2026 12:36:34 +0200 Subject: [PATCH] Add regional DHMV provisioning --- backend/README.md | 16 + .../test_sprint211_regional_flood_hazards.py | 1 + backend/tests/test_sprint218_regional_dhmv.py | 185 ++++++++++ deploy/unraid/Dockerfile.all-in-one | 1 + docs/DATA_SOURCES.md | 11 +- docs/STORAGE_ARCHITECTURE.md | 7 + docs/TODO.md | 2 + scripts/README.md | 26 ++ scripts/provision_regional_dhmv.py | 337 ++++++++++++++++++ scripts/provision_regional_flood_hazards.py | 38 +- scripts/run_readiness_check.sh | 1 + 11 files changed, 611 insertions(+), 14 deletions(-) create mode 100644 backend/tests/test_sprint218_regional_dhmv.py create mode 100644 scripts/provision_regional_dhmv.py diff --git a/backend/README.md b/backend/README.md index bfcc865e..7a0e1961 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1230,6 +1230,22 @@ needed. `POST .../raster/terrain/select` returns height in m TAW, relief in metres and slope in degrees. `GET .../raster/terrain/image` returns the constrained MapLibre PNG. Water depth, volume and drainage remain unavailable. +Provision the same governed DTM/DSM pair for every persisted municipality in +the approved Kempen scope: + +```bash +docker exec geointel python /app/scripts/provision_regional_dhmv.py \ + --scope kempen-transport-region --dry-run +docker exec geointel python /app/scripts/provision_regional_dhmv.py \ + --scope kempen-transport-region +``` + +This plans 56 municipality/product acquisitions. It supports bounded +`--members` and `--products` subsets, backend cache reuse, per-item progress +and a complete failure summary. Persistence remains inside the canonical +DHMV acquisition service and Dataset/DatasetVersion/Job flow; the operator +does not fetch WCS bytes or write raster metadata directly. + Settings: `DHMV_ENABLED`, `DHMV_WCS_URL`, `DHMV_RESOLUTION_M`, `DHMV_MIN_SIDE_M`, `DHMV_MAX_SIDE_M`, `DHMV_MAX_PIXELS`, `DHMV_TIMEOUT_SECONDS` and `DHMV_MAX_RESPONSE_MB`. diff --git a/backend/tests/test_sprint211_regional_flood_hazards.py b/backend/tests/test_sprint211_regional_flood_hazards.py index 833b6637..974c1d61 100644 --- a/backend/tests/test_sprint211_regional_flood_hazards.py +++ b/backend/tests/test_sprint211_regional_flood_hazards.py @@ -155,6 +155,7 @@ def test_regional_flood_operator_calls_acquisition_and_selection(monkeypatch, ca assert result == 0 assert '"completed_count": 1' in output + assert '"status": "completed_item"' in output assert len(fake_session.posts) == 2 acquisition_payload = fake_session.posts[0][1] assert fake_session.posts[0][0].endswith("/datasets/flood-hazard/acquire") diff --git a/backend/tests/test_sprint218_regional_dhmv.py b/backend/tests/test_sprint218_regional_dhmv.py new file mode 100644 index 00000000..ba9b1cdf --- /dev/null +++ b/backend/tests/test_sprint218_regional_dhmv.py @@ -0,0 +1,185 @@ +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path +from typing import Any + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = ROOT / "scripts" + + +def load_operator(): + if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + spec = importlib.util.spec_from_file_location( + "test_provision_regional_dhmv", + SCRIPTS / "provision_regional_dhmv.py", + ) + module = importlib.util.module_from_spec(spec) + assert spec and spec.loader + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +class FakeResponse: + def __init__(self, payload: dict[str, Any]): + self.payload = payload + self.url = "http://test.local" + + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, Any]: + return self.payload + + +class FakeSession: + def __init__(self, module): + self.module = module + self.posts: list[tuple[str, dict[str, Any]]] = [] + self.gets: list[tuple[str, dict[str, Any]]] = [] + self.headers: dict[str, str] = {} + + def get(self, url: str, **kwargs): + self.gets.append((url, kwargs.get("params") or {})) + if url.endswith("/api/v1/projects"): + return FakeResponse({"data": {"items": [{"id": "project-1", "name": "Kempen Regional Workbench"}]}}) + if url.endswith("/areas"): + return FakeResponse( + { + "data": { + "items": [ + { + "id": "area-mol", + "name": "Gemeente Mol - officiele grens", + "geometry": { + "type": "Polygon", + "coordinates": [[ + [5.0, 51.0], + [5.1, 51.0], + [5.1, 51.1], + [5.0, 51.1], + [5.0, 51.0], + ]], + }, + } + ], + "total": 1, + } + } + ) + if url.endswith("/datasets/dhmv/products"): + return FakeResponse({"data": {"items": [{"key": key} for key in self.module.PRODUCTS]}}) + raise AssertionError(url) + + def post(self, url: str, json: dict[str, Any], **_kwargs): + self.posts.append((url, json)) + if url.endswith("/datasets/dhmv/acquire"): + return FakeResponse( + { + "data": { + "id": "job-1", + "status": "success", + "output_dataset_id": "dataset-1", + "result_json": {"reused": True}, + } + } + ) + if url.endswith("/raster/terrain/select"): + return FakeResponse( + { + "data": { + "resolution_m": 5.0, + "sample_count": 100, + "coverage_ratio": 1.0, + "unsupported_metrics": ["water_depth_m", "water_volume_m3"], + "summary": {"metrics": [{"metric_key": "elevation_mean_m", "metric_value": 24.5}]}, + } + } + ) + raise AssertionError(url) + + +def test_regional_dhmv_operator_resolves_products_and_members() -> None: + module = load_operator() + + products = module.requested_products("dtm_1m,dsm_1m", set(module.PRODUCTS)) + members = module.requested_members("Mol,13008", module.KEMPEN_TRANSPORT_REGION_SCOPE.members) + + assert products == ["dtm_1m", "dsm_1m"] + assert [member.nis_code for member in members] == ["13025", "13008"] + + +def test_regional_dhmv_operator_rejects_unknown_scope_inputs() -> None: + module = load_operator() + + with pytest.raises(RuntimeError, match="Unsupported DHMV"): + module.requested_products("custom", set(module.PRODUCTS)) + with pytest.raises(RuntimeError, match="Unknown scope members"): + module.requested_members("Atlantis", module.KEMPEN_TRANSPORT_REGION_SCOPE.members) + + +def test_regional_dhmv_operator_dry_run_uses_canonical_registry(monkeypatch, capsys) -> None: + module = load_operator() + fake_session = FakeSession(module) + monkeypatch.setattr(module.requests, "Session", lambda: fake_session) + + result = module.main(["--members", "Mol", "--products", "dtm_1m", "--dry-run"]) + output = capsys.readouterr().out + + assert result == 0 + assert '"status": "dry_run"' in output + assert '"planned_acquisitions": 1' in output + assert fake_session.posts == [] + assert any(params.get("limit") == 200 for url, params in fake_session.gets if url.endswith("/areas")) + + +def test_regional_dhmv_operator_calls_acquisition_and_selection(monkeypatch, capsys) -> None: + module = load_operator() + fake_session = FakeSession(module) + monkeypatch.setattr(module.requests, "Session", lambda: fake_session) + + result = module.main(["--members", "Mol", "--products", "dtm_1m"]) + output = capsys.readouterr().out + + assert result == 0 + assert '"completed_count": 1' in output + assert '"status": "completed_item"' in output + assert len(fake_session.posts) == 2 + acquisition_payload = fake_session.posts[0][1] + assert fake_session.posts[0][0].endswith("/datasets/dhmv/acquire") + assert acquisition_payload["area_id"] == "area-mol" + assert acquisition_payload["product_key"] == "dtm_1m" + assert acquisition_payload["bbox"]["crs"] == "EPSG:4326" + + +def test_regional_dhmv_operator_rejects_missing_water_limitations(monkeypatch) -> None: + module = load_operator() + fake_session = FakeSession(module) + original_post = fake_session.post + + def post_without_limitations(url: str, json: dict[str, Any], **kwargs): + response = original_post(url, json, **kwargs) + if url.endswith("/raster/terrain/select"): + response.payload["data"]["unsupported_metrics"] = [] + return response + + fake_session.post = post_without_limitations + monkeypatch.setattr(module.requests, "Session", lambda: fake_session) + + assert module.main(["--members", "Mol", "--products", "dtm_1m"]) == 2 + + +def test_regional_dhmv_operator_is_packaged() -> None: + readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") + dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8") + docs = (ROOT / "scripts" / "README.md").read_text(encoding="utf-8") + + assert "py_compile scripts/provision_regional_dhmv.py" in readiness + assert "COPY scripts/provision_regional_dhmv.py" in dockerfile + assert "provision_regional_dhmv.py" in docs diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index e7f2a329..2d52227c 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -75,6 +75,7 @@ COPY scripts/prepare_operator_real_data_samples.py /app/scripts/prepare_operator COPY scripts/provision_mol_municipality_workspace.py /app/scripts/provision_mol_municipality_workspace.py COPY scripts/provision_mol_context_layers.py /app/scripts/provision_mol_context_layers.py COPY scripts/provision_mol_dhmv.py /app/scripts/provision_mol_dhmv.py +COPY scripts/provision_regional_dhmv.py /app/scripts/provision_regional_dhmv.py COPY scripts/provision_mol_flood_hazards.py /app/scripts/provision_mol_flood_hazards.py COPY scripts/provision_regional_flood_hazards.py /app/scripts/provision_regional_flood_hazards.py COPY scripts/provision_thematic_rasters.py /app/scripts/provision_thematic_rasters.py diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md index 84b18815..c9d93981 100644 --- a/docs/DATA_SOURCES.md +++ b/docs/DATA_SOURCES.md @@ -385,8 +385,9 @@ quality metrics. - Native raster: 1 m Float32, EPSG:31370, nodata `-9999`, hoogte in TAW - Opnameperiode: 2013-2015; geen uniforme recente peildatum - Cache: canonical raster Dataset plus WCS request/response/output checksums -- Operator: `scripts/provision_mol_dhmv.py` -- Prioriteit: P4 uitgevoerd voor Mol +- Operators: `scripts/provision_mol_dhmv.py`, + `scripts/provision_regional_dhmv.py` +- Prioriteit: P4 uitgevoerd voor Mol en operationeel regionaal uitbreidbaar The operator requests a bounded 5 m analysis copy by default so a complete municipality remains operationally manageable while retaining the official @@ -396,6 +397,12 @@ DSM includes buildings and vegetation. Neither is exposed as water depth, water volume or a directly measured building-height product. A drainage model would require a separately governed hydrological processing pass. +De regionale operator gebruikt exact de 28 persistente gemeente-Areas van de +goedgekeurde Kempen-scope en plant twee outputs per gemeente. Die 56 +gemeentepartities vermijden een onnodig monolithisch hoogtebestand, blijven +binnen WCS/pixelgrenzen en sluiten aan op de gebiedsgebonden datasetselectie in +de kaart. Herhaalruns gebruiken de bestaande checksummed requestcache. + ## VMM overstromingsgevaarkaarten - Naam: Overstromingsgevaarkaart Waterdiepte diff --git a/docs/STORAGE_ARCHITECTURE.md b/docs/STORAGE_ARCHITECTURE.md index e00cff04..257e3ec6 100644 --- a/docs/STORAGE_ARCHITECTURE.md +++ b/docs/STORAGE_ARCHITECTURE.md @@ -118,6 +118,13 @@ metadata records the 1 m native product, 5 m default analysis grid, EPSG:31370, derived browser views and are never authoritative. No raster binary is stored in PostgreSQL and no DHMV file is treated as water depth or volume. +`provision_regional_dhmv.py` creates no separate evidence hierarchy and no +monolithic regional raster. Each municipality/product result remains one +ordinary Area-linked raster Dataset plus DatasetVersion. The retained +provenance already includes every bounded WCS tile URL and aggregate response, +coverage and normalized-output checksum. A repeat operator run resolves the +same request identity and reuses the immutable file. + VMM flood-hazard scenario outputs follow the same raster Dataset policy. The bounded acquisition service stores one normalized, compressed, Area-clipped GeoTIFF per scenario/request identity. `provision_regional_flood_hazards.py` diff --git a/docs/TODO.md b/docs/TODO.md index 44cd7836..39926dfa 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -35,6 +35,8 @@ - [x] Execute the governed thematic-raster operator for every persisted Kempen municipality after the Mol live gate passes. - [x] Persist one bounded whole-region snapshot per thematic raster so drawn selections can cross municipality boundaries without changing source semantics. - [x] Generalize the DOV soil-map operator to all 28 approved Kempen municipality partitions with one regional snapshot manifest. +- [x] Add a resumable regional DHMV DTM/DSM operator for all 28 approved Kempen municipality Areas. +- [ ] Execute and audit the complete 336-product VMM and 56-product DHMV regional runtime matrices. ## Governed source expansion backlog diff --git a/scripts/README.md b/scripts/README.md index 9c65cc12..e2875253 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -1556,6 +1556,32 @@ docker exec geointel python /app/scripts/provision_mol_dhmv.py --resolution-m 5 Do not use DHMV output as water depth or water volume. The command fails when the API no longer reports those metrics as explicitly unsupported. +## Regional DHMV terrain rasters + +Plan and provision the governed DTM/DSM pair for all 28 persisted Kempen +municipality Areas: + +```bash +docker exec geointel python /app/scripts/provision_regional_dhmv.py \ + --scope kempen-transport-region --dry-run + +docker exec geointel python /app/scripts/provision_regional_dhmv.py \ + --scope kempen-transport-region +``` + +The complete run plans 56 canonical acquisitions. `--members Mol,Geel` and +`--products dtm_1m` provide bounded validation subsets; `--stop-on-error` +turns the default complete failure report into fail-fast behavior. Existing +checksummed request identities are reused unless `--force` is explicit. Each +successful item is reported immediately so a long operator run remains +observable. Every output is still an ordinary Area-linked Dataset and +DatasetVersion produced by the existing DHMV service. + +The regional operator deliberately retains municipality partitions. It does +not assemble a monolithic Kempen height raster, does not claim annual terrain +change and rejects any terrain-analysis response that stops listing water +depth and water volume as unsupported. + ## Mol VMM flood-hazard scenarios Acquire and validate all twelve official VMM fluvial/pluvial flood-depth diff --git a/scripts/provision_regional_dhmv.py b/scripts/provision_regional_dhmv.py new file mode 100644 index 00000000..2bd963ce --- /dev/null +++ b/scripts/provision_regional_dhmv.py @@ -0,0 +1,337 @@ +"""Provision governed DHMV II rasters for an approved GeoIntel scope. + +The command coordinates canonical API calls only. It does not fetch rasters +directly, write database rows directly or run implicitly at startup. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from dataclasses import dataclass +from typing import Any, Iterable, Sequence + +import requests + +from geographic_scopes import GEOGRAPHIC_SCOPES, KEMPEN_TRANSPORT_REGION_SCOPE, ScopeMember + + +DEFAULT_API_URL = "http://127.0.0.1:8000" +DEFAULT_SCOPE = KEMPEN_TRANSPORT_REGION_SCOPE.key +PRODUCTS = ("dtm_1m", "dsm_1m") + + +@dataclass(frozen=True) +class ResolvedArea: + id: str + name: str + member_name: str + nis_code: str + geometry: dict[str, Any] + + +def parse_args(argv: Sequence[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Provision official DHMV II rasters for an approved scope.") + parser.add_argument("--base-url", default=os.environ.get("GEOINTEL_INTERNAL_API_URL", DEFAULT_API_URL)) + parser.add_argument("--scope", choices=sorted(GEOGRAPHIC_SCOPES), default=DEFAULT_SCOPE) + parser.add_argument("--products", default=",".join(PRODUCTS), help="Comma-separated governed product keys.") + parser.add_argument( + "--members", + default="", + help="Optional comma-separated municipality names or NIS codes. Empty means every scope member.", + ) + parser.add_argument("--resolution-m", type=float, default=5.0) + parser.add_argument("--timeout", type=int, default=1800) + parser.add_argument("--force", action="store_true") + parser.add_argument("--stop-on-error", action="store_true") + parser.add_argument("--dry-run", action="store_true", help="Resolve scope/products only; do not acquire data.") + return parser.parse_args(argv) + + +def unwrap(response: requests.Response) -> Any: + response.raise_for_status() + payload = response.json() + if not isinstance(payload, dict) or "data" not in payload: + raise RuntimeError(f"Non-canonical API response from {response.url}") + return payload["data"] + + +def coordinates(geometry: dict[str, Any]) -> Iterable[tuple[float, float]]: + def walk(value: Any): + if isinstance(value, list) and len(value) >= 2 and all(isinstance(item, (int, float)) for item in value[:2]): + yield float(value[0]), float(value[1]) + return + if isinstance(value, list): + for child in value: + yield from walk(child) + + yield from walk(geometry.get("coordinates", [])) + + +def geometry_bbox(geometry: dict[str, Any]) -> dict[str, float | str]: + points = list(coordinates(geometry)) + if not points: + raise RuntimeError("Persisted Area geometry contains no coordinates") + xs = [point[0] for point in points] + ys = [point[1] for point in points] + return {"min_x": min(xs), "min_y": min(ys), "max_x": max(xs), "max_y": max(ys), "crs": "EPSG:4326"} + + +def requested_products(raw_products: str, registry_keys: set[str]) -> list[str]: + products = [item.strip().lower() for item in raw_products.split(",") if item.strip()] + invalid = sorted(set(products) - registry_keys) + if invalid: + raise RuntimeError(f"Unsupported DHMV product keys: {', '.join(invalid)}") + if not products: + raise RuntimeError("At least one DHMV product key is required") + return products + + +def requested_members(raw_members: str, members: tuple[ScopeMember, ...]) -> list[ScopeMember]: + if not raw_members.strip(): + return list(members) + index: dict[str, ScopeMember] = {} + for member in members: + index[member.name.casefold()] = member + index[member.nis_code] = member + selected: list[ScopeMember] = [] + unknown: list[str] = [] + for token in [item.strip() for item in raw_members.split(",") if item.strip()]: + member = index.get(token.casefold()) or index.get(token) + if member is None: + unknown.append(token) + elif member not in selected: + selected.append(member) + if unknown: + raise RuntimeError(f"Unknown scope members: {', '.join(unknown)}") + if not selected: + raise RuntimeError("At least one scope member is required") + return selected + + +def resolve_project(session: requests.Session, base_url: str, project_name: str) -> dict[str, Any]: + projects = unwrap(session.get(f"{base_url}/api/v1/projects", params={"limit": 200, "offset": 0}, timeout=60))["items"] + project = next((item for item in projects if item["name"] == project_name), None) + if project is None: + raise RuntimeError(f"Project {project_name!r} was not found. Run provision_geographic_scope.py first.") + return project + + +def resolve_areas( + session: requests.Session, + base_url: str, + project_id: str, + members: Sequence[ScopeMember], +) -> list[ResolvedArea]: + areas: list[dict[str, Any]] = [] + limit = 200 + offset = 0 + while True: + page = unwrap( + session.get( + f"{base_url}/api/v1/projects/{project_id}/areas", + params={"limit": limit, "offset": offset}, + timeout=60, + ) + ) + page_items = list(page["items"]) + areas.extend(page_items) + total = int(page.get("total", len(areas))) + if len(areas) >= total or len(page_items) < limit: + break + offset += limit + + resolved: list[ResolvedArea] = [] + missing: list[str] = [] + for member in members: + area = next( + ( + item + for item in areas + if member.name.casefold() in str(item.get("name", "")).casefold() + or member.nis_code in str(item.get("name", "")) + or member.nis_code in str(item.get("source_metadata", "")) + ), + None, + ) + if area is None: + missing.append(f"{member.name} ({member.nis_code})") + continue + if not area.get("geometry"): + raise RuntimeError(f"Area {area.get('name')!r} has no geometry in the canonical API response") + resolved.append( + ResolvedArea( + id=area["id"], + name=area["name"], + member_name=member.name, + nis_code=member.nis_code, + geometry=area["geometry"], + ) + ) + if missing: + raise RuntimeError( + "Persisted municipality Areas are missing: " + + ", ".join(missing) + + ". Run provision_geographic_scope.py for the selected scope first." + ) + return resolved + + +def validate_registry(session: requests.Session, base_url: str, project_id: str) -> set[str]: + registry = unwrap(session.get(f"{base_url}/api/v1/projects/{project_id}/datasets/dhmv/products", timeout=60))["items"] + registry_keys = {item["key"] for item in registry} + if registry_keys != set(PRODUCTS): + raise RuntimeError("Backend DHMV registry does not expose the governed DTM/DSM product set") + return registry_keys + + +def acquire_one( + session: requests.Session, + base_url: str, + project_id: str, + area: ResolvedArea, + product_key: str, + resolution_m: float, + timeout: int, + force: bool, +) -> dict[str, Any]: + bbox = geometry_bbox(area.geometry) + job = unwrap( + session.post( + f"{base_url}/api/v1/projects/{project_id}/datasets/dhmv/acquire", + json={ + "bbox": bbox, + "area_id": area.id, + "product_key": product_key, + "resolution_m": resolution_m, + "force_refresh": force, + }, + timeout=timeout, + ) + ) + if job.get("status") != "success" or not job.get("output_dataset_id"): + raise RuntimeError(f"DHMV acquisition failed for {area.member_name} {product_key}: {job.get('error_message') or job}") + analysis = unwrap( + session.post( + f"{base_url}/api/v1/projects/{project_id}/datasets/{job['output_dataset_id']}/raster/terrain/select", + json={"bbox": bbox, "area_id": area.id}, + timeout=timeout, + ) + ) + if sorted(analysis.get("unsupported_metrics", [])) != ["water_depth_m", "water_volume_m3"]: + raise RuntimeError("DHMV terrain contract must explicitly keep water depth and volume unavailable") + return { + "member_name": area.member_name, + "nis_code": area.nis_code, + "area_id": area.id, + "area_name": area.name, + "product_key": product_key, + "dataset_id": job["output_dataset_id"], + "reused": bool((job.get("result_json") or {}).get("reused")), + "resolution_m": analysis["resolution_m"], + "sample_count": analysis["sample_count"], + "coverage_ratio": analysis["coverage_ratio"], + "metrics": analysis["summary"]["metrics"], + } + + +def main(argv: Sequence[str] | None = None) -> int: + args = parse_args(argv) + scope = GEOGRAPHIC_SCOPES[args.scope] + base_url = args.base_url.rstrip("/") + selected_members = requested_members(args.members, scope.members) + + session = requests.Session() + session.headers.update({"User-Agent": "GeoIntel-Regional-DHMV-Operator/1.0"}) + project = resolve_project(session, base_url, scope.project_name) + registry_keys = validate_registry(session, base_url, project["id"]) + products = requested_products(args.products, registry_keys) + areas = resolve_areas(session, base_url, project["id"], selected_members) + + if args.dry_run: + print( + json.dumps( + { + "status": "dry_run", + "scope": scope.key, + "project_id": project["id"], + "member_count": len(areas), + "product_count": len(products), + "planned_acquisitions": len(areas) * len(products), + "members": [{"name": area.member_name, "nis_code": area.nis_code, "area_id": area.id} for area in areas], + "products": products, + }, + ensure_ascii=False, + indent=2, + ) + ) + return 0 + + results: list[dict[str, Any]] = [] + failures: list[dict[str, Any]] = [] + for area in areas: + for product_key in products: + try: + result = acquire_one( + session, + base_url, + project["id"], + area, + product_key, + args.resolution_m, + args.timeout, + args.force, + ) + results.append(result) + print( + json.dumps( + { + "status": "completed_item", + "completed": len(results), + "planned": len(areas) * len(products), + "member_name": area.member_name, + "product_key": product_key, + "dataset_id": result["dataset_id"], + "reused": result["reused"], + }, + ensure_ascii=False, + ), + flush=True, + ) + except Exception as exc: # noqa: BLE001 - retain every failed member/product in the operator summary. + failure = { + "member_name": area.member_name, + "nis_code": area.nis_code, + "area_id": area.id, + "product_key": product_key, + "error": str(exc), + } + failures.append(failure) + print(json.dumps({"status": "failed_item", **failure}, ensure_ascii=False), file=sys.stderr, flush=True) + if args.stop_on_error: + raise + + print( + json.dumps( + { + "status": "ok" if not failures else "partial", + "scope": scope.key, + "project_id": project["id"], + "member_count": len(areas), + "product_count": len(products), + "completed_count": len(results), + "failure_count": len(failures), + "products": results, + "failures": failures, + }, + ensure_ascii=False, + indent=2, + ) + ) + return 0 if not failures else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/provision_regional_flood_hazards.py b/scripts/provision_regional_flood_hazards.py index 2eb1054f..a6bff434 100644 --- a/scripts/provision_regional_flood_hazards.py +++ b/scripts/provision_regional_flood_hazards.py @@ -283,17 +283,31 @@ def main(argv: Sequence[str] | None = None) -> int: for area in areas: for product_key in products: try: - results.append( - acquire_one( - session, - base_url, - project["id"], - area, - product_key, - args.resolution_m, - args.timeout, - args.force, - ) + result = acquire_one( + session, + base_url, + project["id"], + area, + product_key, + args.resolution_m, + args.timeout, + args.force, + ) + results.append(result) + print( + json.dumps( + { + "status": "completed_item", + "completed": len(results), + "planned": len(areas) * len(products), + "member_name": area.member_name, + "product_key": product_key, + "dataset_id": result["dataset_id"], + "reused": result["reused"], + }, + ensure_ascii=False, + ), + flush=True, ) except Exception as exc: # noqa: BLE001 - operator summary should retain every failed member/product. failure = { @@ -304,7 +318,7 @@ def main(argv: Sequence[str] | None = None) -> int: "error": str(exc), } failures.append(failure) - print(json.dumps({"status": "failed_item", **failure}, ensure_ascii=False), file=sys.stderr) + print(json.dumps({"status": "failed_item", **failure}, ensure_ascii=False), file=sys.stderr, flush=True) if args.stop_on_error: raise diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 5e007fa6..90aa99cb 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -54,6 +54,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_regional_bwk_natura2000.py ${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py ${PYTHON_BIN} -m py_compile scripts/provision_buildings_addresses_register.py ${PYTHON_BIN} -m py_compile scripts/provision_mol_dhmv.py +${PYTHON_BIN} -m py_compile scripts/provision_regional_dhmv.py ${PYTHON_BIN} -m py_compile scripts/provision_mol_flood_hazards.py ${PYTHON_BIN} -m py_compile scripts/provision_regional_flood_hazards.py ${PYTHON_BIN} -m py_compile scripts/provision_thematic_rasters.py