From 06d05f03a6b419dba32cc925de6316b78e684d2d Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 17 Jul 2026 02:17:04 +0200 Subject: [PATCH] Add governed orthophoto release promotion --- CHANGELOG.md | 19 + backend/README.md | 48 +- .../services/source_catalog_probe_service.py | 15 +- ...sprint231_orthophoto_release_management.py | 405 +++++++ deploy/unraid/Dockerfile.all-in-one | 1 + docs/API_CONTRACTS.md | 11 + docs/CODEX_EXECUTION_LOG.md | 32 + docs/DATABASE_IMPLEMENTATION_PLAN.md | 16 +- docs/DATA_SOURCES.md | 15 +- docs/DATA_SPECIFICATION.md | 10 +- docs/STORAGE_ARCHITECTURE.md | 27 +- docs/TODO.md | 6 +- scripts/manage_orthophoto_release.py | 1001 +++++++++++++++++ scripts/run_readiness_check.sh | 1 + 14 files changed, 1579 insertions(+), 28 deletions(-) create mode 100644 backend/tests/test_sprint231_orthophoto_release_management.py create mode 100644 scripts/manage_orthophoto_release.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 69dae516..5f271fab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,25 @@ # Changelog +## Sprint 231 Governed orthophoto release promotion (2026-07-17) + +- Added an operator-only `plan -> stage -> review -> apply` coordinator for one + bounded current Digitaal Vlaanderen orthophoto selection. It reuses and + revalidates the Sprint 230 catalog, WMS, WCS and flight-day preflight. +- Stage performs exactly one bounded allowlisted WMS `GetMap`, retains the raw + response, normalizes a three-band EPSG:31370 GeoTIFF and creates a PNG review + preview. Every source, raster, preview, request and preflight identity is + SHA-256-bound under persistent operator evidence. +- Review requires a named explicit approval. Apply requires exact plan and + review hashes, rechecks remote and local state, and uploads only the approved + GeoTIFF through the existing dataset upload/DatasetService transaction. +- Added an explicit first-baseline transition for legacy + `most_recent_at_*` markers. It requires both the exact legacy marker and the + official edition; no existing Dataset is rewritten or deleted. +- Made official `YYYY.NN` orthophoto Datasets take precedence over rolling + acquisition markers in source-catalog comparison. No API, migration, + scheduler, browser fetch or automatic refresh was added. + ## Sprint 230 Governed orthophoto release preflight (2026-07-17) - Added an operator-only, read-only preflight for the current Digitaal diff --git a/backend/README.md b/backend/README.md index 605981dc..43d09232 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1313,12 +1313,48 @@ docker exec geointel python /app/scripts/orthophoto_release_preflight.py \ The command reads canonical API envelopes, exact official WMS capabilities, WCS `DescribeCoverage` and at most 64 queryable flight-day points. It never -requests raster pixels or mutates application/storage state. Only -`staging_permitted=true` may feed a future separate staging command. `current`, -remote-older, mixed/incorrect flight years and legacy local values such as -`most_recent_at_2026-07-15` remain non-stageable. The report's point grid is -flight-date evidence; complete selected-area coverage comes from containment -inside the official 15 cm WCS raster domain. +requests raster pixels or mutates application/storage state. `current`, +remote-older and mixed/incorrect flight years remain non-stageable. The +report's point grid is flight-date evidence; complete selected-area coverage +comes from containment inside the official 15 cm WCS raster domain. + +Official release promotion is a separate four-action operator workflow. Run it +inside the all-in-one container so stage/apply can use only the loopback API: + +```bash +# Read-only decision; copy the reported edition and current local marker. +docker exec geointel python /app/scripts/manage_orthophoto_release.py plan \ + --project-id 82a85913-c522-45d7-84a1-02b393d89e55 \ + --bbox 5.110 51.180 5.117 51.185 --refresh-catalog + +# First official baseline only: both values must match the fresh preflight. +docker exec geointel python /app/scripts/manage_orthophoto_release.py stage \ + --project-id 82a85913-c522-45d7-84a1-02b393d89e55 \ + --bbox 5.110 51.180 5.117 51.185 \ + --confirm-edition 2025.04 \ + --establish-official-baseline \ + --confirm-local-version most_recent_at_2026-07-15 + +# Inspect review-preview.png, then use the exact plan SHA printed by stage. +docker exec geointel python /app/scripts/manage_orthophoto_release.py review \ + --project-id 82a85913-c522-45d7-84a1-02b393d89e55 \ + --bbox 5.110 51.180 5.117 51.185 \ + --confirm-edition 2025.04 --confirm-plan-sha256 \ + --approve --reviewer "" --review-note "" + +# Apply only the exact approved bytes and hashes. +docker exec geointel python /app/scripts/manage_orthophoto_release.py apply \ + --project-id 82a85913-c522-45d7-84a1-02b393d89e55 \ + --bbox 5.110 51.180 5.117 51.185 \ + --confirm-edition 2025.04 --confirm-plan-sha256 \ + --confirm-review-sha256 +``` + +For a later comparable `YYYY.NN` update, omit the two first-baseline flags. +Stage performs one bounded pixel request but no database mutation. Apply is +idempotent for the exact plan/raster checksum, creates a new immutable raster +Dataset and DatasetVersion with the official edition, and retains every older +snapshot. No command is scheduled or invoked by startup or browser actions. ## Governed DHMV terrain acquisition diff --git a/backend/app/services/source_catalog_probe_service.py b/backend/app/services/source_catalog_probe_service.py index db56ae71..d54dfa21 100644 --- a/backend/app/services/source_catalog_probe_service.py +++ b/backend/app/services/source_catalog_probe_service.py @@ -37,7 +37,7 @@ _WFS = "http://www.opengis.net/wfs/2.0" _XLINK = "http://www.w3.org/1999/xlink" _METADATA_HOST = "metadata.vlaanderen.be" _VERSION_DATE = re.compile(r"^(?:toestand\s+)?(\d{4}-\d{2}-\d{2})$", re.IGNORECASE) -_ORTHOPHOTO_EDITION = re.compile(r"^\d{4}\.\d{2}$") +_ORTHOPHOTO_EDITION = re.compile(r"^(20\d{2})\.(\d{2})$") _ALZ_SOURCE_NAME = "agentschap_landbouw_zeevisserij_agricultural_parcels" _ALZ_RELEASE_HOST = "landbouwcijfers.vlaanderen.be" _ALZ_RELEASE_PATH = "/open-geodata-landbouwgebruikspercelen" @@ -681,6 +681,19 @@ def _dataset_source_name(dataset: Dataset) -> str: def _latest_local_version(source_name: str, datasets: list[Dataset]) -> str | None: candidates = [item for item in datasets if _dataset_source_name(item) == source_name and item.source_version] if source_name == "digitaal_vlaanderen_orthophoto": + official_editions = [ + item for item in candidates if _ORTHOPHOTO_EDITION.fullmatch((item.source_version or "").strip()) + ] + if official_editions: + latest = max( + official_editions, + key=lambda item: ( + tuple(int(value) for value in (item.source_version or "0.0").split(".")), + _utc(item.imported_at) if item.imported_at else datetime.min.replace(tzinfo=timezone.utc), + str(item.id), + ), + ) + return latest.source_version explicit_current = [ item for item in candidates diff --git a/backend/tests/test_sprint231_orthophoto_release_management.py b/backend/tests/test_sprint231_orthophoto_release_management.py new file mode 100644 index 00000000..c4bbbc04 --- /dev/null +++ b/backend/tests/test_sprint231_orthophoto_release_management.py @@ -0,0 +1,405 @@ +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import importlib.util +import json +from pathlib import Path +from types import SimpleNamespace +import sys + +import numpy as np +import pytest +from rasterio.io import MemoryFile +from rasterio.transform import from_origin + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + + +def load_script(name: str): + path = SCRIPTS / name + module_name = f"test_{path.stem}_sprint231" + spec = importlib.util.spec_from_file_location(module_name, path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + + +MANAGER = load_script("manage_orthophoto_release.py") + + +def arguments(tmp_path: Path, **overrides) -> argparse.Namespace: + values = { + "action": "plan", + "project_id": "00000000-0000-0000-0000-000000000001", + "scope": "kempen-transport-region", + "api_url": "http://127.0.0.1:8000/api/v1", + "bbox": [5.110, 51.180, 5.113, 51.182], + "area_id": None, + "confirm_edition": None, + "confirm_plan_sha256": None, + "confirm_review_sha256": None, + "approve": False, + "reviewer": None, + "review_note": "", + "establish_official_baseline": False, + "confirm_local_version": None, + "plan_path": None, + "review_path": None, + "evidence_root": tmp_path / "operator-evidence" / "orthophoto-release", + "refresh_catalog": False, + "api_timeout": 180, + "wms_timeout": 60, + "import_timeout": 600, + "max_response_mb": 32, + } + values.update(overrides) + return argparse.Namespace(**values) + + +def report(*, status: str = "update_available", local: str | None = "2024.03") -> dict: + return { + "schema_version": 1, + "status": "passed", + "generated_at": "2026-07-17T10:00:00Z", + "project_id": "00000000-0000-0000-0000-000000000001", + "scope": "kempen-transport-region", + "product": { + "key": "most_recent", + "display_name": "Orthofoto meest recent", + "temporal_granularity": "snapshot", + "native_resolution_m": 0.15, + "supports_detection": True, + "color_mode": "rgb", + "catalog_url": MANAGER.preflight.CATALOG_URL, + }, + "release": { + "status": status, + "remote_edition": "2025.04", + "remote_year": 2025, + "local_source_version": local, + "comparison_status": "different" if status == "update_available" else "not_comparable", + "metadata_identifier": MANAGER.preflight.METADATA_IDENTIFIER, + "metadata_url": "https://metadata.vlaanderen.be/srv/dut/catalog.search#/metadata/f5304d6d", + "remote_title": "Orthofoto meest recent, 2025.04", + "remote_modified_at": "2026-04-27T00:00:00Z", + "remote_published_at": "2025-12-11T00:00:00Z", + "catalog_checked_at": "2026-07-17T10:00:00Z", + "capabilities_url": MANAGER.WMS_BASE_URL + "?SERVICE=WMS&REQUEST=GetCapabilities", + "capabilities_sha256": "a" * 64, + }, + "capabilities": { + "service_version": "1.3.0", + "capabilities_sha256": "a" * 64, + "layers": ["Ortho", "Vliegdagcontour"], + "vliegdagcontour_queryable": True, + "feature_info_format": "application/geo+json", + "extent_epsg31370": [0.0, 0.0, 300000.0, 300000.0], + "metadata_identifier": MANAGER.preflight.METADATA_IDENTIFIER, + }, + "coverage_domain": { + "coverage_id": "Ortho", + "crs": "EPSG:31370", + "extent_epsg31370": [0.0, 0.0, 300000.0, 300000.0], + "native_resolution_m": 0.15, + "band_count": 3, + "native_format": "image/tiff", + "coverage_description_sha256": "b" * 64, + "selected_area_fully_inside_domain": True, + "pixel_data_requested": False, + }, + "selection": { + "bbox_epsg4326": [5.110, 51.180, 5.113, 51.182], + "bbox_epsg31370": [200000.0, 210000.0, 200200.0, 210160.0], + "width_m": 200.0, + "height_m": 160.0, + }, + "flight_day_coverage": { + "status": "passed", + "mode": "official_queryable_flight_day_grid", + "sample_count": 4, + "grid_columns": 2, + "grid_rows": 2, + "covered_sample_count": 4, + "sample_coverage_ratio": 1.0, + "flight_dates": ["5/4/2025"], + "flight_years": [2025], + "feature_ids": ["123"], + "sample_evidence_sha256": "c" * 64, + "claim_boundary": "Bounded point evidence, not a polygon-union proof.", + }, + "flight_year_matches_release": True, + "staging_permitted": status in {"not_loaded", "update_available"}, + "next_action": "governed_pixel_stage", + "pixel_requests_performed": 0, + "datasets_mutated": 0, + "automatic_staging": False, + "automatic_import": False, + } + + +def raw_rgb_tiff(width: int = 200, height: int = 160) -> bytes: + pixels = np.zeros((3, height, width), dtype=np.uint8) + pixels[0] = 80 + pixels[1] = np.arange(width, dtype=np.uint8)[None, :] + pixels[2] = np.arange(height, dtype=np.uint8)[:, None] + with MemoryFile() as memory: + with memory.open( + driver="GTiff", + width=width, + height=height, + count=3, + dtype="uint8", + crs="EPSG:31370", + transform=from_origin(200000.0, 210160.0, 1.0, 1.0), + ) as dataset: + dataset.write(pixels) + return memory.read() + + +class Response: + def __init__(self, body: bytes, url: str, content_type: str = "image/tiff") -> None: + self.body = body + self.url = url + self.headers = {"Content-Type": content_type, "Content-Length": str(len(body))} + + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def geturl(self) -> str: + return self.url + + def read(self, amount: int | None = None) -> bytes: + return self.body if amount is None else self.body[:amount] + + +def staged_plan(tmp_path: Path) -> tuple[argparse.Namespace, Path, dict, dict]: + args = arguments( + tmp_path, + action="stage", + confirm_edition="2025.04", + ) + release_report = report() + request = MANAGER.map_request(release_report) + staged = MANAGER.stage_artifacts( + args, + release_report, + request, + opener=lambda _request, timeout: Response(raw_rgb_tiff(), request["url"]), + ) + plan = MANAGER.build_staged_plan( + args, + release_report, + MANAGER.authorize_stage(args, release_report), + staged, + ) + path = MANAGER.default_plan_path(args, "2025.04") + MANAGER.write_json(path, plan) + return args, path, plan, staged + + +def test_orthophoto_catalog_prefers_latest_official_edition_over_rolling_marker() -> None: + from app.services.source_catalog_probe_service import _latest_local_version + + now = datetime(2026, 7, 17, tzinfo=timezone.utc) + datasets = [ + SimpleNamespace( + source_name=MANAGER.SOURCE_NAME, + source="", + source_version="2025.04", + imported_at=now, + observed_at=now, + id="official-2025", + ), + SimpleNamespace( + source_name=MANAGER.SOURCE_NAME, + source="", + source_version="most_recent_at_2026-07-18", + imported_at=datetime(2026, 7, 18, tzinfo=timezone.utc), + observed_at=now, + id="rolling", + ), + SimpleNamespace( + source_name=MANAGER.SOURCE_NAME, + source="", + source_version="2026.02", + imported_at=datetime(2026, 7, 16, tzinfo=timezone.utc), + observed_at=now, + id="official-2026", + ), + ] + + assert _latest_local_version(MANAGER.SOURCE_NAME, datasets) == "2026.02" + + +def test_stage_requires_exact_official_or_explicit_legacy_baseline_confirmation(tmp_path: Path) -> None: + args = arguments(tmp_path) + normal = report() + assert MANAGER.authorize_stage(args, normal)["mode"] == "normal_release" + + legacy = report(status="blocked_local_version", local="most_recent_at_2026-07-15") + with pytest.raises(RuntimeError, match="establish-official-baseline"): + MANAGER.authorize_stage(args, legacy) + args.establish_official_baseline = True + args.confirm_local_version = "most_recent_at_2026-07-15" + assert MANAGER.authorize_stage(args, legacy)["mode"] == "explicit_legacy_baseline_transition" + args.confirm_local_version = "different" + with pytest.raises(RuntimeError, match="exact"): + MANAGER.authorize_stage(args, legacy) + + +def test_stage_fetches_one_allowlisted_map_and_writes_reviewable_rgb_evidence(tmp_path: Path) -> None: + args = arguments(tmp_path) + release_report = report() + request = MANAGER.map_request(release_report) + calls = [] + + def opener(http_request, timeout): + calls.append((http_request.full_url, timeout)) + return Response(raw_rgb_tiff(), request["url"]) + + staged = MANAGER.stage_artifacts(args, release_report, request, opener=opener) + manifest = MANAGER.validate_staged_artifacts(args, Path(staged["manifest_path"])) + + assert calls == [(request["url"], 60)] + assert manifest["pixel_request_count"] == 1 + assert manifest["datasets_mutated"] == 0 + assert manifest["normalized_geotiff"]["crs"] == "EPSG:31370" + assert manifest["normalized_geotiff"]["band_count"] == 3 + assert Path(manifest["review_preview"]["path"]).read_bytes().startswith(b"\x89PNG") + + +def test_getmap_redirect_and_response_limits_fail_closed(tmp_path: Path) -> None: + request = MANAGER.map_request(report()) + with pytest.raises(RuntimeError, match="allowlist"): + MANAGER.fetch_map( + request, + timeout=5, + max_bytes=10_000_000, + opener=lambda *_args, **_kwargs: Response(raw_rgb_tiff(), "https://example.com/image.tif"), + ) + body = raw_rgb_tiff() + with pytest.raises(RuntimeError, match="release limit"): + MANAGER.fetch_map( + request, + timeout=5, + max_bytes=len(body) - 1, + opener=lambda *_args, **_kwargs: Response(body, request["url"]), + ) + + +def test_plan_and_artifact_tampering_are_rejected(tmp_path: Path) -> None: + args, path, plan, staged = staged_plan(tmp_path) + args.confirm_plan_sha256 = plan["plan_sha256"] + _, loaded, _ = MANAGER.load_staged_plan(args, "2025.04") + assert loaded == plan + + Path(staged["normalized_geotiff"]["path"]).write_bytes(b"tampered") + with pytest.raises(RuntimeError, match="no longer match"): + MANAGER.load_staged_plan(args, "2025.04") + + outside = tmp_path / "outside.json" + with pytest.raises(RuntimeError, match="outside the governed"): + MANAGER.governed_path(args, outside) + + +def test_review_requires_named_approval_exact_plan_and_unchanged_preflight(tmp_path: Path) -> None: + args, path, plan, staged = staged_plan(tmp_path) + manifest = MANAGER.validate_staged_artifacts(args, Path(staged["manifest_path"])) + with pytest.raises(RuntimeError, match="named --reviewer"): + MANAGER.build_review_evidence(args, path, plan, manifest) + args.approve = True + args.reviewer = "Jens" + review = MANAGER.build_review_evidence(args, path, plan, manifest) + assert review["review_preview_sha256"] == manifest["review_preview"]["sha256"] + assert review["review_sha256"] == MANAGER.canonical_sha256(review, "review_sha256") + + changed = report() + changed["capabilities"]["capabilities_sha256"] = "d" * 64 + with pytest.raises(RuntimeError, match="evidence changed"): + MANAGER.require_preflight_unchanged(plan, changed, require_local=True) + + +def test_approved_upload_uses_canonical_dataset_route_and_complete_provenance( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + args, path, plan, staged = staged_plan(tmp_path) + manifest = MANAGER.validate_staged_artifacts(args, Path(staged["manifest_path"])) + args.approve = True + args.reviewer = "Jens" + review = MANAGER.build_review_evidence(args, path, plan, manifest) + captured = {} + + class UploadResponse: + def __enter__(self): + return self + + def __exit__(self, *_args): + return None + + def read(self): + return json.dumps( + { + "data": { + "id": "dataset-1", + "status": "ready", + "source_name": MANAGER.SOURCE_NAME, + "source_version": "2025.04", + "checksum_sha256": manifest["normalized_geotiff"]["sha256"], + } + } + ).encode("utf-8") + + def fake_urlopen(request, timeout): + captured["url"] = request.full_url + captured["body"] = request.data + captured["timeout"] = timeout + return UploadResponse() + + monkeypatch.setattr(MANAGER, "urlopen", fake_urlopen) + dataset = MANAGER.upload_approved_dataset(args, plan, manifest, review) + + assert dataset["id"] == "dataset-1" + assert captured["url"].endswith(f"/projects/{args.project_id}/datasets/upload") + assert b'name="source_version"\r\n\r\n2025.04' in captured["body"] + assert b'name="temporal_series_key"' in captured["body"] + assert plan["plan_sha256"].encode("ascii") in captured["body"] + assert review["review_sha256"].encode("ascii") in captured["body"] + plan_path_json = json.dumps(str(MANAGER.default_plan_path(args, "2025.04")))[1:-1] + review_path_json = json.dumps(str(MANAGER.default_review_path(args, "2025.04")))[1:-1] + assert plan_path_json.encode("utf-8") in captured["body"] + assert review_path_json.encode("utf-8") in captured["body"] + + +def test_apply_target_must_be_loopback_api() -> None: + assert MANAGER.internal_api_url("http://127.0.0.1:8000/api/v1").endswith("/api/v1") + with pytest.raises(RuntimeError, match="local /api/v1"): + MANAGER.internal_api_url("http://192.168.10.150:1202/api/v1") + with pytest.raises(RuntimeError, match="local /api/v1"): + MANAGER.internal_api_url("http://127.0.0.1:8000/not-api") + + +def test_official_flight_dates_are_persisted_without_inventing_a_catalog_date() -> None: + dates = MANAGER.parse_flight_dates(["5/4/2025", "2025-04-06", "05-04-2025"]) + assert [value.date().isoformat() for value in dates] == ["2025-04-05", "2025-04-06"] + with pytest.raises(RuntimeError, match="not safely parseable"): + MANAGER.parse_flight_dates(["spring 2025"]) + + +def test_release_manager_is_packaged_and_compiled_by_readiness() -> None: + dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8") + readiness = (ROOT / "scripts" / "run_readiness_check.sh").read_text(encoding="utf-8") + + assert "COPY scripts/manage_orthophoto_release.py /app/scripts/manage_orthophoto_release.py" in dockerfile + assert "-m py_compile scripts/manage_orthophoto_release.py" in readiness diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index 11530c49..29b6836d 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -93,6 +93,7 @@ COPY scripts/provision_regional_bwk_natura2000.py /app/scripts/provision_regiona COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py COPY scripts/manage_alz_agriculture_release.py /app/scripts/manage_alz_agriculture_release.py COPY scripts/orthophoto_release_preflight.py /app/scripts/orthophoto_release_preflight.py +COPY scripts/manage_orthophoto_release.py /app/scripts/manage_orthophoto_release.py COPY scripts/provision_buildings_addresses_register.py /app/scripts/provision_buildings_addresses_register.py COPY scripts/provision_regional_timeseries.py /app/scripts/provision_regional_timeseries.py COPY scripts/geographic_scopes.py /app/scripts/geographic_scopes.py diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index f0e5c3d0..00c8f0eb 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -476,6 +476,17 @@ containment, sampled flight dates/years, local comparison state and `staging_permitted`. It performs no pixel request, upload, Job, Dataset write or legacy metadata rewrite. This operator script adds no public API contract. +Governed pixel promotion remains outside the HTTP request cycle in +`scripts/manage_orthophoto_release.py`. `plan` reruns that read-only preflight; +`stage` makes exactly one allowlisted bounded `Ortho` GetMap request and writes +only checksummed source/raster/preview evidence; `review` requires a named +approval; and `apply` requires the exact plan/review SHA-256 values. Apply +revalidates the remote identity and local comparison state, then delegates to +the existing `POST .../datasets/upload` contract with `dataset_type=raster`, +`source_name=digitaal_vlaanderen_orthophoto`, the official `YYYY.NN` +`source_version` and complete source/provenance metadata. No release endpoint, +provider URL parameter, Job type or alternate response envelope is added. + The endpoint accepts no arbitrary URL, feature query, area or layer. It does not fetch vector features, raster pixels or models, create jobs/datasets, write to PostGIS or trigger an import. The normal `source-freshness` endpoint remains diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 30c3977b..a1c0fee5 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -9767,3 +9767,35 @@ Boundary: backfill existing orthophotos. A future stage/review/apply coordinator must revalidate and retain the exact preflight identity before creating a new immutable raster Dataset with official `YYYY.NN` source version. + +## Sprint 231 - Governed orthophoto release promotion (2026-07-17) + +Implemented: +- Added `scripts/manage_orthophoto_release.py` with separate `plan`, `stage`, + named `review` and checksum-confirmed `apply` actions for one bounded current + orthophoto selection. No action is scheduled or browser-triggered. +- Reused the complete Sprint 230 preflight identity. Stage performs exactly one + allowlisted WMS `Ortho` GetMap, retains the exact source response, creates a + three-band EPSG:31370 GeoTIFF plus PNG preview and mutates no Dataset. +- Bound remote catalog/WMS/WCS/flight evidence, request identity, all staged + file hashes, reviewer and the final Dataset checksum into atomic persistent + evidence. Host/path drift, oversize responses, modified bytes, stale local + state and missing exact confirmations fail closed. +- Added a double-confirmed first official baseline transition for legacy + `most_recent_at_*` markers. Apply retains every legacy raster and uses only + the existing canonical upload/DatasetService transaction. +- Made the latest official `YYYY.NN` Dataset authoritative for source-catalog + comparison even when a newer-imported rolling marker also exists. No API, + migration, release table, Job type or frontend behavior changed. + +Validation so far: +- 38 focused Sprint 222/230/231 tests pass with deprecations treated as errors. + Coverage includes official-edition ordering, first-baseline authorization, + GetMap allowlisting/limits, RGB/CRS normalization, preview generation, + tampering, preflight drift, named review, loopback-only apply and complete + upload provenance. + +Next: +- Run complete readiness, deploy the packaged operator to Tower, inspect the + live bounded Mol review preview and only then apply the first official + `2025.04` immutable baseline with explicit review evidence. diff --git a/docs/DATABASE_IMPLEMENTATION_PLAN.md b/docs/DATABASE_IMPLEMENTATION_PLAN.md index 828ceb60..2e944ebc 100644 --- a/docs/DATABASE_IMPLEMENTATION_PLAN.md +++ b/docs/DATABASE_IMPLEMENTATION_PLAN.md @@ -291,13 +291,15 @@ annual Dataset, DatasetVersion and vector_features records with `source_version=-definitive`. Earlier annual snapshots are retained and provisional v1/v2 publications cannot create rows. -The current-orthophoto release preflight likewise adds no lifecycle table or -migration. It is read-only and creates neither Dataset nor Job. It compares the -existing local `source_version` with the official `YYYY.NN` catalog edition -and verifies WMS/WCS/flight-day evidence for one bounded selection. A later -governed pixel apply must still create an immutable raster Dataset plus -DatasetVersion through DatasetService and retain that exact edition/evidence; -direct metadata backfill of legacy `most_recent_at_*` rows is prohibited. +Current-orthophoto release management likewise adds no lifecycle table or +migration. Preflight is read-only; stage and named review are filesystem-only. +Approved apply uploads the exact checksummed EPSG:31370 GeoTIFF through the +existing dataset endpoint and DatasetService transaction, creating one normal +immutable raster Dataset and DatasetVersion with official `YYYY.NN` +`source_version`, temporal flight-date evidence and release provenance. +Official editions take precedence over legacy rolling markers in catalog +comparison, but direct metadata backfill, update or deletion of those legacy +rows remains prohibited. ## Geometry normalization diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md index 66fb6f11..569b8ad5 100644 --- a/docs/DATA_SOURCES.md +++ b/docs/DATA_SOURCES.md @@ -24,7 +24,7 @@ aanvraag. GeoIntel verzint geen historische pixelopnamedatum. Bronnen: - https://www.vlaanderen.be/datavindplaats/catalogus/orthofotomozaiek-middenschalig-winteropnamen-kleur-meest-recent-vlaanderen - https://www.vlaanderen.be/digitaal-vlaanderen/onze-diensten-en-platformen/luchtopnamen/gebruik-orthofotomozaieken -Voor een toekomstige rolling-releasebeslissing gebruikt +Voor een rolling-releasebeslissing gebruikt `scripts/orthophoto_release_preflight.py` uitsluitend metadata. Het bindt de lokale `most_recent`-productvariant aan de officiële ISO-editie en exacte WMS- capabilitieshash, controleert het EPSG:31370/15 cm/driebanden-rasterdomein via @@ -39,6 +39,17 @@ union. Een selectie met ontbrekende contourpunten, meerdere/afwijkende vluchtjaren, gewijzigde service-identiteit of een niet-vergelijkbare lokale `most_recent_at_*` marker is niet stagebaar. +`scripts/manage_orthophoto_release.py` voert daarna uitsluitend op expliciet +operatorverzoek `plan`, `stage`, `review` en `apply` uit. Stage haalt exact een +begrensde `Ortho`-GetMap op, bewaart bronresponse, genormaliseerde RGB-GeoTIFF +en review-PNG met checksums, maar schrijft geen Dataset. Review koppelt een +benoemde goedkeuring aan exact die bytes. Apply valideert preflight, plan, +review en lokale bronstaat opnieuw en gebruikt vervolgens de bestaande +Dataset-uploadroute. De nieuwe Dataset krijgt de officiële `YYYY.NN`-editie; +oude `most_recent_at_*` rasters blijven onveranderd bewaard. De eerste overgang +vereist een aparte baselinevlag plus de exacte oude marker. Er is geen +scheduler, browserfetch of automatische vervanging. + Dit document verzamelt concrete databronnen voor GeoIntel Kempen. ## Cross-domain official area profile @@ -189,7 +200,7 @@ gebeurd. | GRB gebouwen/wegen/water/percelen | operationele, expliciete plan-stage-apply refresh met onveranderlijke snapshots | alleen een nieuw officieel gedateerd cataloguseditie na operatorbevestiging ophalen | | Statbel bevolking | jaarlijkse, expliciete edities in één tijdreeks; officiële DCAT-releaseprobe | een nieuwe publicatie alleen na schema-, sectorgeometrie- en totalencontrole toevoegen | | ALZ landbouwgebruikspercelen | definitieve jaarlijkse edities 2008-2025; expliciete publicatieprobe en plan-stage-review-apply promotie; metricvergelijking zonder objectlineage | alleen een nieuwere definitieve v3-editie na gestagede schema-/codelijst-/scopecontrole en benoemde review toevoegen | -| orthofoto | vaste lokale opname per expliciete analysezone; read-only releasepreflight voor variant, officiële editie, exact WCS-domein en begrensd vluchtjaarbewijs | eerst officiële lokale editieprovenance vastleggen; daarna pas een afzonderlijke menselijke pixel-stage/apply-flow bouwen | +| orthofoto | vaste lokale analyseopnamen plus operationele preflight en benoemde plan-stage-review-apply-promotie voor een officiële `YYYY.NN`-editie | alleen een nieuwere officiële editie als afzonderlijke immutable Dataset promoveren na verse evidence en review | | landgebruik, thematische rasters, DHMV en VMM-scenario's | vaste product-/scenario-edities, geen rolling snapshot | alleen een nieuwe gedocumenteerde producteditie als afzonderlijke Dataset verwerven | | bodemkaart en historische kaarten | historische referentie-editie | niet als verouderde actuele bron labelen; alleen vervangen bij een officiële inhoudelijke heruitgave | | BWK/Natura 2000 en gebouwen-/adressenregister | expliciete actuele snapshot met eigen methodologische betekenis | eerst een stabiele officiële editieprobe en bron-specifieke reconciliatiecontrole toevoegen | diff --git a/docs/DATA_SPECIFICATION.md b/docs/DATA_SPECIFICATION.md index 81400068..c7f923b3 100644 --- a/docs/DATA_SPECIFICATION.md +++ b/docs/DATA_SPECIFICATION.md @@ -274,12 +274,16 @@ layer, observation label, `observed_at`, optional `valid_from`/`valid_to`, temporal granularity, request/spatial hash, attribution and a limitation that states whether the product is annual, multi-year or merely most recent. -A future rolling `most_recent` import must also retain the exact official +A governed rolling `most_recent` release import must also retain the exact official `YYYY.NN` edition and the preflight identities for WMS capabilities, WCS coverage description, selected EPSG:31370 domain and sampled flight year. A legacy `most_recent_at_` value is acquisition timing, not an official -edition, and cannot be promoted or compared as if it were one. The read-only -preflight creates no Dataset and does not retroactively rewrite that evidence. +edition, and cannot be compared as if it were one. The read-only preflight +creates no Dataset. Stage retains the exact source response, normalized +three-band EPSG:31370 GeoTIFF and review preview. Apply requires exact plan and +review hashes, persists sampled official flight dates as the temporal evidence +range and creates a new Dataset/DatasetVersion through DatasetService. It does +not retroactively rewrite or delete legacy rows. ### Hydrological station observations diff --git a/docs/STORAGE_ARCHITECTURE.md b/docs/STORAGE_ARCHITECTURE.md index 3ff75023..18014b89 100644 --- a/docs/STORAGE_ARCHITECTURE.md +++ b/docs/STORAGE_ARCHITECTURE.md @@ -148,12 +148,27 @@ in source/provenance metadata. Browser PNG rendering is derived on request and does not replace the stored GeoTIFF. The orthophoto release preflight writes no source file, raster or database row. -Its JSON stdout may be retained by an operator as review evidence, but it is -not itself staging authorization. The report binds official WMS and WCS XML -hashes, the exact selected domain and hashed flight-day sample evidence. A -future pixel stage must persist and revalidate that identity separately before -DatasetService is called; existing `most_recent_at_*` raster metadata is not -silently rewritten. +Its JSON stdout is not staging authorization. Governed release evidence is +retained separately per scope, official edition and exact selection hash: + +```text +storage/operator-evidence/orthophoto-release/{scope}/{YYYY.NN}/{selection-hash}/ + official-wms-response.tif + orthophoto_{YYYY.NN}_{selection-hash}.tif + review-preview.png + staged-manifest.json + staged-plan.json + review-evidence.json + applied-evidence.json +``` + +The manifest binds the one bounded source response, normalized three-band +EPSG:31370 GeoTIFF and PNG preview. The plan also binds WMS/WCS/catalog and +flight-day preflight identities; review binds a named approval; applied +evidence binds both to the immutable Dataset id and checksum. Paths outside +this root, changed files and changed provider/local state fail closed. Only the +normalized GeoTIFF enters ordinary Dataset storage through DatasetService. +Existing `most_recent_at_*` raster metadata is never rewritten. DHMV II DTM/DSM outputs are also normal raster Dataset files. The provider WCS returns multipart coverage data; GeoIntel retains response and extracted diff --git a/docs/TODO.md b/docs/TODO.md index d86a92e3..8894c0b9 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -692,6 +692,6 @@ This file now starts with the current implementation status. Older preparation/b campaign-snapshot and definitive-archive publication contract. - [x] Add a read-only orthophoto preflight for product variant, official edition, exact WCS selected-area domain and deterministic flight-year points. -- [ ] Keep orthophoto pixel refresh manual and blocked until a separate - plan-stage-review-apply flow can retain the passed preflight identity and - create a new immutable Dataset with official `YYYY.NN` source version. +- [x] Keep orthophoto pixel refresh manual through a separate + plan-stage-review-apply flow that retains the passed preflight identity and + creates a new immutable Dataset with official `YYYY.NN` source version. diff --git a/scripts/manage_orthophoto_release.py b/scripts/manage_orthophoto_release.py new file mode 100644 index 00000000..ff23ac72 --- /dev/null +++ b/scripts/manage_orthophoto_release.py @@ -0,0 +1,1001 @@ +#!/usr/bin/env python3 +"""Govern one bounded current-orthophoto release from plan through apply. + +The operator has four deliberately separate actions. Planning is read-only, +staging retains source and normalized raster evidence without database writes, +review records a named approval, and apply uploads the exact approved GeoTIFF +through GeoIntel's canonical DatasetService route. No action refreshes data +automatically. +""" + +from __future__ import annotations + +import argparse +from datetime import datetime, time, timezone +from hashlib import sha256 +import json +import math +import os +from pathlib import Path +import re +import sys +from typing import Any, Callable +from urllib.error import HTTPError, URLError +from urllib.parse import parse_qs, urlencode, urlparse +from urllib.request import Request, urlopen +import uuid + +import orthophoto_release_preflight as preflight + + +DEFAULT_API_URL = "http://127.0.0.1:8000/api/v1" +DEFAULT_SCOPE = preflight.DEFAULT_SCOPE +DEFAULT_EVIDENCE_ROOT = Path("/app/storage/operator-evidence/orthophoto-release") +SOURCE_NAME = preflight.SOURCE_NAME +WMS_BASE_URL = f"https://{preflight.WMS_HOST}{preflight.WMS_PATH}" +LAYER = "Ortho" +OUTPUT_CRS = "EPSG:31370" +OUTPUT_RESOLUTION_M = 1.0 +MAX_RESPONSE_MB = 32 +MAX_PREVIEW_SIDE = 640 +SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") +SAFE_EDITION_PATTERN = preflight.EDITION_PATTERN +ACTIONABLE_STATUSES = {"not_loaded", "update_available"} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Governed orthophoto release: plan, stage, review, then checksum-confirmed apply." + ) + parser.add_argument("action", choices=("plan", "stage", "review", "apply")) + parser.add_argument("--project-id", required=True, help="GeoIntel project UUID for the governed scope") + parser.add_argument("--scope", choices=(DEFAULT_SCOPE,), default=DEFAULT_SCOPE) + parser.add_argument("--api-url", default=os.environ.get("GEOINTEL_API_URL", DEFAULT_API_URL)) + parser.add_argument( + "--bbox", + nargs=4, + type=float, + metavar=("MIN_LON", "MIN_LAT", "MAX_LON", "MAX_LAT"), + required=True, + help="Exact bounded EPSG:4326 selection; projected sides must be 128-1024 metres", + ) + parser.add_argument("--area-id", help="Optional existing Area UUID to bind to the imported Dataset") + parser.add_argument("--confirm-edition", help="Exact official edition, for example 2025.04") + parser.add_argument("--confirm-plan-sha256", help="Exact staged-plan hash required for review/apply") + parser.add_argument("--confirm-review-sha256", help="Exact review-evidence hash required for apply") + parser.add_argument("--approve", action="store_true", help="Explicitly approve staged evidence during review") + parser.add_argument("--reviewer", help="Named human/operator approving the staged evidence") + parser.add_argument("--review-note", default="", help="Optional bounded review note") + parser.add_argument( + "--establish-official-baseline", + action="store_true", + help="Explicitly replace a non-comparable rolling marker with the first official edition baseline", + ) + parser.add_argument( + "--confirm-local-version", + help="Exact non-comparable local marker required with --establish-official-baseline", + ) + parser.add_argument("--plan-path", type=Path, help="Override the governed staged-plan path") + parser.add_argument("--review-path", type=Path, help="Override the governed review-evidence path") + parser.add_argument( + "--evidence-root", + type=Path, + default=Path(os.environ.get("GEOINTEL_ORTHOPHOTO_RELEASE_EVIDENCE_ROOT", DEFAULT_EVIDENCE_ROOT)), + ) + parser.add_argument("--refresh-catalog", action="store_true", help="Bypass the short catalog cache") + parser.add_argument("--api-timeout", type=int, default=180) + parser.add_argument("--wms-timeout", type=int, default=60) + parser.add_argument("--import-timeout", type=int, default=600) + parser.add_argument("--max-response-mb", type=int, default=MAX_RESPONSE_MB) + return parser.parse_args() + + +def canonical_sha256(payload: dict[str, Any], hash_field: str) -> str: + content = {key: value for key, value in payload.items() if key != hash_field} + encoded = json.dumps(content, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + return sha256(encoded).hexdigest() + + +def sha256_file(path: Path) -> str: + digest = sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def write_bytes(path: Path, content: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".partial") + temporary.write_bytes(content) + temporary.replace(path) + + +def write_json(path: Path, payload: dict[str, Any]) -> None: + write_bytes( + path, + (json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n").encode("utf-8"), + ) + + +def selection_key(bbox: list[float]) -> str: + canonical = json.dumps([round(float(value), 8) for value in bbox], separators=(",", ":")).encode("ascii") + return sha256(canonical).hexdigest()[:16] + + +def evidence_directory(args: argparse.Namespace, edition: str) -> Path: + return args.evidence_root / args.scope / edition / selection_key(list(args.bbox)) + + +def default_plan_path(args: argparse.Namespace, edition: str) -> Path: + return evidence_directory(args, edition) / "staged-plan.json" + + +def default_review_path(args: argparse.Namespace, edition: str) -> Path: + return evidence_directory(args, edition) / "review-evidence.json" + + +def governed_path(args: argparse.Namespace, path: Path) -> Path: + resolved = path.resolve() + if not resolved.is_relative_to(args.evidence_root.resolve()): + raise RuntimeError(f"Orthophoto release evidence is outside the governed root: {path}") + return path + + +def internal_api_url(api_url: str) -> str: + parsed = urlparse(api_url.rstrip("/")) + if ( + parsed.scheme not in {"http", "https"} + or parsed.hostname not in {"127.0.0.1", "localhost", "::1"} + or parsed.username + or parsed.password + or parsed.fragment + or parsed.path.rstrip("/") != "/api/v1" + ): + raise RuntimeError("Stage/apply must run inside GeoIntel against the local /api/v1 backend") + return api_url.rstrip("/") + + +def run_preflight(args: argparse.Namespace, *, refresh: bool) -> dict[str, Any]: + namespace = argparse.Namespace( + project_id=args.project_id, + scope=args.scope, + api_url=args.api_url, + bbox=list(args.bbox), + refresh_catalog=refresh, + api_timeout=args.api_timeout, + wms_timeout=args.wms_timeout, + ) + return preflight.run_preflight(namespace) + + +def preflight_identity(report: dict[str, Any]) -> dict[str, Any]: + release = report["release"] + coverage = report["flight_day_coverage"] + identity = { + "project_id": report["project_id"], + "scope": report["scope"], + "product": report["product"], + "remote_release": { + key: release.get(key) + for key in ( + "remote_edition", + "remote_year", + "metadata_identifier", + "metadata_url", + "remote_title", + "remote_modified_at", + "remote_published_at", + "capabilities_url", + "capabilities_sha256", + ) + }, + "capabilities": report["capabilities"], + "coverage_domain": report["coverage_domain"], + "selection": report["selection"], + "flight_day_coverage": { + key: coverage.get(key) + for key in ( + "mode", + "sample_count", + "grid_columns", + "grid_rows", + "covered_sample_count", + "sample_coverage_ratio", + "flight_dates", + "flight_years", + "feature_ids", + "sample_evidence_sha256", + "claim_boundary", + ) + }, + "flight_year_matches_release": report["flight_year_matches_release"], + } + identity["preflight_identity_sha256"] = canonical_sha256(identity, "preflight_identity_sha256") + return identity + + +def require_edition(args: argparse.Namespace, report: dict[str, Any]) -> str: + edition = str(report["release"]["remote_edition"]) + if not SAFE_EDITION_PATTERN.fullmatch(edition) or args.confirm_edition != edition: + raise RuntimeError(f"Explicit --confirm-edition {edition} is required") + return edition + + +def authorize_stage(args: argparse.Namespace, report: dict[str, Any]) -> dict[str, Any]: + release = report["release"] + status = release["status"] + local_version = release.get("local_source_version") + if report.get("staging_permitted") and status in ACTIONABLE_STATUSES: + return {"mode": "normal_release", "local_source_version": local_version} + if status == "blocked_local_version": + if not args.establish_official_baseline or args.confirm_local_version != local_version: + raise RuntimeError( + "The first official baseline requires --establish-official-baseline and exact " + f"--confirm-local-version {local_version}" + ) + if not report.get("flight_year_matches_release"): + raise RuntimeError("The bounded flight-day evidence does not match the official edition year") + return {"mode": "explicit_legacy_baseline_transition", "local_source_version": local_version} + raise RuntimeError(f"Orthophoto release is not safely stageable: {status}") + + +def map_request(report: dict[str, Any]) -> dict[str, Any]: + selection = report["selection"] + width = max(1, math.ceil(float(selection["width_m"]) / OUTPUT_RESOLUTION_M)) + height = max(1, math.ceil(float(selection["height_m"]) / OUTPUT_RESOLUTION_M)) + bbox = [float(value) for value in selection["bbox_epsg31370"]] + params = { + "SERVICE": "WMS", + "VERSION": "1.3.0", + "REQUEST": "GetMap", + "LAYERS": LAYER, + "STYLES": "", + "FORMAT": "image/tiff", + "CRS": OUTPUT_CRS, + "BBOX": ",".join(f"{value:.3f}" for value in bbox), + "WIDTH": str(width), + "HEIGHT": str(height), + } + request_url = f"{WMS_BASE_URL}?{urlencode(params)}" + request_identity = { + "url": request_url, + "layer": LAYER, + "crs": OUTPUT_CRS, + "bbox_epsg31370": bbox, + "bbox_epsg4326": list(selection["bbox_epsg4326"]), + "width": width, + "height": height, + "resolution_m": OUTPUT_RESOLUTION_M, + } + request_identity["request_sha256"] = canonical_sha256(request_identity, "request_sha256") + return request_identity + + +def validate_map_url(url: str, expected: dict[str, Any]) -> None: + parsed = urlparse(url) + if ( + parsed.scheme.lower() != "https" + or parsed.hostname != preflight.WMS_HOST + or parsed.port not in (None, 443) + or parsed.username + or parsed.password + or parsed.fragment + or parsed.path.rstrip("/").lower() != preflight.WMS_PATH.lower() + ): + raise RuntimeError("Orthophoto GetMap URL is outside the official allowlist") + query = {key.upper(): values for key, values in parse_qs(parsed.query, keep_blank_values=True).items()} + required = { + "SERVICE": "WMS", + "VERSION": "1.3.0", + "REQUEST": "GetMap", + "LAYERS": LAYER, + "STYLES": "", + "FORMAT": "image/tiff", + "CRS": OUTPUT_CRS, + "BBOX": ",".join(f"{value:.3f}" for value in expected["bbox_epsg31370"]), + "WIDTH": str(expected["width"]), + "HEIGHT": str(expected["height"]), + } + if set(query) != set(required) or any(query.get(key) != [value] for key, value in required.items()): + raise RuntimeError("Orthophoto GetMap URL does not match the approved bounded request") + + +def fetch_map( + request_identity: dict[str, Any], + *, + timeout: int, + max_bytes: int, + opener: Callable[..., Any] | None = None, +) -> tuple[bytes, str, str]: + validate_map_url(request_identity["url"], request_identity) + request = Request( + request_identity["url"], + headers={"Accept": "image/tiff", "User-Agent": "GeoIntel-orthophoto-release/1.0"}, + ) + fetch = opener or urlopen + try: + response = fetch(request, timeout=timeout) + with response: + final_url = response.geturl() + validate_map_url(final_url, request_identity) + content_type = str(response.headers.get("Content-Type") or "").split(";", 1)[0].strip().lower() + declared = response.headers.get("Content-Length") + if declared: + try: + if int(declared) > max_bytes: + raise RuntimeError("Official orthophoto response exceeds the configured release limit") + except ValueError as exc: + raise RuntimeError("Official orthophoto response has an invalid Content-Length") from exc + content = response.read(max_bytes + 1) + except RuntimeError: + raise + except HTTPError as exc: + raise RuntimeError(f"Official orthophoto WMS returned HTTP {exc.code}") from exc + except URLError as exc: + raise RuntimeError(f"Official orthophoto WMS is unreachable: {exc.reason}") from exc + if len(content) > max_bytes: + raise RuntimeError("Official orthophoto response exceeds the configured release limit") + if content_type not in {"image/tiff", "image/geotiff", "image/x-geotiff"}: + raise RuntimeError(f"Official orthophoto WMS returned unsupported content type {content_type or 'unknown'}") + return content, content_type, final_url + + +def normalize_and_preview( + raw_content: bytes, + request_identity: dict[str, Any], + geotiff_path: Path, + preview_path: Path, +) -> dict[str, Any]: + try: + import numpy as np + from PIL import Image + import rasterio + from rasterio.enums import Resampling + from rasterio.errors import NotGeoreferencedWarning + from rasterio.io import MemoryFile + from rasterio.transform import from_bounds + import warnings + except ImportError as exc: + raise RuntimeError("Rasterio, NumPy and Pillow are required for orthophoto release staging") from exc + + geotiff_path.parent.mkdir(parents=True, exist_ok=True) + temporary = geotiff_path.with_suffix(geotiff_path.suffix + ".partial") + try: + with MemoryFile(raw_content) as source_memory: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", NotGeoreferencedWarning) + with source_memory.open() as source: + if ( + source.width != request_identity["width"] + or source.height != request_identity["height"] + or source.count < 3 + or any(dtype != "uint8" for dtype in source.dtypes[:3]) + ): + raise RuntimeError("Official orthophoto response does not match the approved RGB dimensions") + rgb = source.read((1, 2, 3)) + profile = source.profile.copy() + profile.update( + driver="GTiff", + count=3, + dtype="uint8", + crs=OUTPUT_CRS, + transform=from_bounds( + *request_identity["bbox_epsg31370"], + source.width, + source.height, + ), + nodata=None, + compress="deflate", + tiled=False, + ) + for key in ("blockxsize", "blockysize", "photometric", "interleave"): + profile.pop(key, None) + with rasterio.open(temporary, "w", **profile) as output: + output.write(rgb) + output.update_tags( + source=f"Digitaal Vlaanderen WMS {LAYER}", + source_url=request_identity["url"], + attribution="Digitaal Vlaanderen", + acquisition="governed_orthophoto_release", + ) + temporary.replace(geotiff_path) + + with rasterio.open(geotiff_path) as source: + scale = min(1.0, MAX_PREVIEW_SIDE / max(source.width, source.height)) + preview_width = max(1, round(source.width * scale)) + preview_height = max(1, round(source.height * scale)) + preview = source.read( + (1, 2, 3), + out_shape=(3, preview_height, preview_width), + resampling=Resampling.bilinear, + ) + image = Image.fromarray(np.moveaxis(preview, 0, 2)) + preview_temporary = preview_path.with_suffix(preview_path.suffix + ".partial") + image.save(preview_temporary, format="PNG", optimize=True) + preview_temporary.replace(preview_path) + return { + "crs": source.crs.to_string() if source.crs else None, + "width": source.width, + "height": source.height, + "band_count": source.count, + "dtypes": list(source.dtypes), + "bounds": [float(value) for value in source.bounds], + "transform": [float(value) for value in source.transform[:6]], + "preview_width": preview_width, + "preview_height": preview_height, + } + except RuntimeError: + temporary.unlink(missing_ok=True) + raise + except Exception as exc: + temporary.unlink(missing_ok=True) + raise RuntimeError(f"Official orthophoto response could not be normalized: {exc}") from exc + + +def stage_artifacts( + args: argparse.Namespace, + report: dict[str, Any], + request_identity: dict[str, Any], + *, + opener: Callable[..., Any] | None = None, +) -> dict[str, Any]: + edition = str(report["release"]["remote_edition"]) + directory = governed_path(args, evidence_directory(args, edition)) + raw_path = directory / "official-wms-response.tif" + geotiff_path = directory / f"orthophoto_{edition}_{selection_key(list(args.bbox))}.tif" + preview_path = directory / "review-preview.png" + manifest_path = directory / "staged-manifest.json" + max_bytes = args.max_response_mb * 1024 * 1024 + content, content_type, final_url = fetch_map( + request_identity, + timeout=args.wms_timeout, + max_bytes=max_bytes, + opener=opener, + ) + write_bytes(raw_path, content) + raster = normalize_and_preview(content, request_identity, geotiff_path, preview_path) + payload: dict[str, Any] = { + "schema_version": 1, + "status": "staged", + "created_at": datetime.now(timezone.utc).isoformat(), + "project_id": args.project_id, + "scope": args.scope, + "edition": edition, + "preflight_identity_sha256": preflight_identity(report)["preflight_identity_sha256"], + "request": request_identity, + "response": { + "final_url": final_url, + "content_type": content_type, + "path": str(raw_path), + "size_bytes": len(content), + "sha256": sha256(content).hexdigest(), + }, + "normalized_geotiff": { + "path": str(geotiff_path), + "size_bytes": geotiff_path.stat().st_size, + "sha256": sha256_file(geotiff_path), + **raster, + }, + "review_preview": { + "path": str(preview_path), + "size_bytes": preview_path.stat().st_size, + "sha256": sha256_file(preview_path), + "width": raster["preview_width"], + "height": raster["preview_height"], + }, + "pixel_request_count": 1, + "datasets_mutated": 0, + } + payload["manifest_sha256"] = canonical_sha256(payload, "manifest_sha256") + write_json(manifest_path, payload) + return {"manifest_path": str(manifest_path), **payload} + + +def validate_staged_artifacts(args: argparse.Namespace, manifest_path: Path) -> dict[str, Any]: + governed_path(args, manifest_path) + if not manifest_path.is_file(): + raise RuntimeError(f"Staged orthophoto manifest is missing: {manifest_path}") + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + if manifest.get("manifest_sha256") != canonical_sha256(manifest, "manifest_sha256"): + raise RuntimeError("Staged orthophoto manifest checksum is invalid") + if ( + manifest.get("status") != "staged" + or manifest.get("project_id") != args.project_id + or manifest.get("scope") != args.scope + ): + raise RuntimeError("Staged orthophoto manifest identity is invalid") + for section, expected_content_type in ( + ("response", None), + ("normalized_geotiff", None), + ("review_preview", None), + ): + evidence = manifest.get(section) + if not isinstance(evidence, dict) or not isinstance(evidence.get("path"), str): + raise RuntimeError(f"Staged orthophoto {section} evidence is incomplete") + path = governed_path(args, Path(evidence["path"])) + if ( + not path.is_file() + or path.stat().st_size != evidence.get("size_bytes") + or sha256_file(path) != evidence.get("sha256") + ): + raise RuntimeError(f"Staged orthophoto {section} bytes no longer match their evidence") + raster = manifest["normalized_geotiff"] + request_identity = manifest["request"] + if ( + manifest.get("pixel_request_count") != 1 + or manifest.get("datasets_mutated") != 0 + or raster.get("crs") != OUTPUT_CRS + or raster.get("width") != request_identity.get("width") + or raster.get("height") != request_identity.get("height") + or raster.get("band_count") != 3 + or raster.get("dtypes") != ["uint8", "uint8", "uint8"] + ): + raise RuntimeError("Staged orthophoto raster no longer satisfies the release contract") + validate_map_url(request_identity["url"], request_identity) + return manifest + + +def build_staged_plan( + args: argparse.Namespace, + report: dict[str, Any], + authorization: dict[str, Any], + staged: dict[str, Any], +) -> dict[str, Any]: + identity = preflight_identity(report) + payload: dict[str, Any] = { + "schema_version": 1, + "status": "staged", + "created_at": datetime.now(timezone.utc).isoformat(), + "project_id": args.project_id, + "scope": args.scope, + "area_id": args.area_id, + "edition": report["release"]["remote_edition"], + "local_source_version_before_apply": report["release"].get("local_source_version"), + "baseline_authorization": authorization, + "preflight_identity": identity, + "catalog_checked_at": report["release"].get("catalog_checked_at"), + "staged_manifest_path": staged["manifest_path"], + "staged_manifest_sha256": staged["manifest_sha256"], + "review_required": True, + "apply_requires_plan_sha256": True, + "apply_requires_review_sha256": True, + "automatic_stage": False, + "automatic_import": False, + "destructive_replacement": False, + "existing_snapshots_retained": True, + } + payload["plan_sha256"] = canonical_sha256(payload, "plan_sha256") + return payload + + +def load_staged_plan(args: argparse.Namespace, edition: str) -> tuple[Path, dict[str, Any], dict[str, Any]]: + path = governed_path(args, args.plan_path or default_plan_path(args, edition)) + if not path.is_file(): + raise RuntimeError(f"Staged orthophoto plan is missing: {path}") + payload = json.loads(path.read_text(encoding="utf-8")) + actual_sha = canonical_sha256(payload, "plan_sha256") + if payload.get("plan_sha256") != actual_sha: + raise RuntimeError("Staged orthophoto plan checksum is invalid") + if args.confirm_plan_sha256 != actual_sha: + raise RuntimeError(f"Explicit --confirm-plan-sha256 {actual_sha} is required") + if ( + payload.get("status") != "staged" + or payload.get("project_id") != args.project_id + or payload.get("scope") != args.scope + or payload.get("area_id") != args.area_id + or payload.get("edition") != edition + ): + raise RuntimeError("Staged orthophoto plan identity is invalid") + manifest_path = Path(str(payload.get("staged_manifest_path") or "")) + manifest = validate_staged_artifacts(args, manifest_path) + if manifest.get("manifest_sha256") != payload.get("staged_manifest_sha256"): + raise RuntimeError("Staged orthophoto manifest no longer matches the plan") + if manifest.get("preflight_identity_sha256") != payload["preflight_identity"].get("preflight_identity_sha256"): + raise RuntimeError("Staged orthophoto preflight identity no longer matches the plan") + return path, payload, manifest + + +def require_preflight_unchanged(plan: dict[str, Any], report: dict[str, Any], *, require_local: bool) -> None: + if plan.get("preflight_identity") != preflight_identity(report): + raise RuntimeError("Official orthophoto evidence changed; create and review a new staged plan") + if require_local and report["release"].get("local_source_version") != plan.get("local_source_version_before_apply"): + raise RuntimeError("Local orthophoto edition changed after staging; create and review a new plan") + + +def build_review_evidence( + args: argparse.Namespace, + plan_path: Path, + plan: dict[str, Any], + manifest: dict[str, Any], +) -> dict[str, Any]: + reviewer = str(args.reviewer or "").strip() + note = str(args.review_note or "").strip() + if not args.approve or len(reviewer) < 2 or len(reviewer) > 120: + raise RuntimeError("Review requires --approve and a named --reviewer between 2 and 120 characters") + if len(note) > 1000: + raise RuntimeError("Review note must not exceed 1000 characters") + payload: dict[str, Any] = { + "schema_version": 1, + "status": "approved", + "reviewed_at": datetime.now(timezone.utc).isoformat(), + "reviewer": reviewer, + "review_note": note or None, + "project_id": args.project_id, + "scope": args.scope, + "edition": plan["edition"], + "staged_plan_path": str(plan_path), + "staged_plan_sha256": plan["plan_sha256"], + "staged_manifest_sha256": manifest["manifest_sha256"], + "review_preview_path": manifest["review_preview"]["path"], + "review_preview_sha256": manifest["review_preview"]["sha256"], + "reviewed_checks": [ + "official_catalog_and_service_identity", + "wcs_native_domain_and_rgb_contract", + "bounded_flight_day_evidence", + "exact_getmap_request_and_source_checksum", + "normalized_epsg31370_rgb_geotiff", + "visual_preview", + "temporal_and_spatial_provenance", + "immutable_dataset_apply", + ], + } + payload["review_sha256"] = canonical_sha256(payload, "review_sha256") + return payload + + +def load_review_evidence( + args: argparse.Namespace, + edition: str, + plan: dict[str, Any], + manifest: dict[str, Any], +) -> tuple[Path, dict[str, Any]]: + path = governed_path(args, args.review_path or default_review_path(args, edition)) + if not path.is_file(): + raise RuntimeError(f"Approved orthophoto review evidence is missing: {path}") + payload = json.loads(path.read_text(encoding="utf-8")) + actual_sha = canonical_sha256(payload, "review_sha256") + if payload.get("review_sha256") != actual_sha: + raise RuntimeError("Orthophoto review evidence checksum is invalid") + if args.confirm_review_sha256 != actual_sha: + raise RuntimeError(f"Explicit --confirm-review-sha256 {actual_sha} is required") + if ( + payload.get("status") != "approved" + or payload.get("project_id") != args.project_id + or payload.get("scope") != args.scope + or payload.get("edition") != edition + or payload.get("staged_plan_sha256") != plan.get("plan_sha256") + or payload.get("staged_manifest_sha256") != manifest.get("manifest_sha256") + or payload.get("review_preview_sha256") != manifest["review_preview"].get("sha256") + or not str(payload.get("reviewer") or "").strip() + ): + raise RuntimeError("Orthophoto review evidence does not authorize this staged plan") + return path, payload + + +def parse_flight_dates(values: list[str]) -> list[datetime]: + parsed: list[datetime] = [] + for value in values: + candidate = str(value).strip() + result = None + for date_format in ("%d/%m/%Y", "%Y-%m-%d", "%d-%m-%Y"): + try: + result = datetime.strptime(candidate, date_format).date() + break + except ValueError: + continue + if result is None: + raise RuntimeError(f"Official flight date is not safely parseable: {candidate}") + parsed.append(datetime.combine(result, time.min, tzinfo=timezone.utc)) + if not parsed: + raise RuntimeError("Official flight-day evidence has no dates") + return sorted(set(parsed)) + + +def api_data(api_url: str, path: str, timeout: int) -> dict[str, Any]: + endpoint = f"{api_url.rstrip('/')}/{path.lstrip('/')}" + request = Request(endpoint, headers={"Accept": "application/json", "User-Agent": "GeoIntel-orthophoto-release/1.0"}) + try: + with urlopen(request, timeout=timeout) as response: + payload = json.load(response) + except HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"GeoIntel API returned HTTP {exc.code}: {body[-1000:]}") from exc + except URLError as exc: + raise RuntimeError(f"GeoIntel API is unreachable: {exc.reason}") from exc + if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict): + raise RuntimeError("GeoIntel API response is not a canonical data envelope") + return payload["data"] + + +def project_datasets(args: argparse.Namespace) -> list[dict[str, Any]]: + items: list[dict[str, Any]] = [] + offset = 0 + total: int | None = None + while total is None or offset < total: + page = api_data( + args.api_url, + f"projects/{args.project_id}/datasets?limit=200&offset={offset}", + args.api_timeout, + ) + page_items = list(page.get("items") or []) + page_total = int(page.get("total") or 0) + if total is None: + total = page_total + elif total != page_total: + raise RuntimeError("Dataset pagination total changed while checking orthophoto idempotence") + items.extend(page_items) + if not page_items: + break + offset += len(page_items) + if total is not None and len(items) != total: + raise RuntimeError(f"Dataset pagination returned {len(items)} of {total} items") + return items + + +def matching_dataset(args: argparse.Namespace, plan: dict[str, Any], manifest: dict[str, Any]) -> dict[str, Any] | None: + matches = [] + for dataset in project_datasets(args): + provenance = dataset.get("provenance_metadata") or {} + if ( + dataset.get("source_name") == SOURCE_NAME + and dataset.get("source_version") == plan["edition"] + and dataset.get("status") == "ready" + and provenance.get("release_plan_sha256") == plan["plan_sha256"] + and provenance.get("normalized_geotiff_sha256") == manifest["normalized_geotiff"]["sha256"] + ): + matches.append(dataset) + if len(matches) > 1: + raise RuntimeError("More than one Dataset matches the exact approved orthophoto release") + return matches[0] if matches else None + + +def multipart_body(fields: dict[str, str], filename: str, content: bytes) -> tuple[bytes, str]: + boundary = f"geointel-{uuid.uuid4().hex}" + chunks: list[bytes] = [] + for name, value in fields.items(): + chunks.extend( + ( + f"--{boundary}\r\n".encode("ascii"), + f'Content-Disposition: form-data; name="{name}"\r\n\r\n'.encode("ascii"), + value.encode("utf-8"), + b"\r\n", + ) + ) + safe_filename = filename.replace('"', "_").replace("\r", "_").replace("\n", "_") + chunks.extend( + ( + f"--{boundary}\r\n".encode("ascii"), + f'Content-Disposition: form-data; name="file"; filename="{safe_filename}"\r\n'.encode("ascii"), + b"Content-Type: image/tiff\r\n\r\n", + content, + b"\r\n", + f"--{boundary}--\r\n".encode("ascii"), + ) + ) + return b"".join(chunks), boundary + + +def upload_approved_dataset( + args: argparse.Namespace, + plan: dict[str, Any], + manifest: dict[str, Any], + review: dict[str, Any], +) -> dict[str, Any]: + internal_api_url(args.api_url) + geotiff_path = Path(manifest["normalized_geotiff"]["path"]) + dates = parse_flight_dates(plan["preflight_identity"]["flight_day_coverage"]["flight_dates"]) + observed_at = dates[-1].isoformat() + source_metadata = { + "provider": SOURCE_NAME, + "service": "WMS", + "service_version": "1.3.0", + "product_key": preflight.PRODUCT_KEY, + "official_edition": plan["edition"], + "metadata_identifier": preflight.METADATA_IDENTIFIER, + "metadata_url": plan["preflight_identity"]["remote_release"]["metadata_url"], + "catalog_url": preflight.CATALOG_URL, + "layer": LAYER, + "native_crs": OUTPUT_CRS, + "native_resolution_m": plan["preflight_identity"]["coverage_domain"]["native_resolution_m"], + "requested_resolution_m": OUTPUT_RESOLUTION_M, + "band_count": 3, + "color_mode": "rgb", + "authority_level": "authoritative", + "flight_dates": [value.isoformat() for value in dates], + "flight_date_evidence_mode": "official_queryable_flight_day_grid", + "attribution": "Digitaal Vlaanderen", + "license_note": "Gebruik volgens het gebruiksrecht geografische webdiensten van Digitaal Vlaanderen.", + "limitation_message": plan["preflight_identity"]["flight_day_coverage"]["claim_boundary"], + } + provenance_metadata = { + "operator_tool": "manage_orthophoto_release.py", + "operator_explicit_apply": True, + "release_plan_sha256": plan["plan_sha256"], + "release_plan_path": str(args.plan_path or default_plan_path(args, plan["edition"])), + "review_sha256": review["review_sha256"], + "review_evidence_path": str(args.review_path or default_review_path(args, plan["edition"])), + "reviewer": review["reviewer"], + "staged_manifest_sha256": manifest["manifest_sha256"], + "source_response_sha256": manifest["response"]["sha256"], + "normalized_geotiff_sha256": manifest["normalized_geotiff"]["sha256"], + "review_preview_sha256": manifest["review_preview"]["sha256"], + "preflight_identity_sha256": plan["preflight_identity"]["preflight_identity_sha256"], + "request": manifest["request"], + "response_content_type": manifest["response"]["content_type"], + "source_response_path": manifest["response"]["path"], + "staged_manifest_path": plan["staged_manifest_path"], + "review_preview_path": manifest["review_preview"]["path"], + "baseline_authorization": plan["baseline_authorization"], + "existing_snapshots_retained": True, + } + fields = { + "dataset_type": "raster", + "source": "operator_official_import", + "dataset_role": "source", + "source_name": SOURCE_NAME, + "source_metadata_json": json.dumps(source_metadata, ensure_ascii=False), + "provenance_metadata_json": json.dumps(provenance_metadata, ensure_ascii=False), + "temporal_series_key": f"digitaal-vlaanderen:orthophoto:{selection_key(list(args.bbox))}", + "observed_at": observed_at, + "valid_from": dates[0].isoformat(), + "valid_to": dates[-1].isoformat(), + "temporal_granularity": "snapshot", + "source_version": plan["edition"], + } + if args.area_id: + fields["area_id"] = args.area_id + body, boundary = multipart_body(fields, geotiff_path.name, geotiff_path.read_bytes()) + request = Request( + f"{args.api_url}/projects/{args.project_id}/datasets/upload", + data=body, + method="POST", + headers={ + "Accept": "application/json", + "Content-Type": f"multipart/form-data; boundary={boundary}", + "User-Agent": "GeoIntel-orthophoto-release/1.0", + }, + ) + try: + with urlopen(request, timeout=args.import_timeout) as response: + payload = json.load(response) + except HTTPError as exc: + detail = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"GeoIntel orthophoto upload returned HTTP {exc.code}: {detail[-1500:]}") from exc + except URLError as exc: + raise RuntimeError(f"GeoIntel orthophoto upload is unreachable: {exc.reason}") from exc + if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict): + raise RuntimeError("GeoIntel orthophoto upload did not return a canonical data envelope") + dataset = payload["data"] + if ( + dataset.get("status") != "ready" + or dataset.get("source_name") != SOURCE_NAME + or dataset.get("source_version") != plan["edition"] + or dataset.get("checksum_sha256") != manifest["normalized_geotiff"]["sha256"] + ): + raise RuntimeError("GeoIntel did not persist the exact approved orthophoto Dataset") + return dataset + + +def applied_evidence( + args: argparse.Namespace, + plan_path: Path, + plan: dict[str, Any], + review_path: Path, + review: dict[str, Any], + dataset: dict[str, Any], + final_report: dict[str, Any], + *, + reused: bool, +) -> tuple[Path, dict[str, Any]]: + payload: dict[str, Any] = { + "schema_version": 1, + "status": "applied", + "applied_at": datetime.now(timezone.utc).isoformat(), + "project_id": args.project_id, + "scope": args.scope, + "edition": plan["edition"], + "staged_plan_path": str(plan_path), + "staged_plan_sha256": plan["plan_sha256"], + "review_path": str(review_path), + "review_sha256": review["review_sha256"], + "reviewer": review["reviewer"], + "dataset_id": dataset["id"], + "dataset_checksum_sha256": dataset["checksum_sha256"], + "dataset_status": "existing" if reused else "imported", + "final_local_source_version": final_report["release"].get("local_source_version"), + "final_comparison_status": final_report["release"].get("comparison_status"), + "existing_snapshots_retained": True, + } + payload["applied_evidence_sha256"] = canonical_sha256(payload, "applied_evidence_sha256") + path = plan_path.with_name("applied-evidence.json") + write_json(path, payload) + return path, payload + + +def main() -> int: + args = parse_args() + try: + if min(args.api_timeout, args.wms_timeout, args.import_timeout, args.max_response_mb) <= 0: + raise ValueError("All timeout and response limits must be positive") + if args.max_response_mb > MAX_RESPONSE_MB: + raise ValueError(f"--max-response-mb may not exceed {MAX_RESPONSE_MB}") + if args.action in {"stage", "apply"}: + internal_api_url(args.api_url) + report = run_preflight(args, refresh=args.refresh_catalog or args.action != "plan") + if args.action == "plan": + output = { + "status": "ok", + "action": "plan", + "preflight": report, + "preflight_identity": preflight_identity(report), + "baseline_transition_required": report["release"]["status"] == "blocked_local_version", + "automatic_stage": False, + "automatic_import": False, + } + print(json.dumps(output, ensure_ascii=False, indent=2)) + return 0 + + edition = require_edition(args, report) + if args.action == "stage": + authorization = authorize_stage(args, report) + request_identity = map_request(report) + staged = stage_artifacts(args, report, request_identity) + plan = build_staged_plan(args, report, authorization, staged) + plan_path = governed_path(args, args.plan_path or default_plan_path(args, edition)) + write_json(plan_path, plan) + print( + json.dumps( + {"status": "staged", "plan_path": str(plan_path), "preview_path": staged["review_preview"]["path"], **plan}, + ensure_ascii=False, + indent=2, + ) + ) + return 0 + + plan_path, plan, manifest = load_staged_plan(args, edition) + require_preflight_unchanged(plan, report, require_local=args.action == "review") + if args.action == "review": + review = build_review_evidence(args, plan_path, plan, manifest) + review_path = governed_path(args, args.review_path or default_review_path(args, edition)) + write_json(review_path, review) + print(json.dumps({"status": "approved", "review_path": str(review_path), **review}, ensure_ascii=False, indent=2)) + return 0 + + review_path, review = load_review_evidence(args, edition, plan, manifest) + existing = matching_dataset(args, plan, manifest) + if existing is None: + if report["release"].get("local_source_version") != plan.get("local_source_version_before_apply"): + raise RuntimeError("Local orthophoto edition changed after review; create and review a new plan") + dataset = upload_approved_dataset(args, plan, manifest, review) + reused = False + else: + dataset = existing + reused = True + final_report = run_preflight(args, refresh=True) + require_preflight_unchanged(plan, final_report, require_local=False) + if ( + final_report["release"].get("status") != "current" + or final_report["release"].get("local_source_version") != edition + or final_report["release"].get("comparison_status") != "same" + ): + raise RuntimeError("Applied orthophoto Dataset did not become the current official local edition") + evidence_path, evidence = applied_evidence( + args, + plan_path, + plan, + review_path, + review, + dataset, + final_report, + reused=reused, + ) + print(json.dumps({"status": "applied", "evidence_path": str(evidence_path), **evidence}, ensure_ascii=False, indent=2)) + return 0 + except (OSError, RuntimeError, ValueError, KeyError, json.JSONDecodeError) as exc: + print(json.dumps({"status": "error", "action": args.action, "message": str(exc)}, ensure_ascii=False), file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 64156a32..2f81b57c 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -56,6 +56,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_regional_bwk_natura2000.py ${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py ${PYTHON_BIN} -m py_compile scripts/manage_alz_agriculture_release.py ${PYTHON_BIN} -m py_compile scripts/orthophoto_release_preflight.py +${PYTHON_BIN} -m py_compile scripts/manage_orthophoto_release.py ${PYTHON_BIN} -m py_compile scripts/provision_buildings_addresses_register.py ${PYTHON_BIN} -m py_compile scripts/provision_mol_dhmv.py ${PYTHON_BIN} -m py_compile scripts/provision_regional_dhmv.py