diff --git a/CHANGELOG.md b/CHANGELOG.md index eb1b89a7..69dae516 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ # Changelog +## Sprint 230 Governed orthophoto release preflight (2026-07-17) + +- Added an operator-only, read-only preflight for the current Digitaal + Vlaanderen orthophoto product. It binds the canonical product and catalog + envelopes to exact official WMS capabilities and ISO edition evidence. +- Added metadata-only WCS `DescribeCoverage` validation for the governed + EPSG:31370, 15 cm, three-band `Ortho` raster domain and deterministic bounded + `Vliegdagcontour` sampling for selected-area flight-year evidence. +- Classified local official editions as current/update/baseline/remote-older + and kept legacy `most_recent_at_*` values explicitly non-comparable. No + raster pixels, Datasets, Jobs, migrations or automatic refresh were added. +- Added trust-boundary, hash-drift, product, coverage, flight-year, bounds and + runtime-packaging tests plus readiness compilation and operator docs. + ## Sprint 229 Governed ALZ definitive release promotion (2026-07-17) - Added an operator-only `plan -> stage -> review -> apply` coordinator for diff --git a/backend/README.md b/backend/README.md index 451e8c4c..605981dc 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1300,6 +1300,26 @@ Settings: `ORTHOPHOTO_ENABLED`, `ORTHOPHOTO_WMS_URL`, `ORTHOPHOTO_CACHE_TTL_HOURS`. Keep the official HTTPS URL and 1 m profile unless a separately verified deployment/model profile requires a change. +Before a future `most_recent` source release is allowed into a governed pixel +stage, run the metadata-only preflight for the exact intended rectangle: + +```bash +docker exec geointel python /app/scripts/orthophoto_release_preflight.py \ + --project-id 82a85913-c522-45d7-84a1-02b393d89e55 \ + --api-url http://127.0.0.1:8000/api/v1 \ + --bbox 5.110 51.180 5.117 51.185 \ + --refresh-catalog +``` + +The command reads canonical API envelopes, exact official WMS capabilities, +WCS `DescribeCoverage` and at most 64 queryable flight-day points. It never +requests raster pixels or mutates application/storage state. Only +`staging_permitted=true` may feed a future separate staging command. `current`, +remote-older, mixed/incorrect flight years and legacy local values such as +`most_recent_at_2026-07-15` remain non-stageable. The report's point grid is +flight-date evidence; complete selected-area coverage comes from containment +inside the official 15 cm WCS raster domain. + ## Governed DHMV terrain acquisition `GET /api/v1/projects/{project_id}/datasets/dhmv/products` exposes the fixed diff --git a/backend/tests/test_docker_runtime_config.py b/backend/tests/test_docker_runtime_config.py index af17a8b1..b9932ca0 100644 --- a/backend/tests/test_docker_runtime_config.py +++ b/backend/tests/test_docker_runtime_config.py @@ -92,6 +92,7 @@ def test_all_in_one_dockerfile_copies_operator_scripts_for_runtime_use() -> None "run_split_background_promotion_workflow.sh", "activate_promoted_yolo_candidate.py", "manage_grb_refresh.py", + "orthophoto_release_preflight.py", } for script_name in required_runtime_scripts: assert f"COPY scripts/{script_name} /app/scripts/{script_name}" in dockerfile diff --git a/backend/tests/test_sprint230_orthophoto_release_preflight.py b/backend/tests/test_sprint230_orthophoto_release_preflight.py new file mode 100644 index 00000000..b43bb896 --- /dev/null +++ b/backend/tests/test_sprint230_orthophoto_release_preflight.py @@ -0,0 +1,365 @@ +from __future__ import annotations + +import argparse +from email.message import Message +from hashlib import sha256 +import importlib.util +import json +from pathlib import Path +import sys +from urllib.parse import parse_qs, urlparse + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + + +def load_script(): + path = SCRIPTS / "orthophoto_release_preflight.py" + module_name = "test_orthophoto_release_preflight_sprint230" + spec = importlib.util.spec_from_file_location(module_name, path) + assert spec is not None + assert spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[module_name] = module + spec.loader.exec_module(module) + return module + + +PREFLIGHT = load_script() +PROJECT_ID = "00000000-0000-0000-0000-000000000001" + + +def arguments(**overrides) -> argparse.Namespace: + values = { + "project_id": PROJECT_ID, + "scope": PREFLIGHT.DEFAULT_SCOPE, + "api_url": "http://127.0.0.1:8000/api/v1", + "bbox": [5.110, 51.180, 5.117, 51.185], + "refresh_catalog": True, + "api_timeout": 30, + "wms_timeout": 10, + } + values.update(overrides) + return argparse.Namespace(**values) + + +def capabilities(*, queryable: bool = True, feature_info: bool = True) -> bytes: + info_format = "application/geo+json" if feature_info else "text/plain" + queryable_value = "1" if queryable else "0" + metadata_url = ( + "https://metadata.vlaanderen.be/srv/dut/csw?request=GetRecordById&service=CSW&" + f"id={PREFLIGHT.METADATA_IDENTIFIER}" + ) + return f""" + + + {info_format} + + EPSG:31370 + + Ortho + Vliegdagcontour + + + + """.encode() + + +def coverage_description(*, coverage_id: str = "Ortho", resolution: float = 0.15) -> bytes: + return f""" + + + + 21375 152250259500 244875 + + {coverage_id} + + {resolution} 00 -{resolution} + + + + + RectifiedGridCoverage + image/tiff + + + """.encode() + + +def catalog_item(body: bytes, *, local: str | None = "most_recent_at_2026-07-15", remote: str = "2025.04") -> dict: + if local is None: + comparison = "no_local_data" + elif PREFLIGHT.EDITION_PATTERN.fullmatch(local): + comparison = "same" if local == remote else "different" + else: + comparison = "not_comparable" + return { + "source_name": PREFLIGHT.SOURCE_NAME, + "status": "available", + "reachable": True, + "matched_layers": ["Ortho", "Vliegdagcontour"], + "missing_layers": [], + "metadata_identifier": PREFLIGHT.METADATA_IDENTIFIER, + "metadata_url": f"https://metadata.vlaanderen.be/{PREFLIGHT.METADATA_IDENTIFIER}", + "remote_title": f"Orthofoto meest recent, {remote}", + "remote_version": remote, + "remote_modified_at": "2026-04-27T00:00:00Z", + "remote_published_at": "2025-12-11T00:00:00Z", + "local_source_version": local, + "comparison_status": comparison, + "checked_at": "2026-07-17T00:00:00Z", + "endpoint_url": ( + "https://geo.api.vlaanderen.be/OMWRGBMRVL/wms?" + "SERVICE=WMS&VERSION=1.3.0&REQUEST=GetCapabilities" + ), + "capabilities_sha256": sha256(body).hexdigest(), + } + + +def product() -> dict: + return { + "key": "most_recent", + "display_name": "Meest recente winterluchtbeeld", + "observation_label": "Meest recent beschikbaar", + "temporal_granularity": "snapshot", + "native_resolution_m": 0.15, + "supports_detection": True, + "color_mode": "rgb", + "catalog_url": PREFLIGHT.CATALOG_URL, + "limitation_message": "rolling source", + } + + +def loader(body: bytes, item: dict): + def load(_api_url: str, path: str, _timeout: int) -> dict: + if path == f"projects/{PROJECT_ID}": + return {"id": PROJECT_ID, "name": "Kempen Regional Workbench"} + if path.endswith("/datasets/orthophoto/products"): + return {"items": [product()], "total": 1} + if "/datasets/source-catalog-probes?" in path: + return {"items": [item]} + raise AssertionError(path) + + return load + + +class Response: + def __init__(self, body: bytes, *, url: str, content_type: str) -> None: + self.body = body + self.url = url + self.headers = Message() + self.headers["Content-Type"] = content_type + self.headers["Content-Length"] = str(len(body)) + + def __enter__(self): + return self + + def __exit__(self, *_args) -> None: + return None + + def geturl(self) -> str: + return self.url + + def read(self, size: int = -1) -> bytes: + return self.body if size < 0 else self.body[:size] + + +def opener( + body: bytes, + *, + flight_year: int = 2025, + empty: bool = False, + requests: list[str] | None = None, + coverage_body: bytes | None = None, +): + def open_request(request, timeout: int): + assert timeout == 10 + url = request.full_url + query = parse_qs(urlparse(url).query) + request_name = (query.get("REQUEST") or [""])[0] + if requests is not None: + requests.append(request_name) + if request_name == "GetCapabilities": + return Response(body, url=url, content_type="text/xml") + if request_name == "DescribeCoverage": + return Response(coverage_body or coverage_description(), url=url, content_type="text/xml") + assert request_name == "GetFeatureInfo" + assert query["LAYERS"] == ["Vliegdagcontour"] + features = [] if empty else [ + { + "type": "Feature", + "geometry": None, + "properties": {"OpnDatum": f"5/4/{flight_year}", "FID": "4"}, + "layerName": "Vliegdagcontour", + } + ] + payload = json.dumps({"type": "FeatureCollection", "features": features}).encode() + return Response(payload, url=url, content_type="application/geo+json") + + return open_request + + +def run(*, local: str | None = "most_recent_at_2026-07-15", remote: str = "2025.04", flight_year: int = 2025): + body = capabilities() + item = catalog_item(body, local=local, remote=remote) + calls: list[str] = [] + report = PREFLIGHT.run_preflight( + arguments(), + loader=loader(body, item), + opener=opener(body, flight_year=flight_year, requests=calls), + ) + return report, calls + + +def test_live_contract_shape_blocks_non_comparable_legacy_local_version() -> None: + report, calls = run() + + assert report["status"] == "passed" + assert report["release"]["status"] == "blocked_local_version" + assert report["release"]["remote_edition"] == "2025.04" + assert report["flight_day_coverage"]["sample_count"] == 20 + assert report["flight_day_coverage"]["sample_coverage_ratio"] == 1.0 + assert report["flight_day_coverage"]["flight_years"] == [2025] + assert report["staging_permitted"] is False + assert report["next_action"] == "establish_official_local_edition_before_staging" + assert report["pixel_requests_performed"] == 0 + assert set(calls) == {"GetCapabilities", "DescribeCoverage", "GetFeatureInfo"} + assert report["coverage_domain"]["selected_area_fully_inside_domain"] is True + assert report["coverage_domain"]["pixel_data_requested"] is False + + +@pytest.mark.parametrize( + ("local", "remote", "expected_status", "stageable"), + [ + (None, "2025.04", "not_loaded", True), + ("2025.04", "2025.04", "current", False), + ("2024.01", "2025.04", "update_available", True), + ("2026.01", "2025.04", "blocked_remote_older", False), + ], +) +def test_release_ordering_requires_comparable_official_editions( + local: str | None, + remote: str, + expected_status: str, + stageable: bool, +) -> None: + report, _ = run(local=local, remote=remote) + + assert report["release"]["status"] == expected_status + assert report["staging_permitted"] is stageable + + +def test_flight_year_must_match_remote_release_year() -> None: + report, _ = run(local="2024.01", flight_year=2024) + + assert report["release"]["status"] == "update_available" + assert report["flight_year_matches_release"] is False + assert report["staging_permitted"] is False + assert report["next_action"] == "split_or_review_selection_flight_years" + + +def test_capabilities_hash_drift_fails_closed() -> None: + body = capabilities() + item = catalog_item(body, local=None) + changed = body.replace(b"queryable=\"1\"", b"queryable=\"0\"") + + with pytest.raises(RuntimeError, match="changed after"): + PREFLIGHT.run_preflight( + arguments(), + loader=loader(body, item), + opener=opener(changed), + ) + + +@pytest.mark.parametrize( + ("body", "message"), + [ + (capabilities(queryable=False), "not queryable"), + (capabilities(feature_info=False), "does not advertise GeoJSON"), + ], +) +def test_flight_day_capability_requirements_fail_closed(body: bytes, message: str) -> None: + item = catalog_item(body, local=None) + with pytest.raises(RuntimeError, match=message): + PREFLIGHT.run_preflight(arguments(), loader=loader(body, item), opener=opener(body)) + + +def test_missing_flight_day_coverage_fails_closed() -> None: + body = capabilities() + item = catalog_item(body, local=None) + with pytest.raises(RuntimeError, match="has no coverage"): + PREFLIGHT.run_preflight( + arguments(), + loader=loader(body, item), + opener=opener(body, empty=True), + ) + + +@pytest.mark.parametrize( + "coverage_body", + [coverage_description(coverage_id="Other"), coverage_description(resolution=0.25)], +) +def test_coverage_domain_identity_and_resolution_fail_closed(coverage_body: bytes) -> None: + body = capabilities() + item = catalog_item(body, local=None) + with pytest.raises(RuntimeError, match="WCS"): + PREFLIGHT.run_preflight( + arguments(), + loader=loader(body, item), + opener=opener(body, coverage_body=coverage_body), + ) + + +def test_product_variant_and_catalog_comparison_are_bound() -> None: + wrong = product() + wrong["color_mode"] = "panchromatic" + with pytest.raises(RuntimeError, match="product variant"): + PREFLIGHT.validate_product([wrong]) + + body = capabilities() + item = catalog_item(body, local="2025.04") + item["comparison_status"] = "different" + with pytest.raises(RuntimeError, match="internally inconsistent"): + PREFLIGHT.catalog_decision(item) + + +def test_only_exact_official_wms_url_is_allowed() -> None: + valid = "https://geo.api.vlaanderen.be/OMWRGBMRVL/wms?SERVICE=WMS&REQUEST=GetCapabilities" + assert PREFLIGHT._validate_wms_url(valid, request_name="GetCapabilities").endswith("/OMWRGBMRVL/wms") + with pytest.raises(RuntimeError, match="outside the official allowlist"): + PREFLIGHT._validate_wms_url(valid.replace("geo.api.vlaanderen.be", "example.com")) + with pytest.raises(RuntimeError, match="not an exact"): + PREFLIGHT._validate_wms_url(valid.replace("GetCapabilities", "GetMap"), request_name="GetCapabilities") + + +def test_bbox_matches_acquisition_bounds_and_grid_is_bounded() -> None: + extent = [22000.0, 150000.0, 259000.0, 245000.0] + selection = PREFLIGHT.validate_bbox([5.110, 51.180, 5.117, 51.185], extent) + samples = PREFLIGHT._sample_grid(selection) + assert len(samples) == 20 + assert len(samples) <= PREFLIGHT.MAX_SAMPLE_COUNT + with pytest.raises(RuntimeError, match="at least"): + PREFLIGHT.validate_bbox([5.110, 51.180, 5.1105, 51.1805], extent) + with pytest.raises(RuntimeError, match="may not exceed"): + PREFLIGHT.validate_bbox([5.05, 51.15, 5.25, 51.33], extent) + + +def test_operator_preflight_is_read_only_packaged_and_readiness_checked() -> None: + script = (SCRIPTS / "orthophoto_release_preflight.py").read_text(encoding="utf-8") + dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8") + readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8") + + assert "GetMap" not in script + assert "GetCoverage" not in script + assert "/datasets/upload" not in script + assert "INSERT INTO" not in script + assert "COPY scripts/orthophoto_release_preflight.py" in dockerfile + assert "py_compile scripts/orthophoto_release_preflight.py" in readiness diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index 19a9c351..11530c49 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -92,6 +92,7 @@ COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natu COPY scripts/provision_regional_bwk_natura2000.py /app/scripts/provision_regional_bwk_natura2000.py COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py COPY scripts/manage_alz_agriculture_release.py /app/scripts/manage_alz_agriculture_release.py +COPY scripts/orthophoto_release_preflight.py /app/scripts/orthophoto_release_preflight.py COPY scripts/provision_buildings_addresses_register.py /app/scripts/provision_buildings_addresses_register.py COPY scripts/provision_regional_timeseries.py /app/scripts/provision_regional_timeseries.py COPY scripts/geographic_scopes.py /app/scripts/geographic_scopes.py diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index 078db80e..f0e5c3d0 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -466,6 +466,16 @@ revalidates the current catalog and delegates to the existing Dataset upload contract. No ALZ release endpoint, background task or provider URL parameter is added; v1/v2 campaign snapshots remain non-importable. +Current-orthophoto release preflight also remains outside the HTTP request +cycle in `scripts/orthophoto_release_preflight.py`. It composes the existing +product-registry and source-catalog envelopes with allowlisted WMS +`GetCapabilities`, WCS `DescribeCoverage` and bounded `Vliegdagcontour` +`GetFeatureInfo` evidence for one 128-1,024 m EPSG:4326 selection. The result +reports product variant, official `YYYY.NN` edition, exact raster-domain +containment, sampled flight dates/years, local comparison state and +`staging_permitted`. It performs no pixel request, upload, Job, Dataset write +or legacy metadata rewrite. This operator script adds no public API contract. + The endpoint accepts no arbitrary URL, feature query, area or layer. It does not fetch vector features, raster pixels or models, create jobs/datasets, write to PostGIS or trigger an import. The normal `source-freshness` endpoint remains diff --git a/docs/CODEX_EXECUTION_LOG.md b/docs/CODEX_EXECUTION_LOG.md index 96d64e50..ddb67886 100644 --- a/docs/CODEX_EXECUTION_LOG.md +++ b/docs/CODEX_EXECUTION_LOG.md @@ -9715,3 +9715,40 @@ Live validation: Next: - Evaluate ALZ as the next governed edition probe only after its official machine-readable release version and schema stability are verified. +## Sprint 230 - Governed orthophoto release preflight (2026-07-17) + +Implemented: +- Added `scripts/orthophoto_release_preflight.py` as a read-only gate for one + bounded current-orthophoto selection in the approved Kempen regional scope. +- Bound the existing canonical product registry and source-catalog result to + the exact official WMS 1.3.0 capabilities hash, ISO metadata identifier and + `YYYY.NN` edition. Redirects, hosts, paths and response sizes fail closed. +- Added metadata-only WCS `DescribeCoverage` validation for `Ortho`, + EPSG:31370, the complete raster domain, 15 cm rectified grid, three bands and + TIFF native format. Selection limits remain identical to acquisition: + 128-1,024 m per projected side. +- Added a maximum 64-point deterministic `Vliegdagcontour` grid with exact + GeoJSON layer/date/year validation and hashed sample evidence. WCS domain + containment proves raster-domain coverage; point samples remain honestly + labelled as flight-date evidence rather than polygon-union geometry. +- Kept all pixel requests, filesystem staging, Jobs, Dataset/PostGIS writes, + migrations and browser behavior out of this sprint. Local + `most_recent_at_*` acquisition markers remain non-comparable and blocked. + +Validation so far: +- 42 focused preflight/runtime-packaging tests passed. Coverage includes + release ordering, legacy provenance, hash drift, product variant, official + host/path, queryable/GeoJSON capability, WCS identity/resolution, missing + flight coverage, flight-year mismatch, selection bounds and no-write rules. +- Target Python compilation and Ruff passed. +- A read-only compatibility run against Tower for bbox + `5.110,51.180,5.117,51.185` verified official edition `2025.04`, exact Ortho + WCS domain containment and 20/20 flight-day samples from 2025. It performed + zero pixel requests and correctly returned `staging_permitted=false` because + local version `most_recent_at_2026-07-15` is not an official edition. + +Boundary: +- This sprint supplies preflight only. It deliberately does not promote or + backfill existing orthophotos. A future stage/review/apply coordinator must + revalidate and retain the exact preflight identity before creating a new + immutable raster Dataset with official `YYYY.NN` source version. diff --git a/docs/DATABASE_IMPLEMENTATION_PLAN.md b/docs/DATABASE_IMPLEMENTATION_PLAN.md index 8d789113..828ceb60 100644 --- a/docs/DATABASE_IMPLEMENTATION_PLAN.md +++ b/docs/DATABASE_IMPLEMENTATION_PLAN.md @@ -291,6 +291,14 @@ annual Dataset, DatasetVersion and vector_features records with `source_version=-definitive`. Earlier annual snapshots are retained and provisional v1/v2 publications cannot create rows. +The current-orthophoto release preflight likewise adds no lifecycle table or +migration. It is read-only and creates neither Dataset nor Job. It compares the +existing local `source_version` with the official `YYYY.NN` catalog edition +and verifies WMS/WCS/flight-day evidence for one bounded selection. A later +governed pixel apply must still create an immutable raster Dataset plus +DatasetVersion through DatasetService and retain that exact edition/evidence; +direct metadata backfill of legacy `most_recent_at_*` rows is prohibited. + ## Geometry normalization - User-drawn polygons arrive as EPSG:4326. diff --git a/docs/DATA_SOURCES.md b/docs/DATA_SOURCES.md index 22a6e71b..66fb6f11 100644 --- a/docs/DATA_SOURCES.md +++ b/docs/DATA_SOURCES.md @@ -24,6 +24,21 @@ aanvraag. GeoIntel verzint geen historische pixelopnamedatum. Bronnen: - https://www.vlaanderen.be/datavindplaats/catalogus/orthofotomozaiek-middenschalig-winteropnamen-kleur-meest-recent-vlaanderen - https://www.vlaanderen.be/digitaal-vlaanderen/onze-diensten-en-platformen/luchtopnamen/gebruik-orthofotomozaieken +Voor een toekomstige rolling-releasebeslissing gebruikt +`scripts/orthophoto_release_preflight.py` uitsluitend metadata. Het bindt de +lokale `most_recent`-productvariant aan de officiële ISO-editie en exacte WMS- +capabilitieshash, controleert het EPSG:31370/15 cm/driebanden-rasterdomein via +WCS `DescribeCoverage` en bemonstert de querybare `Vliegdagcontour` op een +deterministisch raster met maximaal 128 m afstand. Er worden geen `GetMap`-, +`GetCoverage`- of uploadrequests uitgevoerd. + +De WCS-domeincontrole bewijst dat de volledige begrensde selectie binnen het +officiële rasterdomein valt. De WMS publiceert geen vliegdagpolygonen als WFS; +het vluchtjaar blijft daarom expliciet puntbewijs en geen verzonnen polygon- +union. Een selectie met ontbrekende contourpunten, meerdere/afwijkende +vluchtjaren, gewijzigde service-identiteit of een niet-vergelijkbare lokale +`most_recent_at_*` marker is niet stagebaar. + Dit document verzamelt concrete databronnen voor GeoIntel Kempen. ## Cross-domain official area profile @@ -174,7 +189,7 @@ gebeurd. | GRB gebouwen/wegen/water/percelen | operationele, expliciete plan-stage-apply refresh met onveranderlijke snapshots | alleen een nieuw officieel gedateerd cataloguseditie na operatorbevestiging ophalen | | Statbel bevolking | jaarlijkse, expliciete edities in één tijdreeks; officiële DCAT-releaseprobe | een nieuwe publicatie alleen na schema-, sectorgeometrie- en totalencontrole toevoegen | | ALZ landbouwgebruikspercelen | definitieve jaarlijkse edities 2008-2025; expliciete publicatieprobe en plan-stage-review-apply promotie; metricvergelijking zonder objectlineage | alleen een nieuwere definitieve v3-editie na gestagede schema-/codelijst-/scopecontrole en benoemde review toevoegen | -| orthofoto | vaste lokale opname per expliciete analysezone; catalogusprobe is alleen een signaal | vluchtjaar, productvariant en dekking vergelijken voordat nieuwe pixels worden opgehaald | +| orthofoto | vaste lokale opname per expliciete analysezone; read-only releasepreflight voor variant, officiële editie, exact WCS-domein en begrensd vluchtjaarbewijs | eerst officiële lokale editieprovenance vastleggen; daarna pas een afzonderlijke menselijke pixel-stage/apply-flow bouwen | | landgebruik, thematische rasters, DHMV en VMM-scenario's | vaste product-/scenario-edities, geen rolling snapshot | alleen een nieuwe gedocumenteerde producteditie als afzonderlijke Dataset verwerven | | bodemkaart en historische kaarten | historische referentie-editie | niet als verouderde actuele bron labelen; alleen vervangen bij een officiële inhoudelijke heruitgave | | BWK/Natura 2000 en gebouwen-/adressenregister | expliciete actuele snapshot met eigen methodologische betekenis | eerst een stabiele officiële editieprobe en bron-specifieke reconciliatiecontrole toevoegen | diff --git a/docs/DATA_SPECIFICATION.md b/docs/DATA_SPECIFICATION.md index 8ddb2201..81400068 100644 --- a/docs/DATA_SPECIFICATION.md +++ b/docs/DATA_SPECIFICATION.md @@ -274,6 +274,13 @@ layer, observation label, `observed_at`, optional `valid_from`/`valid_to`, temporal granularity, request/spatial hash, attribution and a limitation that states whether the product is annual, multi-year or merely most recent. +A future rolling `most_recent` import must also retain the exact official +`YYYY.NN` edition and the preflight identities for WMS capabilities, WCS +coverage description, selected EPSG:31370 domain and sampled flight year. A +legacy `most_recent_at_` value is acquisition timing, not an official +edition, and cannot be promoted or compared as if it were one. The read-only +preflight creates no Dataset and does not retroactively rewrite that evidence. + ### Hydrological station observations Waterinfo observations are persisted as EPSG:4326 Point features, one station diff --git a/docs/STORAGE_ARCHITECTURE.md b/docs/STORAGE_ARCHITECTURE.md index 32cb1804..3ff75023 100644 --- a/docs/STORAGE_ARCHITECTURE.md +++ b/docs/STORAGE_ARCHITECTURE.md @@ -147,6 +147,14 @@ product/layer, request/spatial hash, temporal validity and limitations are held in source/provenance metadata. Browser PNG rendering is derived on request and does not replace the stored GeoTIFF. +The orthophoto release preflight writes no source file, raster or database row. +Its JSON stdout may be retained by an operator as review evidence, but it is +not itself staging authorization. The report binds official WMS and WCS XML +hashes, the exact selected domain and hashed flight-day sample evidence. A +future pixel stage must persist and revalidate that identity separately before +DatasetService is called; existing `most_recent_at_*` raster metadata is not +silently rewritten. + DHMV II DTM/DSM outputs are also normal raster Dataset files. The provider WCS returns multipart coverage data; GeoIntel retains response and extracted coverage SHA256 values in provenance, then stores one normalized, compressed, diff --git a/docs/TODO.md b/docs/TODO.md index 267b241b..d86a92e3 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -690,5 +690,8 @@ This file now starts with the current implementation status. Older preparation/b - [x] Document refresh readiness across the complete official-source portfolio. - [x] Verify and add the governed ALZ catalog probe against the official campaign-snapshot and definitive-archive publication contract. -- [ ] Keep orthophoto refresh manual until product variant, flight year and - complete selected-area coverage can be compared deterministically. +- [x] Add a read-only orthophoto preflight for product variant, official + edition, exact WCS selected-area domain and deterministic flight-year points. +- [ ] Keep orthophoto pixel refresh manual and blocked until a separate + plan-stage-review-apply flow can retain the passed preflight identity and + create a new immutable Dataset with official `YYYY.NN` source version. diff --git a/scripts/orthophoto_release_preflight.py b/scripts/orthophoto_release_preflight.py new file mode 100644 index 00000000..a6762b1c --- /dev/null +++ b/scripts/orthophoto_release_preflight.py @@ -0,0 +1,694 @@ +#!/usr/bin/env python3 +"""Read-only release preflight for the governed current orthophoto product. + +The preflight does not request or stage raster pixels. It binds the canonical +GeoIntel catalog result to the current product registry and the exact official +WMS capabilities, then probes the queryable flight-day layer on a bounded, +deterministic grid. Local rolling markers remain non-comparable and blocked. +""" + +from __future__ import annotations + +import argparse +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime, timezone +from hashlib import sha256 +import json +import math +import os +import re +import sys +from typing import Any, Callable +from urllib.error import HTTPError, URLError +from urllib.parse import parse_qs, urlencode, urlparse +from urllib.request import Request, urlopen +from xml.etree import ElementTree + +from pyproj import Transformer + +from geographic_scopes import GEOGRAPHIC_SCOPES + + +DEFAULT_API_URL = "http://127.0.0.1:8000/api/v1" +DEFAULT_SCOPE = "kempen-transport-region" +SOURCE_NAME = "digitaal_vlaanderen_orthophoto" +PRODUCT_KEY = "most_recent" +WMS_HOST = "geo.api.vlaanderen.be" +WMS_PATH = "/OMWRGBMRVL/wms" +WCS_PATH = "/OMWRGBMRVL/wcs" +CATALOG_URL = ( + "https://www.vlaanderen.be/datavindplaats/catalogus/" + "orthofotomozaiek-middenschalig-winteropnamen-kleur-meest-recent-vlaanderen" +) +METADATA_IDENTIFIER = "f5304d6d-0dd4-43fd-a726-427af31e8d61" +EDITION_PATTERN = re.compile(r"^(20[0-9]{2})\.([0-9]{2})$") +SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") +FLIGHT_YEAR_PATTERN = re.compile(r"(?:^|[^0-9])(20[0-9]{2})(?:$|[^0-9])") +MAX_CAPABILITIES_BYTES = 2 * 1024 * 1024 +MAX_COVERAGE_DESCRIPTION_BYTES = 512 * 1024 +MAX_FEATURE_INFO_BYTES = 128 * 1024 +MAX_SAMPLE_SPACING_M = 128.0 +MAX_SAMPLE_COUNT = 64 +MIN_SIDE_M = 128.0 +MAX_SIDE_M = 1024.0 +FEATURE_INFO_SIZE = 1000 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Read-only current-orthophoto release preflight. No raster pixels are fetched, staged or imported." + ) + ) + parser.add_argument("--project-id", required=True, help="GeoIntel project UUID for the governed scope") + parser.add_argument("--scope", choices=(DEFAULT_SCOPE,), default=DEFAULT_SCOPE) + parser.add_argument("--api-url", default=os.environ.get("GEOINTEL_API_URL", DEFAULT_API_URL)) + parser.add_argument( + "--bbox", + nargs=4, + type=float, + metavar=("MIN_LON", "MIN_LAT", "MAX_LON", "MAX_LAT"), + required=True, + help="Exact EPSG:4326 selection; each projected side must be between 128 and 1024 metres", + ) + parser.add_argument("--refresh-catalog", action="store_true", help="Bypass the short catalog cache") + parser.add_argument("--api-timeout", type=int, default=180) + parser.add_argument("--wms-timeout", type=int, default=30) + return parser.parse_args() + + +def api_data(api_url: str, path: str, timeout: int) -> dict[str, Any]: + endpoint = f"{api_url.rstrip('/')}/{path.lstrip('/')}" + request = Request( + endpoint, + headers={"Accept": "application/json", "User-Agent": "GeoIntel-orthophoto-preflight/1.0"}, + ) + try: + with urlopen(request, timeout=timeout) as response: + payload = json.load(response) + except HTTPError as exc: + body = exc.read().decode("utf-8", errors="replace") + raise RuntimeError(f"GeoIntel API returned HTTP {exc.code}: {body[-1000:]}") from exc + except URLError as exc: + raise RuntimeError(f"GeoIntel API is unreachable: {exc.reason}") from exc + if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict): + raise RuntimeError("GeoIntel API response is not a canonical data envelope") + return payload["data"] + + +def validate_project_scope(args: argparse.Namespace, loader: Callable[[str, str, int], dict[str, Any]]) -> None: + project = loader(args.api_url, f"projects/{args.project_id}", args.api_timeout) + expected_name = GEOGRAPHIC_SCOPES[args.scope].project_name + if project.get("name") != expected_name: + raise RuntimeError( + f"Project {args.project_id} is '{project.get('name')}', but scope {args.scope} requires '{expected_name}'" + ) + + +def _validate_service_url(url: str, *, path: str, request_name: str | None = None) -> str: + parsed = urlparse(url) + if ( + parsed.scheme.lower() != "https" + or parsed.hostname != WMS_HOST + or parsed.port not in (None, 443) + or parsed.username + or parsed.password + or parsed.fragment + or parsed.path.rstrip("/").lower() != path.lower() + ): + raise RuntimeError("Orthophoto service URL is outside the official allowlist") + if request_name is not None: + requests = parse_qs(parsed.query).get("REQUEST") or parse_qs(parsed.query).get("request") or [] + if len(requests) != 1 or requests[0].lower() != request_name.lower(): + raise RuntimeError(f"Orthophoto service URL is not an exact {request_name} request") + return f"https://{WMS_HOST}{path}" + + +def _validate_wms_url(url: str, *, request_name: str | None = None) -> str: + return _validate_service_url(url, path=WMS_PATH, request_name=request_name) + + +def _validate_wcs_url(url: str, *, request_name: str | None = None) -> str: + return _validate_service_url(url, path=WCS_PATH, request_name=request_name) + + +def _bounded_get( + url: str, + *, + timeout: int, + max_bytes: int, + accept: str, + opener: Callable[..., Any] | None = None, + validator: Callable[[str], str] = _validate_wms_url, +) -> tuple[bytes, str, str]: + request = Request(url, headers={"Accept": accept, "User-Agent": "GeoIntel-orthophoto-preflight/1.0"}) + fetch = opener or urlopen + try: + response = fetch(request, timeout=timeout) + with response: + final_url = response.geturl() + validator(final_url) + content_type = str(response.headers.get("Content-Type") or "").split(";", 1)[0].strip().lower() + declared = response.headers.get("Content-Length") + try: + if declared and int(declared) > max_bytes: + raise RuntimeError("Official WMS response exceeds the configured preflight limit") + except ValueError as exc: + raise RuntimeError("Official WMS response has an invalid Content-Length") from exc + body = response.read(max_bytes + 1) + except HTTPError as exc: + raise RuntimeError(f"Official orthophoto WMS returned HTTP {exc.code}") from exc + except URLError as exc: + raise RuntimeError(f"Official orthophoto WMS is unreachable: {exc.reason}") from exc + if len(body) > max_bytes: + raise RuntimeError("Official WMS response exceeds the configured preflight limit") + return body, content_type, final_url + + +def _local_name(tag: str) -> str: + return tag.rsplit("}", 1)[-1] + + +def _children(element: ElementTree.Element, name: str) -> list[ElementTree.Element]: + return [child for child in element if _local_name(child.tag) == name] + + +def _child_text(element: ElementTree.Element, name: str) -> str | None: + for child in element: + if _local_name(child.tag) == name and child.text: + return child.text.strip() + return None + + +def parse_capabilities(body: bytes, expected_sha256: str) -> dict[str, Any]: + if not SHA256_PATTERN.fullmatch(expected_sha256): + raise RuntimeError("Canonical catalog response has no valid capabilities SHA-256") + actual_sha256 = sha256(body).hexdigest() + if actual_sha256 != expected_sha256: + raise RuntimeError("Official WMS capabilities changed after the canonical catalog check") + upper = body[:4096].upper() + if b"= bbox[2] or bbox[1] >= bbox[3]: + raise RuntimeError("Official WMS does not advertise a valid EPSG:31370 extent") + + metadata_identifiers: set[str] = set() + for layer_name in ("Ortho", "Vliegdagcontour"): + for node in layers[layer_name].iter(): + if _local_name(node.tag) != "OnlineResource": + continue + href = next((value for key, value in node.attrib.items() if _local_name(key) == "href"), "") + if METADATA_IDENTIFIER in href: + metadata_identifiers.add(METADATA_IDENTIFIER) + if metadata_identifiers != {METADATA_IDENTIFIER}: + raise RuntimeError("Governed orthophoto layers do not share the expected ISO metadata identity") + return { + "service_version": "1.3.0", + "capabilities_sha256": actual_sha256, + "layers": ["Ortho", "Vliegdagcontour"], + "vliegdagcontour_queryable": True, + "feature_info_format": "application/geo+json", + "extent_epsg31370": bbox, + "metadata_identifier": METADATA_IDENTIFIER, + } + + +def parse_coverage_description(body: bytes) -> dict[str, Any]: + upper = body[:4096].upper() + if b"= upper_corner[0] or lower[1] >= upper_corner[1]: + raise RuntimeError("Official WCS Ortho domain corners are invalid") + grid = next((node for node in root.iter() if _local_name(node.tag) == "RectifiedGrid"), None) + if grid is None or grid.attrib.get("dimension") != "2": + raise RuntimeError("Official WCS Ortho domain is not a two-dimensional rectified grid") + offsets: list[list[float]] = [] + for node in grid.iter(): + if _local_name(node.tag) == "offsetVector" and node.text: + try: + offsets.append([float(value) for value in node.text.split()]) + except ValueError as exc: + raise RuntimeError("Official WCS Ortho grid resolution is invalid") from exc + if ( + len(offsets) != 2 + or any(len(vector) != 2 for vector in offsets) + or not math.isclose(offsets[0][0], 0.15, abs_tol=1e-9) + or not math.isclose(offsets[0][1], 0.0, abs_tol=1e-9) + or not math.isclose(offsets[1][0], 0.0, abs_tol=1e-9) + or not math.isclose(offsets[1][1], -0.15, abs_tol=1e-9) + ): + raise RuntimeError("Official WCS Ortho grid is no longer the governed 15 cm product") + field_count = sum(_local_name(node.tag) == "field" for node in root.iter()) + subtype = next( + ((node.text or "").strip() for node in root.iter() if _local_name(node.tag) == "CoverageSubtype"), + "", + ) + native_format = next( + ((node.text or "").strip() for node in root.iter() if _local_name(node.tag) == "nativeFormat"), + "", + ) + if field_count != 3 or subtype != "RectifiedGridCoverage" or native_format != "image/tiff": + raise RuntimeError("Official WCS Ortho range or native format changed") + return { + "coverage_id": "Ortho", + "crs": "EPSG:31370", + "extent_epsg31370": [lower[0], lower[1], upper_corner[0], upper_corner[1]], + "native_resolution_m": 0.15, + "band_count": 3, + "native_format": "image/tiff", + "coverage_description_sha256": sha256(body).hexdigest(), + } + + +def validate_bbox(values: list[float], advertised_extent: list[float]) -> dict[str, Any]: + if len(values) != 4 or not all(math.isfinite(value) for value in values): + raise RuntimeError("Selection bbox must contain four finite EPSG:4326 values") + min_x, min_y, max_x, max_y = values + if min_x >= max_x or min_y >= max_y or not (-180 <= min_x < max_x <= 180) or not (-90 <= min_y < max_y <= 90): + raise RuntimeError("Selection bbox is not a valid EPSG:4326 rectangle") + transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True) + lambert = [float(value) for value in transformer.transform_bounds(min_x, min_y, max_x, max_y, densify_pts=21)] + width_m = lambert[2] - lambert[0] + height_m = lambert[3] - lambert[1] + if width_m < MIN_SIDE_M or height_m < MIN_SIDE_M: + raise RuntimeError(f"Orthophoto preflight selection must be at least {MIN_SIDE_M:.0f} by {MIN_SIDE_M:.0f} metres") + if width_m > MAX_SIDE_M or height_m > MAX_SIDE_M: + raise RuntimeError(f"Orthophoto preflight selection may not exceed {MAX_SIDE_M:.0f} by {MAX_SIDE_M:.0f} metres") + if not ( + advertised_extent[0] <= lambert[0] + and advertised_extent[1] <= lambert[1] + and lambert[2] <= advertised_extent[2] + and lambert[3] <= advertised_extent[3] + ): + raise RuntimeError("Selection is outside the official WMS advertised EPSG:31370 extent") + return { + "bbox_epsg4326": [min_x, min_y, max_x, max_y], + "bbox_epsg31370": lambert, + "width_m": width_m, + "height_m": height_m, + } + + +def validate_product(items: list[dict[str, Any]]) -> dict[str, Any]: + matches = [item for item in items if item.get("key") == PRODUCT_KEY] + if len(matches) != 1: + raise RuntimeError("Orthophoto product registry does not contain exactly one most_recent product") + product = matches[0] + expected = { + "temporal_granularity": "snapshot", + "native_resolution_m": 0.15, + "supports_detection": True, + "color_mode": "rgb", + "catalog_url": CATALOG_URL, + } + if any(product.get(key) != value for key, value in expected.items()): + raise RuntimeError("Current orthophoto product variant no longer matches the governed registry contract") + return {"key": PRODUCT_KEY, **expected, "display_name": product.get("display_name")} + + +def catalog_decision(item: dict[str, Any]) -> dict[str, Any]: + remote_version = str(item.get("remote_version") or "") + remote_match = EDITION_PATTERN.fullmatch(remote_version) + if ( + item.get("source_name") != SOURCE_NAME + or item.get("status") != "available" + or item.get("reachable") is not True + or set(item.get("matched_layers") or []) != {"Ortho", "Vliegdagcontour"} + or item.get("missing_layers") not in (None, []) + or item.get("metadata_identifier") != METADATA_IDENTIFIER + or not SHA256_PATTERN.fullmatch(str(item.get("capabilities_sha256") or "")) + or not remote_match + or remote_version not in str(item.get("remote_title") or "") + ): + raise RuntimeError("Canonical catalog report does not contain one complete governed orthophoto release") + local_version = str(item.get("local_source_version") or "") + local_match = EDITION_PATTERN.fullmatch(local_version) + if not local_version: + status = "not_loaded" + expected_comparison = "no_local_data" + elif not local_match: + status = "blocked_local_version" + expected_comparison = "not_comparable" + else: + local_order = tuple(int(value) for value in local_match.groups()) + remote_order = tuple(int(value) for value in remote_match.groups()) + if local_order == remote_order: + status = "current" + expected_comparison = "same" + elif local_order < remote_order: + status = "update_available" + expected_comparison = "different" + else: + status = "blocked_remote_older" + expected_comparison = "different" + if item.get("comparison_status") != expected_comparison: + raise RuntimeError("Canonical catalog local/remote comparison is internally inconsistent") + return { + "status": status, + "remote_edition": remote_version, + "remote_year": int(remote_match.group(1)), + "local_source_version": local_version or None, + "comparison_status": expected_comparison, + "metadata_identifier": METADATA_IDENTIFIER, + "metadata_url": item.get("metadata_url"), + "remote_title": item.get("remote_title"), + "remote_modified_at": item.get("remote_modified_at"), + "remote_published_at": item.get("remote_published_at"), + "catalog_checked_at": item.get("checked_at"), + "capabilities_url": item.get("endpoint_url"), + "capabilities_sha256": item.get("capabilities_sha256"), + } + + +def _sample_grid(selection: dict[str, Any]) -> list[dict[str, Any]]: + count_x = max(1, math.ceil(selection["width_m"] / MAX_SAMPLE_SPACING_M)) + count_y = max(1, math.ceil(selection["height_m"] / MAX_SAMPLE_SPACING_M)) + if count_x * count_y > MAX_SAMPLE_COUNT: + raise RuntimeError("Deterministic flight-day grid exceeds the fixed preflight limit") + min_x, min_y, max_x, max_y = selection["bbox_epsg31370"] + samples: list[dict[str, Any]] = [] + for row in range(count_y): + fraction_y = (row + 0.5) / count_y + for column in range(count_x): + fraction_x = (column + 0.5) / count_x + samples.append( + { + "column": column, + "row": row, + "i": min(FEATURE_INFO_SIZE - 1, int(fraction_x * FEATURE_INFO_SIZE)), + "j": min(FEATURE_INFO_SIZE - 1, int((1.0 - fraction_y) * FEATURE_INFO_SIZE)), + "x": min_x + fraction_x * (max_x - min_x), + "y": min_y + fraction_y * (max_y - min_y), + } + ) + return samples + + +def _flight_sample( + base_url: str, + selection: dict[str, Any], + sample: dict[str, Any], + *, + timeout: int, + opener: Callable[..., Any] | None, +) -> dict[str, Any]: + params = { + "SERVICE": "WMS", + "VERSION": "1.3.0", + "REQUEST": "GetFeatureInfo", + "LAYERS": "Vliegdagcontour", + "QUERY_LAYERS": "Vliegdagcontour", + "STYLES": "", + "CRS": "EPSG:31370", + "BBOX": ",".join(f"{value:.3f}" for value in selection["bbox_epsg31370"]), + "WIDTH": str(FEATURE_INFO_SIZE), + "HEIGHT": str(FEATURE_INFO_SIZE), + "I": str(sample["i"]), + "J": str(sample["j"]), + "INFO_FORMAT": "application/geo+json", + "FEATURE_COUNT": "10", + "FORMAT": "image/png", + } + body, content_type, _ = _bounded_get( + f"{base_url}?{urlencode(params)}", + timeout=timeout, + max_bytes=MAX_FEATURE_INFO_BYTES, + accept="application/geo+json", + opener=opener, + ) + if content_type not in {"application/geo+json", "application/json"}: + raise RuntimeError("Vliegdagcontour feature-info response is not GeoJSON") + try: + payload = json.loads(body) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise RuntimeError("Vliegdagcontour feature-info response is invalid JSON") from exc + features = payload.get("features") if isinstance(payload, dict) else None + if not isinstance(payload, dict) or payload.get("type") != "FeatureCollection" or not isinstance(features, list) or not features: + raise RuntimeError("Vliegdagcontour has no coverage at one deterministic selection sample") + dates: set[str] = set() + feature_ids: set[str] = set() + years: set[int] = set() + for feature in features: + if not isinstance(feature, dict) or feature.get("layerName") != "Vliegdagcontour": + raise RuntimeError("Vliegdagcontour feature-info returned an unexpected layer") + properties = feature.get("properties") + if not isinstance(properties, dict): + raise RuntimeError("Vliegdagcontour feature-info has no properties") + date_value = str(properties.get("OpnDatum") or "").strip() + match = FLIGHT_YEAR_PATTERN.search(date_value) + if not match: + raise RuntimeError("Vliegdagcontour feature-info has no parseable acquisition year") + dates.add(date_value) + years.add(int(match.group(1))) + if properties.get("FID") is not None: + feature_ids.add(str(properties["FID"])) + return { + "column": sample["column"], + "row": sample["row"], + "x": round(sample["x"], 3), + "y": round(sample["y"], 3), + "dates": sorted(dates), + "years": sorted(years), + "feature_ids": sorted(feature_ids), + "response_sha256": sha256(body).hexdigest(), + } + + +def verify_flight_coverage( + base_url: str, + selection: dict[str, Any], + *, + timeout: int, + opener: Callable[..., Any] | None = None, +) -> dict[str, Any]: + samples = _sample_grid(selection) + worker_count = min(8, len(samples)) + with ThreadPoolExecutor(max_workers=worker_count) as executor: + evidence = list( + executor.map( + lambda sample: _flight_sample( + base_url, + selection, + sample, + timeout=timeout, + opener=opener, + ), + samples, + ) + ) + years = sorted({year for item in evidence for year in item["years"]}) + dates = sorted({date for item in evidence for date in item["dates"]}) + feature_ids = sorted({feature_id for item in evidence for feature_id in item["feature_ids"]}) + canonical = json.dumps(evidence, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + count_x = len({item["column"] for item in evidence}) + count_y = len({item["row"] for item in evidence}) + return { + "status": "passed", + "mode": "official_queryable_flight_day_grid", + "sample_count": len(evidence), + "grid_columns": count_x, + "grid_rows": count_y, + "maximum_sample_spacing_m": MAX_SAMPLE_SPACING_M, + "covered_sample_count": len(evidence), + "sample_coverage_ratio": 1.0, + "flight_dates": dates, + "flight_years": years, + "feature_ids": feature_ids, + "sample_evidence_sha256": sha256(canonical).hexdigest(), + "claim_boundary": ( + "Every deterministic grid sample is covered by the official queryable flight-day layer. " + "The WMS exposes no vector geometry, so this is bounded point evidence and not a polygon-union proof." + ), + } + + +def run_preflight( + args: argparse.Namespace, + *, + loader: Callable[[str, str, int], dict[str, Any]] = api_data, + opener: Callable[..., Any] | None = None, +) -> dict[str, Any]: + validate_project_scope(args, loader) + products = loader( + args.api_url, + f"projects/{args.project_id}/datasets/orthophoto/products", + args.api_timeout, + ) + product = validate_product(products.get("items") or []) + refresh = "true" if args.refresh_catalog else "false" + report = loader( + args.api_url, + f"projects/{args.project_id}/datasets/source-catalog-probes?refresh={refresh}", + args.api_timeout, + ) + matches = [item for item in report.get("items") or [] if item.get("source_name") == SOURCE_NAME] + if len(matches) != 1: + raise RuntimeError("Source catalog report did not contain exactly one orthophoto contract") + release = catalog_decision(matches[0]) + capabilities_url = str(release["capabilities_url"] or "") + base_url = _validate_wms_url(capabilities_url, request_name="GetCapabilities") + capabilities_body, content_type, _ = _bounded_get( + capabilities_url, + timeout=args.wms_timeout, + max_bytes=MAX_CAPABILITIES_BYTES, + accept="application/xml,text/xml", + opener=opener, + ) + if content_type and "xml" not in content_type and "text" not in content_type: + raise RuntimeError("Official WMS capabilities response is not XML") + capabilities = parse_capabilities(capabilities_body, release["capabilities_sha256"]) + coverage_description_url = ( + f"https://{WMS_HOST}{WCS_PATH}?" + + urlencode( + { + "SERVICE": "WCS", + "VERSION": "2.0.1", + "REQUEST": "DescribeCoverage", + "COVERAGEID": "Ortho", + } + ) + ) + _validate_wcs_url(coverage_description_url, request_name="DescribeCoverage") + coverage_body, coverage_content_type, _ = _bounded_get( + coverage_description_url, + timeout=args.wms_timeout, + max_bytes=MAX_COVERAGE_DESCRIPTION_BYTES, + accept="application/xml,text/xml", + opener=opener, + validator=_validate_wcs_url, + ) + if coverage_content_type and "xml" not in coverage_content_type and "text" not in coverage_content_type: + raise RuntimeError("Official WCS coverage description response is not XML") + coverage_description = parse_coverage_description(coverage_body) + selection = validate_bbox(list(args.bbox), coverage_description["extent_epsg31370"]) + validate_bbox(list(args.bbox), capabilities["extent_epsg31370"]) + coverage = verify_flight_coverage( + base_url, + selection, + timeout=args.wms_timeout, + opener=opener, + ) + flight_year_matches = coverage["flight_years"] == [release["remote_year"]] + release_actionable = release["status"] in {"not_loaded", "update_available"} + staging_permitted = release_actionable and flight_year_matches + if release["status"] == "blocked_local_version": + next_action = "establish_official_local_edition_before_staging" + elif release["status"] == "current": + next_action = "none_current" + elif release["status"] == "blocked_remote_older": + next_action = "investigate_catalog_regression" + elif not flight_year_matches: + next_action = "split_or_review_selection_flight_years" + else: + next_action = "governed_pixel_stage" + return { + "schema_version": 1, + "status": "passed", + "generated_at": datetime.now(timezone.utc).isoformat(), + "project_id": args.project_id, + "scope": args.scope, + "product": product, + "release": release, + "capabilities": capabilities, + "coverage_domain": { + **coverage_description, + "selected_area_fully_inside_domain": True, + "pixel_data_requested": False, + }, + "selection": { + "bbox_epsg4326": selection["bbox_epsg4326"], + "bbox_epsg31370": [round(value, 3) for value in selection["bbox_epsg31370"]], + "width_m": round(selection["width_m"], 3), + "height_m": round(selection["height_m"], 3), + }, + "flight_day_coverage": coverage, + "flight_year_matches_release": flight_year_matches, + "staging_permitted": staging_permitted, + "next_action": next_action, + "pixel_requests_performed": 0, + "datasets_mutated": 0, + "automatic_staging": False, + "automatic_import": False, + } + + +def main() -> int: + try: + report = run_preflight(parse_args()) + print(json.dumps(report, ensure_ascii=False, indent=2)) + return 0 + except (RuntimeError, ValueError) as exc: + print(json.dumps({"status": "error", "error": str(exc)}, ensure_ascii=False, indent=2), file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index df3920d5..64156a32 100755 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -55,6 +55,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py ${PYTHON_BIN} -m py_compile scripts/provision_regional_bwk_natura2000.py ${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py ${PYTHON_BIN} -m py_compile scripts/manage_alz_agriculture_release.py +${PYTHON_BIN} -m py_compile scripts/orthophoto_release_preflight.py ${PYTHON_BIN} -m py_compile scripts/provision_buildings_addresses_register.py ${PYTHON_BIN} -m py_compile scripts/provision_mol_dhmv.py ${PYTHON_BIN} -m py_compile scripts/provision_regional_dhmv.py