#!/usr/bin/env python3 """Read-only release preflight for the governed current orthophoto product. The preflight does not request or stage raster pixels. It binds the canonical GeoIntel catalog result to the current product registry and the exact official WMS capabilities, then probes the queryable flight-day layer on a bounded, deterministic grid. Local rolling markers remain non-comparable and blocked. """ from __future__ import annotations import argparse from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timezone from hashlib import sha256 import json import math import os import re import sys from typing import Any, Callable from urllib.error import HTTPError, URLError from urllib.parse import parse_qs, urlencode, urlparse from urllib.request import Request, urlopen from xml.etree import ElementTree from pyproj import Transformer from geographic_scopes import GEOGRAPHIC_SCOPES DEFAULT_API_URL = "http://127.0.0.1:8000/api/v1" DEFAULT_SCOPE = "kempen-transport-region" SOURCE_NAME = "digitaal_vlaanderen_orthophoto" PRODUCT_KEY = "most_recent" WMS_HOST = "geo.api.vlaanderen.be" WMS_PATH = "/OMWRGBMRVL/wms" WCS_PATH = "/OMWRGBMRVL/wcs" CATALOG_URL = ( "https://www.vlaanderen.be/datavindplaats/catalogus/" "orthofotomozaiek-middenschalig-winteropnamen-kleur-meest-recent-vlaanderen" ) METADATA_IDENTIFIER = "f5304d6d-0dd4-43fd-a726-427af31e8d61" EDITION_PATTERN = re.compile(r"^(20[0-9]{2})\.([0-9]{2})$") SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") FLIGHT_YEAR_PATTERN = re.compile(r"(?:^|[^0-9])(20[0-9]{2})(?:$|[^0-9])") MAX_CAPABILITIES_BYTES = 2 * 1024 * 1024 MAX_COVERAGE_DESCRIPTION_BYTES = 512 * 1024 MAX_FEATURE_INFO_BYTES = 128 * 1024 MAX_SAMPLE_SPACING_M = 128.0 MAX_SAMPLE_COUNT = 64 MIN_SIDE_M = 128.0 MAX_SIDE_M = 1024.0 FEATURE_INFO_SIZE = 1000 def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description=( "Read-only current-orthophoto release preflight. No raster pixels are fetched, staged or imported." ) ) parser.add_argument("--project-id", required=True, help="GeoIntel project UUID for the governed scope") parser.add_argument("--scope", choices=(DEFAULT_SCOPE,), default=DEFAULT_SCOPE) parser.add_argument("--api-url", default=os.environ.get("GEOINTEL_API_URL", DEFAULT_API_URL)) parser.add_argument( "--bbox", nargs=4, type=float, metavar=("MIN_LON", "MIN_LAT", "MAX_LON", "MAX_LAT"), required=True, help="Exact EPSG:4326 selection; each projected side must be between 128 and 1024 metres", ) parser.add_argument("--refresh-catalog", action="store_true", help="Bypass the short catalog cache") parser.add_argument("--api-timeout", type=int, default=180) parser.add_argument("--wms-timeout", type=int, default=30) return parser.parse_args() def api_data(api_url: str, path: str, timeout: int) -> dict[str, Any]: endpoint = f"{api_url.rstrip('/')}/{path.lstrip('/')}" request = Request( endpoint, headers={"Accept": "application/json", "User-Agent": "GeoIntel-orthophoto-preflight/1.0"}, ) try: with urlopen(request, timeout=timeout) as response: payload = json.load(response) except HTTPError as exc: body = exc.read().decode("utf-8", errors="replace") raise RuntimeError(f"GeoIntel API returned HTTP {exc.code}: {body[-1000:]}") from exc except URLError as exc: raise RuntimeError(f"GeoIntel API is unreachable: {exc.reason}") from exc if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict): raise RuntimeError("GeoIntel API response is not a canonical data envelope") return payload["data"] def validate_project_scope(args: argparse.Namespace, loader: Callable[[str, str, int], dict[str, Any]]) -> None: project = loader(args.api_url, f"projects/{args.project_id}", args.api_timeout) expected_name = GEOGRAPHIC_SCOPES[args.scope].project_name if project.get("name") != expected_name: raise RuntimeError( f"Project {args.project_id} is '{project.get('name')}', but scope {args.scope} requires '{expected_name}'" ) def _validate_service_url(url: str, *, path: str, request_name: str | None = None) -> str: parsed = urlparse(url) if ( parsed.scheme.lower() != "https" or parsed.hostname != WMS_HOST or parsed.port not in (None, 443) or parsed.username or parsed.password or parsed.fragment or parsed.path.rstrip("/").lower() != path.lower() ): raise RuntimeError("Orthophoto service URL is outside the official allowlist") if request_name is not None: requests = parse_qs(parsed.query).get("REQUEST") or parse_qs(parsed.query).get("request") or [] if len(requests) != 1 or requests[0].lower() != request_name.lower(): raise RuntimeError(f"Orthophoto service URL is not an exact {request_name} request") return f"https://{WMS_HOST}{path}" def _validate_wms_url(url: str, *, request_name: str | None = None) -> str: return _validate_service_url(url, path=WMS_PATH, request_name=request_name) def _validate_wcs_url(url: str, *, request_name: str | None = None) -> str: return _validate_service_url(url, path=WCS_PATH, request_name=request_name) def _bounded_get( url: str, *, timeout: int, max_bytes: int, accept: str, opener: Callable[..., Any] | None = None, validator: Callable[[str], str] = _validate_wms_url, ) -> tuple[bytes, str, str]: request = Request(url, headers={"Accept": accept, "User-Agent": "GeoIntel-orthophoto-preflight/1.0"}) fetch = opener or urlopen try: response = fetch(request, timeout=timeout) with response: final_url = response.geturl() validator(final_url) content_type = str(response.headers.get("Content-Type") or "").split(";", 1)[0].strip().lower() declared = response.headers.get("Content-Length") try: if declared and int(declared) > max_bytes: raise RuntimeError("Official WMS response exceeds the configured preflight limit") except ValueError as exc: raise RuntimeError("Official WMS response has an invalid Content-Length") from exc body = response.read(max_bytes + 1) except HTTPError as exc: raise RuntimeError(f"Official orthophoto WMS returned HTTP {exc.code}") from exc except URLError as exc: raise RuntimeError(f"Official orthophoto WMS is unreachable: {exc.reason}") from exc if len(body) > max_bytes: raise RuntimeError("Official WMS response exceeds the configured preflight limit") return body, content_type, final_url def _local_name(tag: str) -> str: return tag.rsplit("}", 1)[-1] def _children(element: ElementTree.Element, name: str) -> list[ElementTree.Element]: return [child for child in element if _local_name(child.tag) == name] def _child_text(element: ElementTree.Element, name: str) -> str | None: for child in element: if _local_name(child.tag) == name and child.text: return child.text.strip() return None def parse_capabilities(body: bytes, expected_sha256: str) -> dict[str, Any]: if not SHA256_PATTERN.fullmatch(expected_sha256): raise RuntimeError("Canonical catalog response has no valid capabilities SHA-256") actual_sha256 = sha256(body).hexdigest() if actual_sha256 != expected_sha256: raise RuntimeError("Official WMS capabilities changed after the canonical catalog check") upper = body[:4096].upper() if b"= bbox[2] or bbox[1] >= bbox[3]: raise RuntimeError("Official WMS does not advertise a valid EPSG:31370 extent") metadata_identifiers: set[str] = set() for layer_name in ("Ortho", "Vliegdagcontour"): for node in layers[layer_name].iter(): if _local_name(node.tag) != "OnlineResource": continue href = next((value for key, value in node.attrib.items() if _local_name(key) == "href"), "") if METADATA_IDENTIFIER in href: metadata_identifiers.add(METADATA_IDENTIFIER) if metadata_identifiers != {METADATA_IDENTIFIER}: raise RuntimeError("Governed orthophoto layers do not share the expected ISO metadata identity") return { "service_version": "1.3.0", "capabilities_sha256": actual_sha256, "layers": ["Ortho", "Vliegdagcontour"], "vliegdagcontour_queryable": True, "feature_info_format": "application/geo+json", "extent_epsg31370": bbox, "metadata_identifier": METADATA_IDENTIFIER, } def parse_coverage_description(body: bytes) -> dict[str, Any]: upper = body[:4096].upper() if b"= upper_corner[0] or lower[1] >= upper_corner[1]: raise RuntimeError("Official WCS Ortho domain corners are invalid") grid = next((node for node in root.iter() if _local_name(node.tag) == "RectifiedGrid"), None) if grid is None or grid.attrib.get("dimension") != "2": raise RuntimeError("Official WCS Ortho domain is not a two-dimensional rectified grid") offsets: list[list[float]] = [] for node in grid.iter(): if _local_name(node.tag) == "offsetVector" and node.text: try: offsets.append([float(value) for value in node.text.split()]) except ValueError as exc: raise RuntimeError("Official WCS Ortho grid resolution is invalid") from exc if ( len(offsets) != 2 or any(len(vector) != 2 for vector in offsets) or not math.isclose(offsets[0][0], 0.15, abs_tol=1e-9) or not math.isclose(offsets[0][1], 0.0, abs_tol=1e-9) or not math.isclose(offsets[1][0], 0.0, abs_tol=1e-9) or not math.isclose(offsets[1][1], -0.15, abs_tol=1e-9) ): raise RuntimeError("Official WCS Ortho grid is no longer the governed 15 cm product") field_count = sum(_local_name(node.tag) == "field" for node in root.iter()) subtype = next( ((node.text or "").strip() for node in root.iter() if _local_name(node.tag) == "CoverageSubtype"), "", ) native_format = next( ((node.text or "").strip() for node in root.iter() if _local_name(node.tag) == "nativeFormat"), "", ) if field_count != 3 or subtype != "RectifiedGridCoverage" or native_format != "image/tiff": raise RuntimeError("Official WCS Ortho range or native format changed") return { "coverage_id": "Ortho", "crs": "EPSG:31370", "extent_epsg31370": [lower[0], lower[1], upper_corner[0], upper_corner[1]], "native_resolution_m": 0.15, "band_count": 3, "native_format": "image/tiff", "coverage_description_sha256": sha256(body).hexdigest(), } def validate_bbox(values: list[float], advertised_extent: list[float]) -> dict[str, Any]: if len(values) != 4 or not all(math.isfinite(value) for value in values): raise RuntimeError("Selection bbox must contain four finite EPSG:4326 values") min_x, min_y, max_x, max_y = values if min_x >= max_x or min_y >= max_y or not (-180 <= min_x < max_x <= 180) or not (-90 <= min_y < max_y <= 90): raise RuntimeError("Selection bbox is not a valid EPSG:4326 rectangle") transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True) lambert = [float(value) for value in transformer.transform_bounds(min_x, min_y, max_x, max_y, densify_pts=21)] width_m = lambert[2] - lambert[0] height_m = lambert[3] - lambert[1] if width_m < MIN_SIDE_M or height_m < MIN_SIDE_M: raise RuntimeError(f"Orthophoto preflight selection must be at least {MIN_SIDE_M:.0f} by {MIN_SIDE_M:.0f} metres") if width_m > MAX_SIDE_M or height_m > MAX_SIDE_M: raise RuntimeError(f"Orthophoto preflight selection may not exceed {MAX_SIDE_M:.0f} by {MAX_SIDE_M:.0f} metres") if not ( advertised_extent[0] <= lambert[0] and advertised_extent[1] <= lambert[1] and lambert[2] <= advertised_extent[2] and lambert[3] <= advertised_extent[3] ): raise RuntimeError("Selection is outside the official WMS advertised EPSG:31370 extent") return { "bbox_epsg4326": [min_x, min_y, max_x, max_y], "bbox_epsg31370": lambert, "width_m": width_m, "height_m": height_m, } def validate_product(items: list[dict[str, Any]]) -> dict[str, Any]: matches = [item for item in items if item.get("key") == PRODUCT_KEY] if len(matches) != 1: raise RuntimeError("Orthophoto product registry does not contain exactly one most_recent product") product = matches[0] expected = { "temporal_granularity": "snapshot", "native_resolution_m": 0.15, "supports_detection": True, "color_mode": "rgb", "catalog_url": CATALOG_URL, } if any(product.get(key) != value for key, value in expected.items()): raise RuntimeError("Current orthophoto product variant no longer matches the governed registry contract") return {"key": PRODUCT_KEY, **expected, "display_name": product.get("display_name")} def catalog_decision(item: dict[str, Any]) -> dict[str, Any]: remote_version = str(item.get("remote_version") or "") remote_match = EDITION_PATTERN.fullmatch(remote_version) if ( item.get("source_name") != SOURCE_NAME or item.get("status") != "available" or item.get("reachable") is not True or set(item.get("matched_layers") or []) != {"Ortho", "Vliegdagcontour"} or item.get("missing_layers") not in (None, []) or item.get("metadata_identifier") != METADATA_IDENTIFIER or not SHA256_PATTERN.fullmatch(str(item.get("capabilities_sha256") or "")) or not remote_match or remote_version not in str(item.get("remote_title") or "") ): raise RuntimeError("Canonical catalog report does not contain one complete governed orthophoto release") local_version = str(item.get("local_source_version") or "") local_match = EDITION_PATTERN.fullmatch(local_version) if not local_version: status = "not_loaded" expected_comparison = "no_local_data" elif not local_match: status = "blocked_local_version" expected_comparison = "not_comparable" else: local_order = tuple(int(value) for value in local_match.groups()) remote_order = tuple(int(value) for value in remote_match.groups()) if local_order == remote_order: status = "current" expected_comparison = "same" elif local_order < remote_order: status = "update_available" expected_comparison = "different" else: status = "blocked_remote_older" expected_comparison = "different" if item.get("comparison_status") != expected_comparison: raise RuntimeError("Canonical catalog local/remote comparison is internally inconsistent") return { "status": status, "remote_edition": remote_version, "remote_year": int(remote_match.group(1)), "local_source_version": local_version or None, "comparison_status": expected_comparison, "metadata_identifier": METADATA_IDENTIFIER, "metadata_url": item.get("metadata_url"), "remote_title": item.get("remote_title"), "remote_modified_at": item.get("remote_modified_at"), "remote_published_at": item.get("remote_published_at"), "catalog_checked_at": item.get("checked_at"), "capabilities_url": item.get("endpoint_url"), "capabilities_sha256": item.get("capabilities_sha256"), } def _sample_grid(selection: dict[str, Any]) -> list[dict[str, Any]]: count_x = max(1, math.ceil(selection["width_m"] / MAX_SAMPLE_SPACING_M)) count_y = max(1, math.ceil(selection["height_m"] / MAX_SAMPLE_SPACING_M)) if count_x * count_y > MAX_SAMPLE_COUNT: raise RuntimeError("Deterministic flight-day grid exceeds the fixed preflight limit") min_x, min_y, max_x, max_y = selection["bbox_epsg31370"] samples: list[dict[str, Any]] = [] for row in range(count_y): fraction_y = (row + 0.5) / count_y for column in range(count_x): fraction_x = (column + 0.5) / count_x samples.append( { "column": column, "row": row, "i": min(FEATURE_INFO_SIZE - 1, int(fraction_x * FEATURE_INFO_SIZE)), "j": min(FEATURE_INFO_SIZE - 1, int((1.0 - fraction_y) * FEATURE_INFO_SIZE)), "x": min_x + fraction_x * (max_x - min_x), "y": min_y + fraction_y * (max_y - min_y), } ) return samples def _flight_sample( base_url: str, selection: dict[str, Any], sample: dict[str, Any], *, timeout: int, opener: Callable[..., Any] | None, ) -> dict[str, Any]: params = { "SERVICE": "WMS", "VERSION": "1.3.0", "REQUEST": "GetFeatureInfo", "LAYERS": "Vliegdagcontour", "QUERY_LAYERS": "Vliegdagcontour", "STYLES": "", "CRS": "EPSG:31370", "BBOX": ",".join(f"{value:.3f}" for value in selection["bbox_epsg31370"]), "WIDTH": str(FEATURE_INFO_SIZE), "HEIGHT": str(FEATURE_INFO_SIZE), "I": str(sample["i"]), "J": str(sample["j"]), "INFO_FORMAT": "application/geo+json", "FEATURE_COUNT": "10", "FORMAT": "image/png", } body, content_type, _ = _bounded_get( f"{base_url}?{urlencode(params)}", timeout=timeout, max_bytes=MAX_FEATURE_INFO_BYTES, accept="application/geo+json", opener=opener, ) if content_type not in {"application/geo+json", "application/json"}: raise RuntimeError("Vliegdagcontour feature-info response is not GeoJSON") try: payload = json.loads(body) except (UnicodeDecodeError, json.JSONDecodeError) as exc: raise RuntimeError("Vliegdagcontour feature-info response is invalid JSON") from exc features = payload.get("features") if isinstance(payload, dict) else None if not isinstance(payload, dict) or payload.get("type") != "FeatureCollection" or not isinstance(features, list) or not features: raise RuntimeError("Vliegdagcontour has no coverage at one deterministic selection sample") dates: set[str] = set() feature_ids: set[str] = set() years: set[int] = set() for feature in features: if not isinstance(feature, dict) or feature.get("layerName") != "Vliegdagcontour": raise RuntimeError("Vliegdagcontour feature-info returned an unexpected layer") properties = feature.get("properties") if not isinstance(properties, dict): raise RuntimeError("Vliegdagcontour feature-info has no properties") date_value = str(properties.get("OpnDatum") or "").strip() match = FLIGHT_YEAR_PATTERN.search(date_value) if not match: raise RuntimeError("Vliegdagcontour feature-info has no parseable acquisition year") dates.add(date_value) years.add(int(match.group(1))) if properties.get("FID") is not None: feature_ids.add(str(properties["FID"])) return { "column": sample["column"], "row": sample["row"], "x": round(sample["x"], 3), "y": round(sample["y"], 3), "dates": sorted(dates), "years": sorted(years), "feature_ids": sorted(feature_ids), "response_sha256": sha256(body).hexdigest(), } def verify_flight_coverage( base_url: str, selection: dict[str, Any], *, timeout: int, opener: Callable[..., Any] | None = None, ) -> dict[str, Any]: samples = _sample_grid(selection) worker_count = min(8, len(samples)) with ThreadPoolExecutor(max_workers=worker_count) as executor: evidence = list( executor.map( lambda sample: _flight_sample( base_url, selection, sample, timeout=timeout, opener=opener, ), samples, ) ) years = sorted({year for item in evidence for year in item["years"]}) dates = sorted({date for item in evidence for date in item["dates"]}) feature_ids = sorted({feature_id for item in evidence for feature_id in item["feature_ids"]}) canonical = json.dumps(evidence, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") count_x = len({item["column"] for item in evidence}) count_y = len({item["row"] for item in evidence}) return { "status": "passed", "mode": "official_queryable_flight_day_grid", "sample_count": len(evidence), "grid_columns": count_x, "grid_rows": count_y, "maximum_sample_spacing_m": MAX_SAMPLE_SPACING_M, "covered_sample_count": len(evidence), "sample_coverage_ratio": 1.0, "flight_dates": dates, "flight_years": years, "feature_ids": feature_ids, "sample_evidence_sha256": sha256(canonical).hexdigest(), "claim_boundary": ( "Every deterministic grid sample is covered by the official queryable flight-day layer. " "The WMS exposes no vector geometry, so this is bounded point evidence and not a polygon-union proof." ), } def run_preflight( args: argparse.Namespace, *, loader: Callable[[str, str, int], dict[str, Any]] = api_data, opener: Callable[..., Any] | None = None, ) -> dict[str, Any]: validate_project_scope(args, loader) products = loader( args.api_url, f"projects/{args.project_id}/datasets/orthophoto/products", args.api_timeout, ) product = validate_product(products.get("items") or []) refresh = "true" if args.refresh_catalog else "false" report = loader( args.api_url, f"projects/{args.project_id}/datasets/source-catalog-probes?refresh={refresh}", args.api_timeout, ) matches = [item for item in report.get("items") or [] if item.get("source_name") == SOURCE_NAME] if len(matches) != 1: raise RuntimeError("Source catalog report did not contain exactly one orthophoto contract") release = catalog_decision(matches[0]) capabilities_url = str(release["capabilities_url"] or "") base_url = _validate_wms_url(capabilities_url, request_name="GetCapabilities") capabilities_body, content_type, _ = _bounded_get( capabilities_url, timeout=args.wms_timeout, max_bytes=MAX_CAPABILITIES_BYTES, accept="application/xml,text/xml", opener=opener, ) if content_type and "xml" not in content_type and "text" not in content_type: raise RuntimeError("Official WMS capabilities response is not XML") capabilities = parse_capabilities(capabilities_body, release["capabilities_sha256"]) coverage_description_url = ( f"https://{WMS_HOST}{WCS_PATH}?" + urlencode( { "SERVICE": "WCS", "VERSION": "2.0.1", "REQUEST": "DescribeCoverage", "COVERAGEID": "Ortho", } ) ) _validate_wcs_url(coverage_description_url, request_name="DescribeCoverage") coverage_body, coverage_content_type, _ = _bounded_get( coverage_description_url, timeout=args.wms_timeout, max_bytes=MAX_COVERAGE_DESCRIPTION_BYTES, accept="application/xml,text/xml", opener=opener, validator=_validate_wcs_url, ) if coverage_content_type and "xml" not in coverage_content_type and "text" not in coverage_content_type: raise RuntimeError("Official WCS coverage description response is not XML") coverage_description = parse_coverage_description(coverage_body) selection = validate_bbox(list(args.bbox), coverage_description["extent_epsg31370"]) validate_bbox(list(args.bbox), capabilities["extent_epsg31370"]) coverage = verify_flight_coverage( base_url, selection, timeout=args.wms_timeout, opener=opener, ) flight_year_matches = coverage["flight_years"] == [release["remote_year"]] release_actionable = release["status"] in {"not_loaded", "update_available"} staging_permitted = release_actionable and flight_year_matches if release["status"] == "blocked_local_version": next_action = "establish_official_local_edition_before_staging" elif release["status"] == "current": next_action = "none_current" elif release["status"] == "blocked_remote_older": next_action = "investigate_catalog_regression" elif not flight_year_matches: next_action = "split_or_review_selection_flight_years" else: next_action = "governed_pixel_stage" return { "schema_version": 1, "status": "passed", "generated_at": datetime.now(timezone.utc).isoformat(), "project_id": args.project_id, "scope": args.scope, "product": product, "release": release, "capabilities": capabilities, "coverage_domain": { **coverage_description, "selected_area_fully_inside_domain": True, "pixel_data_requested": False, }, "selection": { "bbox_epsg4326": selection["bbox_epsg4326"], "bbox_epsg31370": [round(value, 3) for value in selection["bbox_epsg31370"]], "width_m": round(selection["width_m"], 3), "height_m": round(selection["height_m"], 3), }, "flight_day_coverage": coverage, "flight_year_matches_release": flight_year_matches, "staging_permitted": staging_permitted, "next_action": next_action, "pixel_requests_performed": 0, "datasets_mutated": 0, "automatic_staging": False, "automatic_import": False, } def main() -> int: try: report = run_preflight(parse_args()) print(json.dumps(report, ensure_ascii=False, indent=2)) return 0 except (RuntimeError, ValueError) as exc: print(json.dumps({"status": "error", "error": str(exc)}, ensure_ascii=False, indent=2), file=sys.stderr) return 1 if __name__ == "__main__": raise SystemExit(main())