Add governed orthophoto release preflight
This commit is contained in:
@@ -7,6 +7,20 @@
|
||||
|
||||
# Changelog
|
||||
|
||||
## Sprint 230 Governed orthophoto release preflight (2026-07-17)
|
||||
|
||||
- Added an operator-only, read-only preflight for the current Digitaal
|
||||
Vlaanderen orthophoto product. It binds the canonical product and catalog
|
||||
envelopes to exact official WMS capabilities and ISO edition evidence.
|
||||
- Added metadata-only WCS `DescribeCoverage` validation for the governed
|
||||
EPSG:31370, 15 cm, three-band `Ortho` raster domain and deterministic bounded
|
||||
`Vliegdagcontour` sampling for selected-area flight-year evidence.
|
||||
- Classified local official editions as current/update/baseline/remote-older
|
||||
and kept legacy `most_recent_at_*` values explicitly non-comparable. No
|
||||
raster pixels, Datasets, Jobs, migrations or automatic refresh were added.
|
||||
- Added trust-boundary, hash-drift, product, coverage, flight-year, bounds and
|
||||
runtime-packaging tests plus readiness compilation and operator docs.
|
||||
|
||||
## Sprint 229 Governed ALZ definitive release promotion (2026-07-17)
|
||||
|
||||
- Added an operator-only `plan -> stage -> review -> apply` coordinator for
|
||||
|
||||
@@ -1300,6 +1300,26 @@ Settings: `ORTHOPHOTO_ENABLED`, `ORTHOPHOTO_WMS_URL`,
|
||||
`ORTHOPHOTO_CACHE_TTL_HOURS`. Keep the official HTTPS URL and 1 m profile
|
||||
unless a separately verified deployment/model profile requires a change.
|
||||
|
||||
Before a future `most_recent` source release is allowed into a governed pixel
|
||||
stage, run the metadata-only preflight for the exact intended rectangle:
|
||||
|
||||
```bash
|
||||
docker exec geointel python /app/scripts/orthophoto_release_preflight.py \
|
||||
--project-id 82a85913-c522-45d7-84a1-02b393d89e55 \
|
||||
--api-url http://127.0.0.1:8000/api/v1 \
|
||||
--bbox 5.110 51.180 5.117 51.185 \
|
||||
--refresh-catalog
|
||||
```
|
||||
|
||||
The command reads canonical API envelopes, exact official WMS capabilities,
|
||||
WCS `DescribeCoverage` and at most 64 queryable flight-day points. It never
|
||||
requests raster pixels or mutates application/storage state. Only
|
||||
`staging_permitted=true` may feed a future separate staging command. `current`,
|
||||
remote-older, mixed/incorrect flight years and legacy local values such as
|
||||
`most_recent_at_2026-07-15` remain non-stageable. The report's point grid is
|
||||
flight-date evidence; complete selected-area coverage comes from containment
|
||||
inside the official 15 cm WCS raster domain.
|
||||
|
||||
## Governed DHMV terrain acquisition
|
||||
|
||||
`GET /api/v1/projects/{project_id}/datasets/dhmv/products` exposes the fixed
|
||||
|
||||
@@ -92,6 +92,7 @@ def test_all_in_one_dockerfile_copies_operator_scripts_for_runtime_use() -> None
|
||||
"run_split_background_promotion_workflow.sh",
|
||||
"activate_promoted_yolo_candidate.py",
|
||||
"manage_grb_refresh.py",
|
||||
"orthophoto_release_preflight.py",
|
||||
}
|
||||
for script_name in required_runtime_scripts:
|
||||
assert f"COPY scripts/{script_name} /app/scripts/{script_name}" in dockerfile
|
||||
|
||||
@@ -0,0 +1,365 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from email.message import Message
|
||||
from hashlib import sha256
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from urllib.parse import parse_qs, urlparse
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
|
||||
|
||||
def load_script():
|
||||
path = SCRIPTS / "orthophoto_release_preflight.py"
|
||||
module_name = "test_orthophoto_release_preflight_sprint230"
|
||||
spec = importlib.util.spec_from_file_location(module_name, path)
|
||||
assert spec is not None
|
||||
assert spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[module_name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
PREFLIGHT = load_script()
|
||||
PROJECT_ID = "00000000-0000-0000-0000-000000000001"
|
||||
|
||||
|
||||
def arguments(**overrides) -> argparse.Namespace:
|
||||
values = {
|
||||
"project_id": PROJECT_ID,
|
||||
"scope": PREFLIGHT.DEFAULT_SCOPE,
|
||||
"api_url": "http://127.0.0.1:8000/api/v1",
|
||||
"bbox": [5.110, 51.180, 5.117, 51.185],
|
||||
"refresh_catalog": True,
|
||||
"api_timeout": 30,
|
||||
"wms_timeout": 10,
|
||||
}
|
||||
values.update(overrides)
|
||||
return argparse.Namespace(**values)
|
||||
|
||||
|
||||
def capabilities(*, queryable: bool = True, feature_info: bool = True) -> bytes:
|
||||
info_format = "<Format>application/geo+json</Format>" if feature_info else "<Format>text/plain</Format>"
|
||||
queryable_value = "1" if queryable else "0"
|
||||
metadata_url = (
|
||||
"https://metadata.vlaanderen.be/srv/dut/csw?request=GetRecordById&service=CSW&"
|
||||
f"id={PREFLIGHT.METADATA_IDENTIFIER}"
|
||||
)
|
||||
return f"""
|
||||
<WMS_Capabilities version="1.3.0" xmlns="http://www.opengis.net/wms"
|
||||
xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<Capability>
|
||||
<Request><GetFeatureInfo>{info_format}</GetFeatureInfo></Request>
|
||||
<Layer>
|
||||
<CRS>EPSG:31370</CRS>
|
||||
<BoundingBox CRS="EPSG:31370" minx="22000" miny="150000" maxx="259000" maxy="245000" />
|
||||
<Layer queryable="0"><Name>Ortho</Name><MetadataURL><OnlineResource xlink:href="{metadata_url}" /></MetadataURL></Layer>
|
||||
<Layer queryable="{queryable_value}"><Name>Vliegdagcontour</Name><MetadataURL><OnlineResource xlink:href="{metadata_url}" /></MetadataURL></Layer>
|
||||
</Layer>
|
||||
</Capability>
|
||||
</WMS_Capabilities>
|
||||
""".encode()
|
||||
|
||||
|
||||
def coverage_description(*, coverage_id: str = "Ortho", resolution: float = 0.15) -> bytes:
|
||||
return f"""
|
||||
<wcs:CoverageDescriptions xmlns:wcs="http://www.opengis.net/wcs/2.0"
|
||||
xmlns:gml="http://www.opengis.net/gml/3.2" xmlns:gmlcov="http://www.opengis.net/gmlcov/1.0"
|
||||
xmlns:swe="http://www.opengis.net/swe/2.0">
|
||||
<wcs:CoverageDescription>
|
||||
<gml:boundedBy><gml:Envelope srsName="http://www.opengis.net/def/crs/EPSG/0/31370">
|
||||
<gml:lowerCorner>21375 152250</gml:lowerCorner><gml:upperCorner>259500 244875</gml:upperCorner>
|
||||
</gml:Envelope></gml:boundedBy>
|
||||
<wcs:CoverageId>{coverage_id}</wcs:CoverageId>
|
||||
<gml:domainSet><gml:RectifiedGrid dimension="2">
|
||||
<gml:offsetVector>{resolution} 0</gml:offsetVector><gml:offsetVector>0 -{resolution}</gml:offsetVector>
|
||||
</gml:RectifiedGrid></gml:domainSet>
|
||||
<gmlcov:rangeType><swe:DataRecord>
|
||||
<swe:field name="Band_1"/><swe:field name="Band_2"/><swe:field name="Band_3"/>
|
||||
</swe:DataRecord></gmlcov:rangeType>
|
||||
<wcs:ServiceParameters><wcs:CoverageSubtype>RectifiedGridCoverage</wcs:CoverageSubtype>
|
||||
<wcs:nativeFormat>image/tiff</wcs:nativeFormat></wcs:ServiceParameters>
|
||||
</wcs:CoverageDescription>
|
||||
</wcs:CoverageDescriptions>
|
||||
""".encode()
|
||||
|
||||
|
||||
def catalog_item(body: bytes, *, local: str | None = "most_recent_at_2026-07-15", remote: str = "2025.04") -> dict:
|
||||
if local is None:
|
||||
comparison = "no_local_data"
|
||||
elif PREFLIGHT.EDITION_PATTERN.fullmatch(local):
|
||||
comparison = "same" if local == remote else "different"
|
||||
else:
|
||||
comparison = "not_comparable"
|
||||
return {
|
||||
"source_name": PREFLIGHT.SOURCE_NAME,
|
||||
"status": "available",
|
||||
"reachable": True,
|
||||
"matched_layers": ["Ortho", "Vliegdagcontour"],
|
||||
"missing_layers": [],
|
||||
"metadata_identifier": PREFLIGHT.METADATA_IDENTIFIER,
|
||||
"metadata_url": f"https://metadata.vlaanderen.be/{PREFLIGHT.METADATA_IDENTIFIER}",
|
||||
"remote_title": f"Orthofoto meest recent, {remote}",
|
||||
"remote_version": remote,
|
||||
"remote_modified_at": "2026-04-27T00:00:00Z",
|
||||
"remote_published_at": "2025-12-11T00:00:00Z",
|
||||
"local_source_version": local,
|
||||
"comparison_status": comparison,
|
||||
"checked_at": "2026-07-17T00:00:00Z",
|
||||
"endpoint_url": (
|
||||
"https://geo.api.vlaanderen.be/OMWRGBMRVL/wms?"
|
||||
"SERVICE=WMS&VERSION=1.3.0&REQUEST=GetCapabilities"
|
||||
),
|
||||
"capabilities_sha256": sha256(body).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def product() -> dict:
|
||||
return {
|
||||
"key": "most_recent",
|
||||
"display_name": "Meest recente winterluchtbeeld",
|
||||
"observation_label": "Meest recent beschikbaar",
|
||||
"temporal_granularity": "snapshot",
|
||||
"native_resolution_m": 0.15,
|
||||
"supports_detection": True,
|
||||
"color_mode": "rgb",
|
||||
"catalog_url": PREFLIGHT.CATALOG_URL,
|
||||
"limitation_message": "rolling source",
|
||||
}
|
||||
|
||||
|
||||
def loader(body: bytes, item: dict):
|
||||
def load(_api_url: str, path: str, _timeout: int) -> dict:
|
||||
if path == f"projects/{PROJECT_ID}":
|
||||
return {"id": PROJECT_ID, "name": "Kempen Regional Workbench"}
|
||||
if path.endswith("/datasets/orthophoto/products"):
|
||||
return {"items": [product()], "total": 1}
|
||||
if "/datasets/source-catalog-probes?" in path:
|
||||
return {"items": [item]}
|
||||
raise AssertionError(path)
|
||||
|
||||
return load
|
||||
|
||||
|
||||
class Response:
|
||||
def __init__(self, body: bytes, *, url: str, content_type: str) -> None:
|
||||
self.body = body
|
||||
self.url = url
|
||||
self.headers = Message()
|
||||
self.headers["Content-Type"] = content_type
|
||||
self.headers["Content-Length"] = str(len(body))
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args) -> None:
|
||||
return None
|
||||
|
||||
def geturl(self) -> str:
|
||||
return self.url
|
||||
|
||||
def read(self, size: int = -1) -> bytes:
|
||||
return self.body if size < 0 else self.body[:size]
|
||||
|
||||
|
||||
def opener(
|
||||
body: bytes,
|
||||
*,
|
||||
flight_year: int = 2025,
|
||||
empty: bool = False,
|
||||
requests: list[str] | None = None,
|
||||
coverage_body: bytes | None = None,
|
||||
):
|
||||
def open_request(request, timeout: int):
|
||||
assert timeout == 10
|
||||
url = request.full_url
|
||||
query = parse_qs(urlparse(url).query)
|
||||
request_name = (query.get("REQUEST") or [""])[0]
|
||||
if requests is not None:
|
||||
requests.append(request_name)
|
||||
if request_name == "GetCapabilities":
|
||||
return Response(body, url=url, content_type="text/xml")
|
||||
if request_name == "DescribeCoverage":
|
||||
return Response(coverage_body or coverage_description(), url=url, content_type="text/xml")
|
||||
assert request_name == "GetFeatureInfo"
|
||||
assert query["LAYERS"] == ["Vliegdagcontour"]
|
||||
features = [] if empty else [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": None,
|
||||
"properties": {"OpnDatum": f"5/4/{flight_year}", "FID": "4"},
|
||||
"layerName": "Vliegdagcontour",
|
||||
}
|
||||
]
|
||||
payload = json.dumps({"type": "FeatureCollection", "features": features}).encode()
|
||||
return Response(payload, url=url, content_type="application/geo+json")
|
||||
|
||||
return open_request
|
||||
|
||||
|
||||
def run(*, local: str | None = "most_recent_at_2026-07-15", remote: str = "2025.04", flight_year: int = 2025):
|
||||
body = capabilities()
|
||||
item = catalog_item(body, local=local, remote=remote)
|
||||
calls: list[str] = []
|
||||
report = PREFLIGHT.run_preflight(
|
||||
arguments(),
|
||||
loader=loader(body, item),
|
||||
opener=opener(body, flight_year=flight_year, requests=calls),
|
||||
)
|
||||
return report, calls
|
||||
|
||||
|
||||
def test_live_contract_shape_blocks_non_comparable_legacy_local_version() -> None:
|
||||
report, calls = run()
|
||||
|
||||
assert report["status"] == "passed"
|
||||
assert report["release"]["status"] == "blocked_local_version"
|
||||
assert report["release"]["remote_edition"] == "2025.04"
|
||||
assert report["flight_day_coverage"]["sample_count"] == 20
|
||||
assert report["flight_day_coverage"]["sample_coverage_ratio"] == 1.0
|
||||
assert report["flight_day_coverage"]["flight_years"] == [2025]
|
||||
assert report["staging_permitted"] is False
|
||||
assert report["next_action"] == "establish_official_local_edition_before_staging"
|
||||
assert report["pixel_requests_performed"] == 0
|
||||
assert set(calls) == {"GetCapabilities", "DescribeCoverage", "GetFeatureInfo"}
|
||||
assert report["coverage_domain"]["selected_area_fully_inside_domain"] is True
|
||||
assert report["coverage_domain"]["pixel_data_requested"] is False
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("local", "remote", "expected_status", "stageable"),
|
||||
[
|
||||
(None, "2025.04", "not_loaded", True),
|
||||
("2025.04", "2025.04", "current", False),
|
||||
("2024.01", "2025.04", "update_available", True),
|
||||
("2026.01", "2025.04", "blocked_remote_older", False),
|
||||
],
|
||||
)
|
||||
def test_release_ordering_requires_comparable_official_editions(
|
||||
local: str | None,
|
||||
remote: str,
|
||||
expected_status: str,
|
||||
stageable: bool,
|
||||
) -> None:
|
||||
report, _ = run(local=local, remote=remote)
|
||||
|
||||
assert report["release"]["status"] == expected_status
|
||||
assert report["staging_permitted"] is stageable
|
||||
|
||||
|
||||
def test_flight_year_must_match_remote_release_year() -> None:
|
||||
report, _ = run(local="2024.01", flight_year=2024)
|
||||
|
||||
assert report["release"]["status"] == "update_available"
|
||||
assert report["flight_year_matches_release"] is False
|
||||
assert report["staging_permitted"] is False
|
||||
assert report["next_action"] == "split_or_review_selection_flight_years"
|
||||
|
||||
|
||||
def test_capabilities_hash_drift_fails_closed() -> None:
|
||||
body = capabilities()
|
||||
item = catalog_item(body, local=None)
|
||||
changed = body.replace(b"queryable=\"1\"", b"queryable=\"0\"")
|
||||
|
||||
with pytest.raises(RuntimeError, match="changed after"):
|
||||
PREFLIGHT.run_preflight(
|
||||
arguments(),
|
||||
loader=loader(body, item),
|
||||
opener=opener(changed),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("body", "message"),
|
||||
[
|
||||
(capabilities(queryable=False), "not queryable"),
|
||||
(capabilities(feature_info=False), "does not advertise GeoJSON"),
|
||||
],
|
||||
)
|
||||
def test_flight_day_capability_requirements_fail_closed(body: bytes, message: str) -> None:
|
||||
item = catalog_item(body, local=None)
|
||||
with pytest.raises(RuntimeError, match=message):
|
||||
PREFLIGHT.run_preflight(arguments(), loader=loader(body, item), opener=opener(body))
|
||||
|
||||
|
||||
def test_missing_flight_day_coverage_fails_closed() -> None:
|
||||
body = capabilities()
|
||||
item = catalog_item(body, local=None)
|
||||
with pytest.raises(RuntimeError, match="has no coverage"):
|
||||
PREFLIGHT.run_preflight(
|
||||
arguments(),
|
||||
loader=loader(body, item),
|
||||
opener=opener(body, empty=True),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"coverage_body",
|
||||
[coverage_description(coverage_id="Other"), coverage_description(resolution=0.25)],
|
||||
)
|
||||
def test_coverage_domain_identity_and_resolution_fail_closed(coverage_body: bytes) -> None:
|
||||
body = capabilities()
|
||||
item = catalog_item(body, local=None)
|
||||
with pytest.raises(RuntimeError, match="WCS"):
|
||||
PREFLIGHT.run_preflight(
|
||||
arguments(),
|
||||
loader=loader(body, item),
|
||||
opener=opener(body, coverage_body=coverage_body),
|
||||
)
|
||||
|
||||
|
||||
def test_product_variant_and_catalog_comparison_are_bound() -> None:
|
||||
wrong = product()
|
||||
wrong["color_mode"] = "panchromatic"
|
||||
with pytest.raises(RuntimeError, match="product variant"):
|
||||
PREFLIGHT.validate_product([wrong])
|
||||
|
||||
body = capabilities()
|
||||
item = catalog_item(body, local="2025.04")
|
||||
item["comparison_status"] = "different"
|
||||
with pytest.raises(RuntimeError, match="internally inconsistent"):
|
||||
PREFLIGHT.catalog_decision(item)
|
||||
|
||||
|
||||
def test_only_exact_official_wms_url_is_allowed() -> None:
|
||||
valid = "https://geo.api.vlaanderen.be/OMWRGBMRVL/wms?SERVICE=WMS&REQUEST=GetCapabilities"
|
||||
assert PREFLIGHT._validate_wms_url(valid, request_name="GetCapabilities").endswith("/OMWRGBMRVL/wms")
|
||||
with pytest.raises(RuntimeError, match="outside the official allowlist"):
|
||||
PREFLIGHT._validate_wms_url(valid.replace("geo.api.vlaanderen.be", "example.com"))
|
||||
with pytest.raises(RuntimeError, match="not an exact"):
|
||||
PREFLIGHT._validate_wms_url(valid.replace("GetCapabilities", "GetMap"), request_name="GetCapabilities")
|
||||
|
||||
|
||||
def test_bbox_matches_acquisition_bounds_and_grid_is_bounded() -> None:
|
||||
extent = [22000.0, 150000.0, 259000.0, 245000.0]
|
||||
selection = PREFLIGHT.validate_bbox([5.110, 51.180, 5.117, 51.185], extent)
|
||||
samples = PREFLIGHT._sample_grid(selection)
|
||||
assert len(samples) == 20
|
||||
assert len(samples) <= PREFLIGHT.MAX_SAMPLE_COUNT
|
||||
with pytest.raises(RuntimeError, match="at least"):
|
||||
PREFLIGHT.validate_bbox([5.110, 51.180, 5.1105, 51.1805], extent)
|
||||
with pytest.raises(RuntimeError, match="may not exceed"):
|
||||
PREFLIGHT.validate_bbox([5.05, 51.15, 5.25, 51.33], extent)
|
||||
|
||||
|
||||
def test_operator_preflight_is_read_only_packaged_and_readiness_checked() -> None:
|
||||
script = (SCRIPTS / "orthophoto_release_preflight.py").read_text(encoding="utf-8")
|
||||
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
|
||||
readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8")
|
||||
|
||||
assert "GetMap" not in script
|
||||
assert "GetCoverage" not in script
|
||||
assert "/datasets/upload" not in script
|
||||
assert "INSERT INTO" not in script
|
||||
assert "COPY scripts/orthophoto_release_preflight.py" in dockerfile
|
||||
assert "py_compile scripts/orthophoto_release_preflight.py" in readiness
|
||||
@@ -92,6 +92,7 @@ COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natu
|
||||
COPY scripts/provision_regional_bwk_natura2000.py /app/scripts/provision_regional_bwk_natura2000.py
|
||||
COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py
|
||||
COPY scripts/manage_alz_agriculture_release.py /app/scripts/manage_alz_agriculture_release.py
|
||||
COPY scripts/orthophoto_release_preflight.py /app/scripts/orthophoto_release_preflight.py
|
||||
COPY scripts/provision_buildings_addresses_register.py /app/scripts/provision_buildings_addresses_register.py
|
||||
COPY scripts/provision_regional_timeseries.py /app/scripts/provision_regional_timeseries.py
|
||||
COPY scripts/geographic_scopes.py /app/scripts/geographic_scopes.py
|
||||
|
||||
@@ -466,6 +466,16 @@ revalidates the current catalog and delegates to the existing Dataset upload
|
||||
contract. No ALZ release endpoint, background task or provider URL parameter
|
||||
is added; v1/v2 campaign snapshots remain non-importable.
|
||||
|
||||
Current-orthophoto release preflight also remains outside the HTTP request
|
||||
cycle in `scripts/orthophoto_release_preflight.py`. It composes the existing
|
||||
product-registry and source-catalog envelopes with allowlisted WMS
|
||||
`GetCapabilities`, WCS `DescribeCoverage` and bounded `Vliegdagcontour`
|
||||
`GetFeatureInfo` evidence for one 128-1,024 m EPSG:4326 selection. The result
|
||||
reports product variant, official `YYYY.NN` edition, exact raster-domain
|
||||
containment, sampled flight dates/years, local comparison state and
|
||||
`staging_permitted`. It performs no pixel request, upload, Job, Dataset write
|
||||
or legacy metadata rewrite. This operator script adds no public API contract.
|
||||
|
||||
The endpoint accepts no arbitrary URL, feature query, area or layer. It does
|
||||
not fetch vector features, raster pixels or models, create jobs/datasets, write
|
||||
to PostGIS or trigger an import. The normal `source-freshness` endpoint remains
|
||||
|
||||
@@ -9715,3 +9715,40 @@ Live validation:
|
||||
Next:
|
||||
- Evaluate ALZ as the next governed edition probe only after its official
|
||||
machine-readable release version and schema stability are verified.
|
||||
## Sprint 230 - Governed orthophoto release preflight (2026-07-17)
|
||||
|
||||
Implemented:
|
||||
- Added `scripts/orthophoto_release_preflight.py` as a read-only gate for one
|
||||
bounded current-orthophoto selection in the approved Kempen regional scope.
|
||||
- Bound the existing canonical product registry and source-catalog result to
|
||||
the exact official WMS 1.3.0 capabilities hash, ISO metadata identifier and
|
||||
`YYYY.NN` edition. Redirects, hosts, paths and response sizes fail closed.
|
||||
- Added metadata-only WCS `DescribeCoverage` validation for `Ortho`,
|
||||
EPSG:31370, the complete raster domain, 15 cm rectified grid, three bands and
|
||||
TIFF native format. Selection limits remain identical to acquisition:
|
||||
128-1,024 m per projected side.
|
||||
- Added a maximum 64-point deterministic `Vliegdagcontour` grid with exact
|
||||
GeoJSON layer/date/year validation and hashed sample evidence. WCS domain
|
||||
containment proves raster-domain coverage; point samples remain honestly
|
||||
labelled as flight-date evidence rather than polygon-union geometry.
|
||||
- Kept all pixel requests, filesystem staging, Jobs, Dataset/PostGIS writes,
|
||||
migrations and browser behavior out of this sprint. Local
|
||||
`most_recent_at_*` acquisition markers remain non-comparable and blocked.
|
||||
|
||||
Validation so far:
|
||||
- 42 focused preflight/runtime-packaging tests passed. Coverage includes
|
||||
release ordering, legacy provenance, hash drift, product variant, official
|
||||
host/path, queryable/GeoJSON capability, WCS identity/resolution, missing
|
||||
flight coverage, flight-year mismatch, selection bounds and no-write rules.
|
||||
- Target Python compilation and Ruff passed.
|
||||
- A read-only compatibility run against Tower for bbox
|
||||
`5.110,51.180,5.117,51.185` verified official edition `2025.04`, exact Ortho
|
||||
WCS domain containment and 20/20 flight-day samples from 2025. It performed
|
||||
zero pixel requests and correctly returned `staging_permitted=false` because
|
||||
local version `most_recent_at_2026-07-15` is not an official edition.
|
||||
|
||||
Boundary:
|
||||
- This sprint supplies preflight only. It deliberately does not promote or
|
||||
backfill existing orthophotos. A future stage/review/apply coordinator must
|
||||
revalidate and retain the exact preflight identity before creating a new
|
||||
immutable raster Dataset with official `YYYY.NN` source version.
|
||||
|
||||
@@ -291,6 +291,14 @@ annual Dataset, DatasetVersion and vector_features records with
|
||||
`source_version=<year>-definitive`. Earlier annual snapshots are retained and
|
||||
provisional v1/v2 publications cannot create rows.
|
||||
|
||||
The current-orthophoto release preflight likewise adds no lifecycle table or
|
||||
migration. It is read-only and creates neither Dataset nor Job. It compares the
|
||||
existing local `source_version` with the official `YYYY.NN` catalog edition
|
||||
and verifies WMS/WCS/flight-day evidence for one bounded selection. A later
|
||||
governed pixel apply must still create an immutable raster Dataset plus
|
||||
DatasetVersion through DatasetService and retain that exact edition/evidence;
|
||||
direct metadata backfill of legacy `most_recent_at_*` rows is prohibited.
|
||||
|
||||
## Geometry normalization
|
||||
|
||||
- User-drawn polygons arrive as EPSG:4326.
|
||||
|
||||
+16
-1
@@ -24,6 +24,21 @@ aanvraag. GeoIntel verzint geen historische pixelopnamedatum. Bronnen:
|
||||
- https://www.vlaanderen.be/datavindplaats/catalogus/orthofotomozaiek-middenschalig-winteropnamen-kleur-meest-recent-vlaanderen
|
||||
- https://www.vlaanderen.be/digitaal-vlaanderen/onze-diensten-en-platformen/luchtopnamen/gebruik-orthofotomozaieken
|
||||
|
||||
Voor een toekomstige rolling-releasebeslissing gebruikt
|
||||
`scripts/orthophoto_release_preflight.py` uitsluitend metadata. Het bindt de
|
||||
lokale `most_recent`-productvariant aan de officiële ISO-editie en exacte WMS-
|
||||
capabilitieshash, controleert het EPSG:31370/15 cm/driebanden-rasterdomein via
|
||||
WCS `DescribeCoverage` en bemonstert de querybare `Vliegdagcontour` op een
|
||||
deterministisch raster met maximaal 128 m afstand. Er worden geen `GetMap`-,
|
||||
`GetCoverage`- of uploadrequests uitgevoerd.
|
||||
|
||||
De WCS-domeincontrole bewijst dat de volledige begrensde selectie binnen het
|
||||
officiële rasterdomein valt. De WMS publiceert geen vliegdagpolygonen als WFS;
|
||||
het vluchtjaar blijft daarom expliciet puntbewijs en geen verzonnen polygon-
|
||||
union. Een selectie met ontbrekende contourpunten, meerdere/afwijkende
|
||||
vluchtjaren, gewijzigde service-identiteit of een niet-vergelijkbare lokale
|
||||
`most_recent_at_*` marker is niet stagebaar.
|
||||
|
||||
Dit document verzamelt concrete databronnen voor GeoIntel Kempen.
|
||||
|
||||
## Cross-domain official area profile
|
||||
@@ -174,7 +189,7 @@ gebeurd.
|
||||
| GRB gebouwen/wegen/water/percelen | operationele, expliciete plan-stage-apply refresh met onveranderlijke snapshots | alleen een nieuw officieel gedateerd cataloguseditie na operatorbevestiging ophalen |
|
||||
| Statbel bevolking | jaarlijkse, expliciete edities in één tijdreeks; officiële DCAT-releaseprobe | een nieuwe publicatie alleen na schema-, sectorgeometrie- en totalencontrole toevoegen |
|
||||
| ALZ landbouwgebruikspercelen | definitieve jaarlijkse edities 2008-2025; expliciete publicatieprobe en plan-stage-review-apply promotie; metricvergelijking zonder objectlineage | alleen een nieuwere definitieve v3-editie na gestagede schema-/codelijst-/scopecontrole en benoemde review toevoegen |
|
||||
| orthofoto | vaste lokale opname per expliciete analysezone; catalogusprobe is alleen een signaal | vluchtjaar, productvariant en dekking vergelijken voordat nieuwe pixels worden opgehaald |
|
||||
| orthofoto | vaste lokale opname per expliciete analysezone; read-only releasepreflight voor variant, officiële editie, exact WCS-domein en begrensd vluchtjaarbewijs | eerst officiële lokale editieprovenance vastleggen; daarna pas een afzonderlijke menselijke pixel-stage/apply-flow bouwen |
|
||||
| landgebruik, thematische rasters, DHMV en VMM-scenario's | vaste product-/scenario-edities, geen rolling snapshot | alleen een nieuwe gedocumenteerde producteditie als afzonderlijke Dataset verwerven |
|
||||
| bodemkaart en historische kaarten | historische referentie-editie | niet als verouderde actuele bron labelen; alleen vervangen bij een officiële inhoudelijke heruitgave |
|
||||
| BWK/Natura 2000 en gebouwen-/adressenregister | expliciete actuele snapshot met eigen methodologische betekenis | eerst een stabiele officiële editieprobe en bron-specifieke reconciliatiecontrole toevoegen |
|
||||
|
||||
@@ -274,6 +274,13 @@ layer, observation label, `observed_at`, optional `valid_from`/`valid_to`,
|
||||
temporal granularity, request/spatial hash, attribution and a limitation that
|
||||
states whether the product is annual, multi-year or merely most recent.
|
||||
|
||||
A future rolling `most_recent` import must also retain the exact official
|
||||
`YYYY.NN` edition and the preflight identities for WMS capabilities, WCS
|
||||
coverage description, selected EPSG:31370 domain and sampled flight year. A
|
||||
legacy `most_recent_at_<date>` value is acquisition timing, not an official
|
||||
edition, and cannot be promoted or compared as if it were one. The read-only
|
||||
preflight creates no Dataset and does not retroactively rewrite that evidence.
|
||||
|
||||
### Hydrological station observations
|
||||
|
||||
Waterinfo observations are persisted as EPSG:4326 Point features, one station
|
||||
|
||||
@@ -147,6 +147,14 @@ product/layer, request/spatial hash, temporal validity and limitations are held
|
||||
in source/provenance metadata. Browser PNG rendering is derived on request and
|
||||
does not replace the stored GeoTIFF.
|
||||
|
||||
The orthophoto release preflight writes no source file, raster or database row.
|
||||
Its JSON stdout may be retained by an operator as review evidence, but it is
|
||||
not itself staging authorization. The report binds official WMS and WCS XML
|
||||
hashes, the exact selected domain and hashed flight-day sample evidence. A
|
||||
future pixel stage must persist and revalidate that identity separately before
|
||||
DatasetService is called; existing `most_recent_at_*` raster metadata is not
|
||||
silently rewritten.
|
||||
|
||||
DHMV II DTM/DSM outputs are also normal raster Dataset files. The provider WCS
|
||||
returns multipart coverage data; GeoIntel retains response and extracted
|
||||
coverage SHA256 values in provenance, then stores one normalized, compressed,
|
||||
|
||||
+5
-2
@@ -690,5 +690,8 @@ This file now starts with the current implementation status. Older preparation/b
|
||||
- [x] Document refresh readiness across the complete official-source portfolio.
|
||||
- [x] Verify and add the governed ALZ catalog probe against the official
|
||||
campaign-snapshot and definitive-archive publication contract.
|
||||
- [ ] Keep orthophoto refresh manual until product variant, flight year and
|
||||
complete selected-area coverage can be compared deterministically.
|
||||
- [x] Add a read-only orthophoto preflight for product variant, official
|
||||
edition, exact WCS selected-area domain and deterministic flight-year points.
|
||||
- [ ] Keep orthophoto pixel refresh manual and blocked until a separate
|
||||
plan-stage-review-apply flow can retain the passed preflight identity and
|
||||
create a new immutable Dataset with official `YYYY.NN` source version.
|
||||
|
||||
@@ -0,0 +1,694 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Read-only release preflight for the governed current orthophoto product.
|
||||
|
||||
The preflight does not request or stage raster pixels. It binds the canonical
|
||||
GeoIntel catalog result to the current product registry and the exact official
|
||||
WMS capabilities, then probes the queryable flight-day layer on a bounded,
|
||||
deterministic grid. Local rolling markers remain non-comparable and blocked.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from hashlib import sha256
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from typing import Any, Callable
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.parse import parse_qs, urlencode, urlparse
|
||||
from urllib.request import Request, urlopen
|
||||
from xml.etree import ElementTree
|
||||
|
||||
from pyproj import Transformer
|
||||
|
||||
from geographic_scopes import GEOGRAPHIC_SCOPES
|
||||
|
||||
|
||||
DEFAULT_API_URL = "http://127.0.0.1:8000/api/v1"
|
||||
DEFAULT_SCOPE = "kempen-transport-region"
|
||||
SOURCE_NAME = "digitaal_vlaanderen_orthophoto"
|
||||
PRODUCT_KEY = "most_recent"
|
||||
WMS_HOST = "geo.api.vlaanderen.be"
|
||||
WMS_PATH = "/OMWRGBMRVL/wms"
|
||||
WCS_PATH = "/OMWRGBMRVL/wcs"
|
||||
CATALOG_URL = (
|
||||
"https://www.vlaanderen.be/datavindplaats/catalogus/"
|
||||
"orthofotomozaiek-middenschalig-winteropnamen-kleur-meest-recent-vlaanderen"
|
||||
)
|
||||
METADATA_IDENTIFIER = "f5304d6d-0dd4-43fd-a726-427af31e8d61"
|
||||
EDITION_PATTERN = re.compile(r"^(20[0-9]{2})\.([0-9]{2})$")
|
||||
SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$")
|
||||
FLIGHT_YEAR_PATTERN = re.compile(r"(?:^|[^0-9])(20[0-9]{2})(?:$|[^0-9])")
|
||||
MAX_CAPABILITIES_BYTES = 2 * 1024 * 1024
|
||||
MAX_COVERAGE_DESCRIPTION_BYTES = 512 * 1024
|
||||
MAX_FEATURE_INFO_BYTES = 128 * 1024
|
||||
MAX_SAMPLE_SPACING_M = 128.0
|
||||
MAX_SAMPLE_COUNT = 64
|
||||
MIN_SIDE_M = 128.0
|
||||
MAX_SIDE_M = 1024.0
|
||||
FEATURE_INFO_SIZE = 1000
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Read-only current-orthophoto release preflight. No raster pixels are fetched, staged or imported."
|
||||
)
|
||||
)
|
||||
parser.add_argument("--project-id", required=True, help="GeoIntel project UUID for the governed scope")
|
||||
parser.add_argument("--scope", choices=(DEFAULT_SCOPE,), default=DEFAULT_SCOPE)
|
||||
parser.add_argument("--api-url", default=os.environ.get("GEOINTEL_API_URL", DEFAULT_API_URL))
|
||||
parser.add_argument(
|
||||
"--bbox",
|
||||
nargs=4,
|
||||
type=float,
|
||||
metavar=("MIN_LON", "MIN_LAT", "MAX_LON", "MAX_LAT"),
|
||||
required=True,
|
||||
help="Exact EPSG:4326 selection; each projected side must be between 128 and 1024 metres",
|
||||
)
|
||||
parser.add_argument("--refresh-catalog", action="store_true", help="Bypass the short catalog cache")
|
||||
parser.add_argument("--api-timeout", type=int, default=180)
|
||||
parser.add_argument("--wms-timeout", type=int, default=30)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def api_data(api_url: str, path: str, timeout: int) -> dict[str, Any]:
|
||||
endpoint = f"{api_url.rstrip('/')}/{path.lstrip('/')}"
|
||||
request = Request(
|
||||
endpoint,
|
||||
headers={"Accept": "application/json", "User-Agent": "GeoIntel-orthophoto-preflight/1.0"},
|
||||
)
|
||||
try:
|
||||
with urlopen(request, timeout=timeout) as response:
|
||||
payload = json.load(response)
|
||||
except HTTPError as exc:
|
||||
body = exc.read().decode("utf-8", errors="replace")
|
||||
raise RuntimeError(f"GeoIntel API returned HTTP {exc.code}: {body[-1000:]}") from exc
|
||||
except URLError as exc:
|
||||
raise RuntimeError(f"GeoIntel API is unreachable: {exc.reason}") from exc
|
||||
if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict):
|
||||
raise RuntimeError("GeoIntel API response is not a canonical data envelope")
|
||||
return payload["data"]
|
||||
|
||||
|
||||
def validate_project_scope(args: argparse.Namespace, loader: Callable[[str, str, int], dict[str, Any]]) -> None:
|
||||
project = loader(args.api_url, f"projects/{args.project_id}", args.api_timeout)
|
||||
expected_name = GEOGRAPHIC_SCOPES[args.scope].project_name
|
||||
if project.get("name") != expected_name:
|
||||
raise RuntimeError(
|
||||
f"Project {args.project_id} is '{project.get('name')}', but scope {args.scope} requires '{expected_name}'"
|
||||
)
|
||||
|
||||
|
||||
def _validate_service_url(url: str, *, path: str, request_name: str | None = None) -> str:
|
||||
parsed = urlparse(url)
|
||||
if (
|
||||
parsed.scheme.lower() != "https"
|
||||
or parsed.hostname != WMS_HOST
|
||||
or parsed.port not in (None, 443)
|
||||
or parsed.username
|
||||
or parsed.password
|
||||
or parsed.fragment
|
||||
or parsed.path.rstrip("/").lower() != path.lower()
|
||||
):
|
||||
raise RuntimeError("Orthophoto service URL is outside the official allowlist")
|
||||
if request_name is not None:
|
||||
requests = parse_qs(parsed.query).get("REQUEST") or parse_qs(parsed.query).get("request") or []
|
||||
if len(requests) != 1 or requests[0].lower() != request_name.lower():
|
||||
raise RuntimeError(f"Orthophoto service URL is not an exact {request_name} request")
|
||||
return f"https://{WMS_HOST}{path}"
|
||||
|
||||
|
||||
def _validate_wms_url(url: str, *, request_name: str | None = None) -> str:
|
||||
return _validate_service_url(url, path=WMS_PATH, request_name=request_name)
|
||||
|
||||
|
||||
def _validate_wcs_url(url: str, *, request_name: str | None = None) -> str:
|
||||
return _validate_service_url(url, path=WCS_PATH, request_name=request_name)
|
||||
|
||||
|
||||
def _bounded_get(
|
||||
url: str,
|
||||
*,
|
||||
timeout: int,
|
||||
max_bytes: int,
|
||||
accept: str,
|
||||
opener: Callable[..., Any] | None = None,
|
||||
validator: Callable[[str], str] = _validate_wms_url,
|
||||
) -> tuple[bytes, str, str]:
|
||||
request = Request(url, headers={"Accept": accept, "User-Agent": "GeoIntel-orthophoto-preflight/1.0"})
|
||||
fetch = opener or urlopen
|
||||
try:
|
||||
response = fetch(request, timeout=timeout)
|
||||
with response:
|
||||
final_url = response.geturl()
|
||||
validator(final_url)
|
||||
content_type = str(response.headers.get("Content-Type") or "").split(";", 1)[0].strip().lower()
|
||||
declared = response.headers.get("Content-Length")
|
||||
try:
|
||||
if declared and int(declared) > max_bytes:
|
||||
raise RuntimeError("Official WMS response exceeds the configured preflight limit")
|
||||
except ValueError as exc:
|
||||
raise RuntimeError("Official WMS response has an invalid Content-Length") from exc
|
||||
body = response.read(max_bytes + 1)
|
||||
except HTTPError as exc:
|
||||
raise RuntimeError(f"Official orthophoto WMS returned HTTP {exc.code}") from exc
|
||||
except URLError as exc:
|
||||
raise RuntimeError(f"Official orthophoto WMS is unreachable: {exc.reason}") from exc
|
||||
if len(body) > max_bytes:
|
||||
raise RuntimeError("Official WMS response exceeds the configured preflight limit")
|
||||
return body, content_type, final_url
|
||||
|
||||
|
||||
def _local_name(tag: str) -> str:
|
||||
return tag.rsplit("}", 1)[-1]
|
||||
|
||||
|
||||
def _children(element: ElementTree.Element, name: str) -> list[ElementTree.Element]:
|
||||
return [child for child in element if _local_name(child.tag) == name]
|
||||
|
||||
|
||||
def _child_text(element: ElementTree.Element, name: str) -> str | None:
|
||||
for child in element:
|
||||
if _local_name(child.tag) == name and child.text:
|
||||
return child.text.strip()
|
||||
return None
|
||||
|
||||
|
||||
def parse_capabilities(body: bytes, expected_sha256: str) -> dict[str, Any]:
|
||||
if not SHA256_PATTERN.fullmatch(expected_sha256):
|
||||
raise RuntimeError("Canonical catalog response has no valid capabilities SHA-256")
|
||||
actual_sha256 = sha256(body).hexdigest()
|
||||
if actual_sha256 != expected_sha256:
|
||||
raise RuntimeError("Official WMS capabilities changed after the canonical catalog check")
|
||||
upper = body[:4096].upper()
|
||||
if b"<!DOCTYPE" in upper or b"<!ENTITY" in upper:
|
||||
raise RuntimeError("Official WMS capabilities contains a prohibited XML declaration")
|
||||
try:
|
||||
root = ElementTree.fromstring(body)
|
||||
except ElementTree.ParseError as exc:
|
||||
raise RuntimeError("Official WMS capabilities is invalid XML") from exc
|
||||
if _local_name(root.tag) != "WMS_Capabilities" or root.attrib.get("version") != "1.3.0":
|
||||
raise RuntimeError("Official orthophoto service is not WMS 1.3.0")
|
||||
|
||||
capability = next((node for node in root if _local_name(node.tag) == "Capability"), None)
|
||||
root_layer = (
|
||||
next((node for node in capability if _local_name(node.tag) == "Layer"), None)
|
||||
if capability is not None
|
||||
else None
|
||||
)
|
||||
if capability is None or root_layer is None:
|
||||
raise RuntimeError("Official WMS capabilities has no root layer")
|
||||
layers: dict[str, ElementTree.Element] = {}
|
||||
for node in root.iter():
|
||||
if _local_name(node.tag) != "Layer":
|
||||
continue
|
||||
name = _child_text(node, "Name")
|
||||
if name:
|
||||
layers[name] = node
|
||||
if set(("Ortho", "Vliegdagcontour")) - set(layers):
|
||||
raise RuntimeError("Official WMS capabilities does not contain both governed orthophoto layers")
|
||||
if layers["Vliegdagcontour"].attrib.get("queryable") != "1":
|
||||
raise RuntimeError("Official Vliegdagcontour layer is not queryable")
|
||||
|
||||
formats: set[str] = set()
|
||||
for request_node in capability:
|
||||
if _local_name(request_node.tag) != "Request":
|
||||
continue
|
||||
for operation in request_node:
|
||||
if _local_name(operation.tag) == "GetFeatureInfo":
|
||||
formats.update(
|
||||
(item.text or "").strip().lower()
|
||||
for item in operation
|
||||
if _local_name(item.tag) == "Format" and item.text
|
||||
)
|
||||
if "application/geo+json" not in formats:
|
||||
raise RuntimeError("Official WMS does not advertise GeoJSON feature info")
|
||||
|
||||
bbox = None
|
||||
for node in _children(root_layer, "BoundingBox"):
|
||||
if node.attrib.get("CRS") == "EPSG:31370":
|
||||
try:
|
||||
bbox = [float(node.attrib[key]) for key in ("minx", "miny", "maxx", "maxy")]
|
||||
except (KeyError, ValueError) as exc:
|
||||
raise RuntimeError("Official WMS EPSG:31370 extent is invalid") from exc
|
||||
break
|
||||
if bbox is None or bbox[0] >= bbox[2] or bbox[1] >= bbox[3]:
|
||||
raise RuntimeError("Official WMS does not advertise a valid EPSG:31370 extent")
|
||||
|
||||
metadata_identifiers: set[str] = set()
|
||||
for layer_name in ("Ortho", "Vliegdagcontour"):
|
||||
for node in layers[layer_name].iter():
|
||||
if _local_name(node.tag) != "OnlineResource":
|
||||
continue
|
||||
href = next((value for key, value in node.attrib.items() if _local_name(key) == "href"), "")
|
||||
if METADATA_IDENTIFIER in href:
|
||||
metadata_identifiers.add(METADATA_IDENTIFIER)
|
||||
if metadata_identifiers != {METADATA_IDENTIFIER}:
|
||||
raise RuntimeError("Governed orthophoto layers do not share the expected ISO metadata identity")
|
||||
return {
|
||||
"service_version": "1.3.0",
|
||||
"capabilities_sha256": actual_sha256,
|
||||
"layers": ["Ortho", "Vliegdagcontour"],
|
||||
"vliegdagcontour_queryable": True,
|
||||
"feature_info_format": "application/geo+json",
|
||||
"extent_epsg31370": bbox,
|
||||
"metadata_identifier": METADATA_IDENTIFIER,
|
||||
}
|
||||
|
||||
|
||||
def parse_coverage_description(body: bytes) -> dict[str, Any]:
|
||||
upper = body[:4096].upper()
|
||||
if b"<!DOCTYPE" in upper or b"<!ENTITY" in upper:
|
||||
raise RuntimeError("Official WCS coverage description contains a prohibited XML declaration")
|
||||
try:
|
||||
root = ElementTree.fromstring(body)
|
||||
except ElementTree.ParseError as exc:
|
||||
raise RuntimeError("Official WCS coverage description is invalid XML") from exc
|
||||
coverage_ids = [
|
||||
(node.text or "").strip()
|
||||
for node in root.iter()
|
||||
if _local_name(node.tag) == "CoverageId" and node.text
|
||||
]
|
||||
if coverage_ids != ["Ortho"]:
|
||||
raise RuntimeError("Official WCS coverage description is not the governed Ortho coverage")
|
||||
envelope = next((node for node in root.iter() if _local_name(node.tag) == "Envelope"), None)
|
||||
if envelope is None or not str(envelope.attrib.get("srsName") or "").endswith("/31370"):
|
||||
raise RuntimeError("Official WCS Ortho domain is not EPSG:31370")
|
||||
try:
|
||||
lower = [float(value) for value in (_child_text(envelope, "lowerCorner") or "").split()]
|
||||
upper_corner = [float(value) for value in (_child_text(envelope, "upperCorner") or "").split()]
|
||||
except ValueError as exc:
|
||||
raise RuntimeError("Official WCS Ortho domain corners are invalid") from exc
|
||||
if len(lower) != 2 or len(upper_corner) != 2 or lower[0] >= upper_corner[0] or lower[1] >= upper_corner[1]:
|
||||
raise RuntimeError("Official WCS Ortho domain corners are invalid")
|
||||
grid = next((node for node in root.iter() if _local_name(node.tag) == "RectifiedGrid"), None)
|
||||
if grid is None or grid.attrib.get("dimension") != "2":
|
||||
raise RuntimeError("Official WCS Ortho domain is not a two-dimensional rectified grid")
|
||||
offsets: list[list[float]] = []
|
||||
for node in grid.iter():
|
||||
if _local_name(node.tag) == "offsetVector" and node.text:
|
||||
try:
|
||||
offsets.append([float(value) for value in node.text.split()])
|
||||
except ValueError as exc:
|
||||
raise RuntimeError("Official WCS Ortho grid resolution is invalid") from exc
|
||||
if (
|
||||
len(offsets) != 2
|
||||
or any(len(vector) != 2 for vector in offsets)
|
||||
or not math.isclose(offsets[0][0], 0.15, abs_tol=1e-9)
|
||||
or not math.isclose(offsets[0][1], 0.0, abs_tol=1e-9)
|
||||
or not math.isclose(offsets[1][0], 0.0, abs_tol=1e-9)
|
||||
or not math.isclose(offsets[1][1], -0.15, abs_tol=1e-9)
|
||||
):
|
||||
raise RuntimeError("Official WCS Ortho grid is no longer the governed 15 cm product")
|
||||
field_count = sum(_local_name(node.tag) == "field" for node in root.iter())
|
||||
subtype = next(
|
||||
((node.text or "").strip() for node in root.iter() if _local_name(node.tag) == "CoverageSubtype"),
|
||||
"",
|
||||
)
|
||||
native_format = next(
|
||||
((node.text or "").strip() for node in root.iter() if _local_name(node.tag) == "nativeFormat"),
|
||||
"",
|
||||
)
|
||||
if field_count != 3 or subtype != "RectifiedGridCoverage" or native_format != "image/tiff":
|
||||
raise RuntimeError("Official WCS Ortho range or native format changed")
|
||||
return {
|
||||
"coverage_id": "Ortho",
|
||||
"crs": "EPSG:31370",
|
||||
"extent_epsg31370": [lower[0], lower[1], upper_corner[0], upper_corner[1]],
|
||||
"native_resolution_m": 0.15,
|
||||
"band_count": 3,
|
||||
"native_format": "image/tiff",
|
||||
"coverage_description_sha256": sha256(body).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def validate_bbox(values: list[float], advertised_extent: list[float]) -> dict[str, Any]:
|
||||
if len(values) != 4 or not all(math.isfinite(value) for value in values):
|
||||
raise RuntimeError("Selection bbox must contain four finite EPSG:4326 values")
|
||||
min_x, min_y, max_x, max_y = values
|
||||
if min_x >= max_x or min_y >= max_y or not (-180 <= min_x < max_x <= 180) or not (-90 <= min_y < max_y <= 90):
|
||||
raise RuntimeError("Selection bbox is not a valid EPSG:4326 rectangle")
|
||||
transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
||||
lambert = [float(value) for value in transformer.transform_bounds(min_x, min_y, max_x, max_y, densify_pts=21)]
|
||||
width_m = lambert[2] - lambert[0]
|
||||
height_m = lambert[3] - lambert[1]
|
||||
if width_m < MIN_SIDE_M or height_m < MIN_SIDE_M:
|
||||
raise RuntimeError(f"Orthophoto preflight selection must be at least {MIN_SIDE_M:.0f} by {MIN_SIDE_M:.0f} metres")
|
||||
if width_m > MAX_SIDE_M or height_m > MAX_SIDE_M:
|
||||
raise RuntimeError(f"Orthophoto preflight selection may not exceed {MAX_SIDE_M:.0f} by {MAX_SIDE_M:.0f} metres")
|
||||
if not (
|
||||
advertised_extent[0] <= lambert[0]
|
||||
and advertised_extent[1] <= lambert[1]
|
||||
and lambert[2] <= advertised_extent[2]
|
||||
and lambert[3] <= advertised_extent[3]
|
||||
):
|
||||
raise RuntimeError("Selection is outside the official WMS advertised EPSG:31370 extent")
|
||||
return {
|
||||
"bbox_epsg4326": [min_x, min_y, max_x, max_y],
|
||||
"bbox_epsg31370": lambert,
|
||||
"width_m": width_m,
|
||||
"height_m": height_m,
|
||||
}
|
||||
|
||||
|
||||
def validate_product(items: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
matches = [item for item in items if item.get("key") == PRODUCT_KEY]
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError("Orthophoto product registry does not contain exactly one most_recent product")
|
||||
product = matches[0]
|
||||
expected = {
|
||||
"temporal_granularity": "snapshot",
|
||||
"native_resolution_m": 0.15,
|
||||
"supports_detection": True,
|
||||
"color_mode": "rgb",
|
||||
"catalog_url": CATALOG_URL,
|
||||
}
|
||||
if any(product.get(key) != value for key, value in expected.items()):
|
||||
raise RuntimeError("Current orthophoto product variant no longer matches the governed registry contract")
|
||||
return {"key": PRODUCT_KEY, **expected, "display_name": product.get("display_name")}
|
||||
|
||||
|
||||
def catalog_decision(item: dict[str, Any]) -> dict[str, Any]:
|
||||
remote_version = str(item.get("remote_version") or "")
|
||||
remote_match = EDITION_PATTERN.fullmatch(remote_version)
|
||||
if (
|
||||
item.get("source_name") != SOURCE_NAME
|
||||
or item.get("status") != "available"
|
||||
or item.get("reachable") is not True
|
||||
or set(item.get("matched_layers") or []) != {"Ortho", "Vliegdagcontour"}
|
||||
or item.get("missing_layers") not in (None, [])
|
||||
or item.get("metadata_identifier") != METADATA_IDENTIFIER
|
||||
or not SHA256_PATTERN.fullmatch(str(item.get("capabilities_sha256") or ""))
|
||||
or not remote_match
|
||||
or remote_version not in str(item.get("remote_title") or "")
|
||||
):
|
||||
raise RuntimeError("Canonical catalog report does not contain one complete governed orthophoto release")
|
||||
local_version = str(item.get("local_source_version") or "")
|
||||
local_match = EDITION_PATTERN.fullmatch(local_version)
|
||||
if not local_version:
|
||||
status = "not_loaded"
|
||||
expected_comparison = "no_local_data"
|
||||
elif not local_match:
|
||||
status = "blocked_local_version"
|
||||
expected_comparison = "not_comparable"
|
||||
else:
|
||||
local_order = tuple(int(value) for value in local_match.groups())
|
||||
remote_order = tuple(int(value) for value in remote_match.groups())
|
||||
if local_order == remote_order:
|
||||
status = "current"
|
||||
expected_comparison = "same"
|
||||
elif local_order < remote_order:
|
||||
status = "update_available"
|
||||
expected_comparison = "different"
|
||||
else:
|
||||
status = "blocked_remote_older"
|
||||
expected_comparison = "different"
|
||||
if item.get("comparison_status") != expected_comparison:
|
||||
raise RuntimeError("Canonical catalog local/remote comparison is internally inconsistent")
|
||||
return {
|
||||
"status": status,
|
||||
"remote_edition": remote_version,
|
||||
"remote_year": int(remote_match.group(1)),
|
||||
"local_source_version": local_version or None,
|
||||
"comparison_status": expected_comparison,
|
||||
"metadata_identifier": METADATA_IDENTIFIER,
|
||||
"metadata_url": item.get("metadata_url"),
|
||||
"remote_title": item.get("remote_title"),
|
||||
"remote_modified_at": item.get("remote_modified_at"),
|
||||
"remote_published_at": item.get("remote_published_at"),
|
||||
"catalog_checked_at": item.get("checked_at"),
|
||||
"capabilities_url": item.get("endpoint_url"),
|
||||
"capabilities_sha256": item.get("capabilities_sha256"),
|
||||
}
|
||||
|
||||
|
||||
def _sample_grid(selection: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
count_x = max(1, math.ceil(selection["width_m"] / MAX_SAMPLE_SPACING_M))
|
||||
count_y = max(1, math.ceil(selection["height_m"] / MAX_SAMPLE_SPACING_M))
|
||||
if count_x * count_y > MAX_SAMPLE_COUNT:
|
||||
raise RuntimeError("Deterministic flight-day grid exceeds the fixed preflight limit")
|
||||
min_x, min_y, max_x, max_y = selection["bbox_epsg31370"]
|
||||
samples: list[dict[str, Any]] = []
|
||||
for row in range(count_y):
|
||||
fraction_y = (row + 0.5) / count_y
|
||||
for column in range(count_x):
|
||||
fraction_x = (column + 0.5) / count_x
|
||||
samples.append(
|
||||
{
|
||||
"column": column,
|
||||
"row": row,
|
||||
"i": min(FEATURE_INFO_SIZE - 1, int(fraction_x * FEATURE_INFO_SIZE)),
|
||||
"j": min(FEATURE_INFO_SIZE - 1, int((1.0 - fraction_y) * FEATURE_INFO_SIZE)),
|
||||
"x": min_x + fraction_x * (max_x - min_x),
|
||||
"y": min_y + fraction_y * (max_y - min_y),
|
||||
}
|
||||
)
|
||||
return samples
|
||||
|
||||
|
||||
def _flight_sample(
|
||||
base_url: str,
|
||||
selection: dict[str, Any],
|
||||
sample: dict[str, Any],
|
||||
*,
|
||||
timeout: int,
|
||||
opener: Callable[..., Any] | None,
|
||||
) -> dict[str, Any]:
|
||||
params = {
|
||||
"SERVICE": "WMS",
|
||||
"VERSION": "1.3.0",
|
||||
"REQUEST": "GetFeatureInfo",
|
||||
"LAYERS": "Vliegdagcontour",
|
||||
"QUERY_LAYERS": "Vliegdagcontour",
|
||||
"STYLES": "",
|
||||
"CRS": "EPSG:31370",
|
||||
"BBOX": ",".join(f"{value:.3f}" for value in selection["bbox_epsg31370"]),
|
||||
"WIDTH": str(FEATURE_INFO_SIZE),
|
||||
"HEIGHT": str(FEATURE_INFO_SIZE),
|
||||
"I": str(sample["i"]),
|
||||
"J": str(sample["j"]),
|
||||
"INFO_FORMAT": "application/geo+json",
|
||||
"FEATURE_COUNT": "10",
|
||||
"FORMAT": "image/png",
|
||||
}
|
||||
body, content_type, _ = _bounded_get(
|
||||
f"{base_url}?{urlencode(params)}",
|
||||
timeout=timeout,
|
||||
max_bytes=MAX_FEATURE_INFO_BYTES,
|
||||
accept="application/geo+json",
|
||||
opener=opener,
|
||||
)
|
||||
if content_type not in {"application/geo+json", "application/json"}:
|
||||
raise RuntimeError("Vliegdagcontour feature-info response is not GeoJSON")
|
||||
try:
|
||||
payload = json.loads(body)
|
||||
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise RuntimeError("Vliegdagcontour feature-info response is invalid JSON") from exc
|
||||
features = payload.get("features") if isinstance(payload, dict) else None
|
||||
if not isinstance(payload, dict) or payload.get("type") != "FeatureCollection" or not isinstance(features, list) or not features:
|
||||
raise RuntimeError("Vliegdagcontour has no coverage at one deterministic selection sample")
|
||||
dates: set[str] = set()
|
||||
feature_ids: set[str] = set()
|
||||
years: set[int] = set()
|
||||
for feature in features:
|
||||
if not isinstance(feature, dict) or feature.get("layerName") != "Vliegdagcontour":
|
||||
raise RuntimeError("Vliegdagcontour feature-info returned an unexpected layer")
|
||||
properties = feature.get("properties")
|
||||
if not isinstance(properties, dict):
|
||||
raise RuntimeError("Vliegdagcontour feature-info has no properties")
|
||||
date_value = str(properties.get("OpnDatum") or "").strip()
|
||||
match = FLIGHT_YEAR_PATTERN.search(date_value)
|
||||
if not match:
|
||||
raise RuntimeError("Vliegdagcontour feature-info has no parseable acquisition year")
|
||||
dates.add(date_value)
|
||||
years.add(int(match.group(1)))
|
||||
if properties.get("FID") is not None:
|
||||
feature_ids.add(str(properties["FID"]))
|
||||
return {
|
||||
"column": sample["column"],
|
||||
"row": sample["row"],
|
||||
"x": round(sample["x"], 3),
|
||||
"y": round(sample["y"], 3),
|
||||
"dates": sorted(dates),
|
||||
"years": sorted(years),
|
||||
"feature_ids": sorted(feature_ids),
|
||||
"response_sha256": sha256(body).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def verify_flight_coverage(
|
||||
base_url: str,
|
||||
selection: dict[str, Any],
|
||||
*,
|
||||
timeout: int,
|
||||
opener: Callable[..., Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
samples = _sample_grid(selection)
|
||||
worker_count = min(8, len(samples))
|
||||
with ThreadPoolExecutor(max_workers=worker_count) as executor:
|
||||
evidence = list(
|
||||
executor.map(
|
||||
lambda sample: _flight_sample(
|
||||
base_url,
|
||||
selection,
|
||||
sample,
|
||||
timeout=timeout,
|
||||
opener=opener,
|
||||
),
|
||||
samples,
|
||||
)
|
||||
)
|
||||
years = sorted({year for item in evidence for year in item["years"]})
|
||||
dates = sorted({date for item in evidence for date in item["dates"]})
|
||||
feature_ids = sorted({feature_id for item in evidence for feature_id in item["feature_ids"]})
|
||||
canonical = json.dumps(evidence, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
|
||||
count_x = len({item["column"] for item in evidence})
|
||||
count_y = len({item["row"] for item in evidence})
|
||||
return {
|
||||
"status": "passed",
|
||||
"mode": "official_queryable_flight_day_grid",
|
||||
"sample_count": len(evidence),
|
||||
"grid_columns": count_x,
|
||||
"grid_rows": count_y,
|
||||
"maximum_sample_spacing_m": MAX_SAMPLE_SPACING_M,
|
||||
"covered_sample_count": len(evidence),
|
||||
"sample_coverage_ratio": 1.0,
|
||||
"flight_dates": dates,
|
||||
"flight_years": years,
|
||||
"feature_ids": feature_ids,
|
||||
"sample_evidence_sha256": sha256(canonical).hexdigest(),
|
||||
"claim_boundary": (
|
||||
"Every deterministic grid sample is covered by the official queryable flight-day layer. "
|
||||
"The WMS exposes no vector geometry, so this is bounded point evidence and not a polygon-union proof."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def run_preflight(
|
||||
args: argparse.Namespace,
|
||||
*,
|
||||
loader: Callable[[str, str, int], dict[str, Any]] = api_data,
|
||||
opener: Callable[..., Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
validate_project_scope(args, loader)
|
||||
products = loader(
|
||||
args.api_url,
|
||||
f"projects/{args.project_id}/datasets/orthophoto/products",
|
||||
args.api_timeout,
|
||||
)
|
||||
product = validate_product(products.get("items") or [])
|
||||
refresh = "true" if args.refresh_catalog else "false"
|
||||
report = loader(
|
||||
args.api_url,
|
||||
f"projects/{args.project_id}/datasets/source-catalog-probes?refresh={refresh}",
|
||||
args.api_timeout,
|
||||
)
|
||||
matches = [item for item in report.get("items") or [] if item.get("source_name") == SOURCE_NAME]
|
||||
if len(matches) != 1:
|
||||
raise RuntimeError("Source catalog report did not contain exactly one orthophoto contract")
|
||||
release = catalog_decision(matches[0])
|
||||
capabilities_url = str(release["capabilities_url"] or "")
|
||||
base_url = _validate_wms_url(capabilities_url, request_name="GetCapabilities")
|
||||
capabilities_body, content_type, _ = _bounded_get(
|
||||
capabilities_url,
|
||||
timeout=args.wms_timeout,
|
||||
max_bytes=MAX_CAPABILITIES_BYTES,
|
||||
accept="application/xml,text/xml",
|
||||
opener=opener,
|
||||
)
|
||||
if content_type and "xml" not in content_type and "text" not in content_type:
|
||||
raise RuntimeError("Official WMS capabilities response is not XML")
|
||||
capabilities = parse_capabilities(capabilities_body, release["capabilities_sha256"])
|
||||
coverage_description_url = (
|
||||
f"https://{WMS_HOST}{WCS_PATH}?"
|
||||
+ urlencode(
|
||||
{
|
||||
"SERVICE": "WCS",
|
||||
"VERSION": "2.0.1",
|
||||
"REQUEST": "DescribeCoverage",
|
||||
"COVERAGEID": "Ortho",
|
||||
}
|
||||
)
|
||||
)
|
||||
_validate_wcs_url(coverage_description_url, request_name="DescribeCoverage")
|
||||
coverage_body, coverage_content_type, _ = _bounded_get(
|
||||
coverage_description_url,
|
||||
timeout=args.wms_timeout,
|
||||
max_bytes=MAX_COVERAGE_DESCRIPTION_BYTES,
|
||||
accept="application/xml,text/xml",
|
||||
opener=opener,
|
||||
validator=_validate_wcs_url,
|
||||
)
|
||||
if coverage_content_type and "xml" not in coverage_content_type and "text" not in coverage_content_type:
|
||||
raise RuntimeError("Official WCS coverage description response is not XML")
|
||||
coverage_description = parse_coverage_description(coverage_body)
|
||||
selection = validate_bbox(list(args.bbox), coverage_description["extent_epsg31370"])
|
||||
validate_bbox(list(args.bbox), capabilities["extent_epsg31370"])
|
||||
coverage = verify_flight_coverage(
|
||||
base_url,
|
||||
selection,
|
||||
timeout=args.wms_timeout,
|
||||
opener=opener,
|
||||
)
|
||||
flight_year_matches = coverage["flight_years"] == [release["remote_year"]]
|
||||
release_actionable = release["status"] in {"not_loaded", "update_available"}
|
||||
staging_permitted = release_actionable and flight_year_matches
|
||||
if release["status"] == "blocked_local_version":
|
||||
next_action = "establish_official_local_edition_before_staging"
|
||||
elif release["status"] == "current":
|
||||
next_action = "none_current"
|
||||
elif release["status"] == "blocked_remote_older":
|
||||
next_action = "investigate_catalog_regression"
|
||||
elif not flight_year_matches:
|
||||
next_action = "split_or_review_selection_flight_years"
|
||||
else:
|
||||
next_action = "governed_pixel_stage"
|
||||
return {
|
||||
"schema_version": 1,
|
||||
"status": "passed",
|
||||
"generated_at": datetime.now(timezone.utc).isoformat(),
|
||||
"project_id": args.project_id,
|
||||
"scope": args.scope,
|
||||
"product": product,
|
||||
"release": release,
|
||||
"capabilities": capabilities,
|
||||
"coverage_domain": {
|
||||
**coverage_description,
|
||||
"selected_area_fully_inside_domain": True,
|
||||
"pixel_data_requested": False,
|
||||
},
|
||||
"selection": {
|
||||
"bbox_epsg4326": selection["bbox_epsg4326"],
|
||||
"bbox_epsg31370": [round(value, 3) for value in selection["bbox_epsg31370"]],
|
||||
"width_m": round(selection["width_m"], 3),
|
||||
"height_m": round(selection["height_m"], 3),
|
||||
},
|
||||
"flight_day_coverage": coverage,
|
||||
"flight_year_matches_release": flight_year_matches,
|
||||
"staging_permitted": staging_permitted,
|
||||
"next_action": next_action,
|
||||
"pixel_requests_performed": 0,
|
||||
"datasets_mutated": 0,
|
||||
"automatic_staging": False,
|
||||
"automatic_import": False,
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
try:
|
||||
report = run_preflight(parse_args())
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
return 0
|
||||
except (RuntimeError, ValueError) as exc:
|
||||
print(json.dumps({"status": "error", "error": str(exc)}, ensure_ascii=False, indent=2), file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -55,6 +55,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_bwk_natura2000.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py
|
||||
${PYTHON_BIN} -m py_compile scripts/manage_alz_agriculture_release.py
|
||||
${PYTHON_BIN} -m py_compile scripts/orthophoto_release_preflight.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_buildings_addresses_register.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_mol_dhmv.py
|
||||
${PYTHON_BIN} -m py_compile scripts/provision_regional_dhmv.py
|
||||
|
||||
Reference in New Issue
Block a user