Add governed orthophoto release preflight
GeoIntel CI / docs-smoke (push) Canceled after 0s
GeoIntel CI / contract-smoke (push) Canceled after 0s

This commit is contained in:
Codex
2026-07-17 01:30:19 +02:00
parent 99f1e393cd
commit f48fe7b091
14 changed files with 1187 additions and 3 deletions
+14
View File
@@ -7,6 +7,20 @@
# Changelog
## Sprint 230 Governed orthophoto release preflight (2026-07-17)
- Added an operator-only, read-only preflight for the current Digitaal
Vlaanderen orthophoto product. It binds the canonical product and catalog
envelopes to exact official WMS capabilities and ISO edition evidence.
- Added metadata-only WCS `DescribeCoverage` validation for the governed
EPSG:31370, 15 cm, three-band `Ortho` raster domain and deterministic bounded
`Vliegdagcontour` sampling for selected-area flight-year evidence.
- Classified local official editions as current/update/baseline/remote-older
and kept legacy `most_recent_at_*` values explicitly non-comparable. No
raster pixels, Datasets, Jobs, migrations or automatic refresh were added.
- Added trust-boundary, hash-drift, product, coverage, flight-year, bounds and
runtime-packaging tests plus readiness compilation and operator docs.
## Sprint 229 Governed ALZ definitive release promotion (2026-07-17)
- Added an operator-only `plan -> stage -> review -> apply` coordinator for
+20
View File
@@ -1300,6 +1300,26 @@ Settings: `ORTHOPHOTO_ENABLED`, `ORTHOPHOTO_WMS_URL`,
`ORTHOPHOTO_CACHE_TTL_HOURS`. Keep the official HTTPS URL and 1 m profile
unless a separately verified deployment/model profile requires a change.
Before a future `most_recent` source release is allowed into a governed pixel
stage, run the metadata-only preflight for the exact intended rectangle:
```bash
docker exec geointel python /app/scripts/orthophoto_release_preflight.py \
--project-id 82a85913-c522-45d7-84a1-02b393d89e55 \
--api-url http://127.0.0.1:8000/api/v1 \
--bbox 5.110 51.180 5.117 51.185 \
--refresh-catalog
```
The command reads canonical API envelopes, exact official WMS capabilities,
WCS `DescribeCoverage` and at most 64 queryable flight-day points. It never
requests raster pixels or mutates application/storage state. Only
`staging_permitted=true` may feed a future separate staging command. `current`,
remote-older, mixed/incorrect flight years and legacy local values such as
`most_recent_at_2026-07-15` remain non-stageable. The report's point grid is
flight-date evidence; complete selected-area coverage comes from containment
inside the official 15 cm WCS raster domain.
## Governed DHMV terrain acquisition
`GET /api/v1/projects/{project_id}/datasets/dhmv/products` exposes the fixed
@@ -92,6 +92,7 @@ def test_all_in_one_dockerfile_copies_operator_scripts_for_runtime_use() -> None
"run_split_background_promotion_workflow.sh",
"activate_promoted_yolo_candidate.py",
"manage_grb_refresh.py",
"orthophoto_release_preflight.py",
}
for script_name in required_runtime_scripts:
assert f"COPY scripts/{script_name} /app/scripts/{script_name}" in dockerfile
@@ -0,0 +1,365 @@
from __future__ import annotations
import argparse
from email.message import Message
from hashlib import sha256
import importlib.util
import json
from pathlib import Path
import sys
from urllib.parse import parse_qs, urlparse
import pytest
ROOT = Path(__file__).resolve().parents[2]
SCRIPTS = ROOT / "scripts"
if str(SCRIPTS) not in sys.path:
sys.path.insert(0, str(SCRIPTS))
def load_script():
path = SCRIPTS / "orthophoto_release_preflight.py"
module_name = "test_orthophoto_release_preflight_sprint230"
spec = importlib.util.spec_from_file_location(module_name, path)
assert spec is not None
assert spec.loader is not None
module = importlib.util.module_from_spec(spec)
sys.modules[module_name] = module
spec.loader.exec_module(module)
return module
PREFLIGHT = load_script()
PROJECT_ID = "00000000-0000-0000-0000-000000000001"
def arguments(**overrides) -> argparse.Namespace:
values = {
"project_id": PROJECT_ID,
"scope": PREFLIGHT.DEFAULT_SCOPE,
"api_url": "http://127.0.0.1:8000/api/v1",
"bbox": [5.110, 51.180, 5.117, 51.185],
"refresh_catalog": True,
"api_timeout": 30,
"wms_timeout": 10,
}
values.update(overrides)
return argparse.Namespace(**values)
def capabilities(*, queryable: bool = True, feature_info: bool = True) -> bytes:
info_format = "<Format>application/geo+json</Format>" if feature_info else "<Format>text/plain</Format>"
queryable_value = "1" if queryable else "0"
metadata_url = (
"https://metadata.vlaanderen.be/srv/dut/csw?request=GetRecordById&amp;service=CSW&amp;"
f"id={PREFLIGHT.METADATA_IDENTIFIER}"
)
return f"""
<WMS_Capabilities version="1.3.0" xmlns="http://www.opengis.net/wms"
xmlns:xlink="http://www.w3.org/1999/xlink">
<Capability>
<Request><GetFeatureInfo>{info_format}</GetFeatureInfo></Request>
<Layer>
<CRS>EPSG:31370</CRS>
<BoundingBox CRS="EPSG:31370" minx="22000" miny="150000" maxx="259000" maxy="245000" />
<Layer queryable="0"><Name>Ortho</Name><MetadataURL><OnlineResource xlink:href="{metadata_url}" /></MetadataURL></Layer>
<Layer queryable="{queryable_value}"><Name>Vliegdagcontour</Name><MetadataURL><OnlineResource xlink:href="{metadata_url}" /></MetadataURL></Layer>
</Layer>
</Capability>
</WMS_Capabilities>
""".encode()
def coverage_description(*, coverage_id: str = "Ortho", resolution: float = 0.15) -> bytes:
return f"""
<wcs:CoverageDescriptions xmlns:wcs="http://www.opengis.net/wcs/2.0"
xmlns:gml="http://www.opengis.net/gml/3.2" xmlns:gmlcov="http://www.opengis.net/gmlcov/1.0"
xmlns:swe="http://www.opengis.net/swe/2.0">
<wcs:CoverageDescription>
<gml:boundedBy><gml:Envelope srsName="http://www.opengis.net/def/crs/EPSG/0/31370">
<gml:lowerCorner>21375 152250</gml:lowerCorner><gml:upperCorner>259500 244875</gml:upperCorner>
</gml:Envelope></gml:boundedBy>
<wcs:CoverageId>{coverage_id}</wcs:CoverageId>
<gml:domainSet><gml:RectifiedGrid dimension="2">
<gml:offsetVector>{resolution} 0</gml:offsetVector><gml:offsetVector>0 -{resolution}</gml:offsetVector>
</gml:RectifiedGrid></gml:domainSet>
<gmlcov:rangeType><swe:DataRecord>
<swe:field name="Band_1"/><swe:field name="Band_2"/><swe:field name="Band_3"/>
</swe:DataRecord></gmlcov:rangeType>
<wcs:ServiceParameters><wcs:CoverageSubtype>RectifiedGridCoverage</wcs:CoverageSubtype>
<wcs:nativeFormat>image/tiff</wcs:nativeFormat></wcs:ServiceParameters>
</wcs:CoverageDescription>
</wcs:CoverageDescriptions>
""".encode()
def catalog_item(body: bytes, *, local: str | None = "most_recent_at_2026-07-15", remote: str = "2025.04") -> dict:
if local is None:
comparison = "no_local_data"
elif PREFLIGHT.EDITION_PATTERN.fullmatch(local):
comparison = "same" if local == remote else "different"
else:
comparison = "not_comparable"
return {
"source_name": PREFLIGHT.SOURCE_NAME,
"status": "available",
"reachable": True,
"matched_layers": ["Ortho", "Vliegdagcontour"],
"missing_layers": [],
"metadata_identifier": PREFLIGHT.METADATA_IDENTIFIER,
"metadata_url": f"https://metadata.vlaanderen.be/{PREFLIGHT.METADATA_IDENTIFIER}",
"remote_title": f"Orthofoto meest recent, {remote}",
"remote_version": remote,
"remote_modified_at": "2026-04-27T00:00:00Z",
"remote_published_at": "2025-12-11T00:00:00Z",
"local_source_version": local,
"comparison_status": comparison,
"checked_at": "2026-07-17T00:00:00Z",
"endpoint_url": (
"https://geo.api.vlaanderen.be/OMWRGBMRVL/wms?"
"SERVICE=WMS&VERSION=1.3.0&REQUEST=GetCapabilities"
),
"capabilities_sha256": sha256(body).hexdigest(),
}
def product() -> dict:
return {
"key": "most_recent",
"display_name": "Meest recente winterluchtbeeld",
"observation_label": "Meest recent beschikbaar",
"temporal_granularity": "snapshot",
"native_resolution_m": 0.15,
"supports_detection": True,
"color_mode": "rgb",
"catalog_url": PREFLIGHT.CATALOG_URL,
"limitation_message": "rolling source",
}
def loader(body: bytes, item: dict):
def load(_api_url: str, path: str, _timeout: int) -> dict:
if path == f"projects/{PROJECT_ID}":
return {"id": PROJECT_ID, "name": "Kempen Regional Workbench"}
if path.endswith("/datasets/orthophoto/products"):
return {"items": [product()], "total": 1}
if "/datasets/source-catalog-probes?" in path:
return {"items": [item]}
raise AssertionError(path)
return load
class Response:
def __init__(self, body: bytes, *, url: str, content_type: str) -> None:
self.body = body
self.url = url
self.headers = Message()
self.headers["Content-Type"] = content_type
self.headers["Content-Length"] = str(len(body))
def __enter__(self):
return self
def __exit__(self, *_args) -> None:
return None
def geturl(self) -> str:
return self.url
def read(self, size: int = -1) -> bytes:
return self.body if size < 0 else self.body[:size]
def opener(
body: bytes,
*,
flight_year: int = 2025,
empty: bool = False,
requests: list[str] | None = None,
coverage_body: bytes | None = None,
):
def open_request(request, timeout: int):
assert timeout == 10
url = request.full_url
query = parse_qs(urlparse(url).query)
request_name = (query.get("REQUEST") or [""])[0]
if requests is not None:
requests.append(request_name)
if request_name == "GetCapabilities":
return Response(body, url=url, content_type="text/xml")
if request_name == "DescribeCoverage":
return Response(coverage_body or coverage_description(), url=url, content_type="text/xml")
assert request_name == "GetFeatureInfo"
assert query["LAYERS"] == ["Vliegdagcontour"]
features = [] if empty else [
{
"type": "Feature",
"geometry": None,
"properties": {"OpnDatum": f"5/4/{flight_year}", "FID": "4"},
"layerName": "Vliegdagcontour",
}
]
payload = json.dumps({"type": "FeatureCollection", "features": features}).encode()
return Response(payload, url=url, content_type="application/geo+json")
return open_request
def run(*, local: str | None = "most_recent_at_2026-07-15", remote: str = "2025.04", flight_year: int = 2025):
body = capabilities()
item = catalog_item(body, local=local, remote=remote)
calls: list[str] = []
report = PREFLIGHT.run_preflight(
arguments(),
loader=loader(body, item),
opener=opener(body, flight_year=flight_year, requests=calls),
)
return report, calls
def test_live_contract_shape_blocks_non_comparable_legacy_local_version() -> None:
report, calls = run()
assert report["status"] == "passed"
assert report["release"]["status"] == "blocked_local_version"
assert report["release"]["remote_edition"] == "2025.04"
assert report["flight_day_coverage"]["sample_count"] == 20
assert report["flight_day_coverage"]["sample_coverage_ratio"] == 1.0
assert report["flight_day_coverage"]["flight_years"] == [2025]
assert report["staging_permitted"] is False
assert report["next_action"] == "establish_official_local_edition_before_staging"
assert report["pixel_requests_performed"] == 0
assert set(calls) == {"GetCapabilities", "DescribeCoverage", "GetFeatureInfo"}
assert report["coverage_domain"]["selected_area_fully_inside_domain"] is True
assert report["coverage_domain"]["pixel_data_requested"] is False
@pytest.mark.parametrize(
("local", "remote", "expected_status", "stageable"),
[
(None, "2025.04", "not_loaded", True),
("2025.04", "2025.04", "current", False),
("2024.01", "2025.04", "update_available", True),
("2026.01", "2025.04", "blocked_remote_older", False),
],
)
def test_release_ordering_requires_comparable_official_editions(
local: str | None,
remote: str,
expected_status: str,
stageable: bool,
) -> None:
report, _ = run(local=local, remote=remote)
assert report["release"]["status"] == expected_status
assert report["staging_permitted"] is stageable
def test_flight_year_must_match_remote_release_year() -> None:
report, _ = run(local="2024.01", flight_year=2024)
assert report["release"]["status"] == "update_available"
assert report["flight_year_matches_release"] is False
assert report["staging_permitted"] is False
assert report["next_action"] == "split_or_review_selection_flight_years"
def test_capabilities_hash_drift_fails_closed() -> None:
body = capabilities()
item = catalog_item(body, local=None)
changed = body.replace(b"queryable=\"1\"", b"queryable=\"0\"")
with pytest.raises(RuntimeError, match="changed after"):
PREFLIGHT.run_preflight(
arguments(),
loader=loader(body, item),
opener=opener(changed),
)
@pytest.mark.parametrize(
("body", "message"),
[
(capabilities(queryable=False), "not queryable"),
(capabilities(feature_info=False), "does not advertise GeoJSON"),
],
)
def test_flight_day_capability_requirements_fail_closed(body: bytes, message: str) -> None:
item = catalog_item(body, local=None)
with pytest.raises(RuntimeError, match=message):
PREFLIGHT.run_preflight(arguments(), loader=loader(body, item), opener=opener(body))
def test_missing_flight_day_coverage_fails_closed() -> None:
body = capabilities()
item = catalog_item(body, local=None)
with pytest.raises(RuntimeError, match="has no coverage"):
PREFLIGHT.run_preflight(
arguments(),
loader=loader(body, item),
opener=opener(body, empty=True),
)
@pytest.mark.parametrize(
"coverage_body",
[coverage_description(coverage_id="Other"), coverage_description(resolution=0.25)],
)
def test_coverage_domain_identity_and_resolution_fail_closed(coverage_body: bytes) -> None:
body = capabilities()
item = catalog_item(body, local=None)
with pytest.raises(RuntimeError, match="WCS"):
PREFLIGHT.run_preflight(
arguments(),
loader=loader(body, item),
opener=opener(body, coverage_body=coverage_body),
)
def test_product_variant_and_catalog_comparison_are_bound() -> None:
wrong = product()
wrong["color_mode"] = "panchromatic"
with pytest.raises(RuntimeError, match="product variant"):
PREFLIGHT.validate_product([wrong])
body = capabilities()
item = catalog_item(body, local="2025.04")
item["comparison_status"] = "different"
with pytest.raises(RuntimeError, match="internally inconsistent"):
PREFLIGHT.catalog_decision(item)
def test_only_exact_official_wms_url_is_allowed() -> None:
valid = "https://geo.api.vlaanderen.be/OMWRGBMRVL/wms?SERVICE=WMS&REQUEST=GetCapabilities"
assert PREFLIGHT._validate_wms_url(valid, request_name="GetCapabilities").endswith("/OMWRGBMRVL/wms")
with pytest.raises(RuntimeError, match="outside the official allowlist"):
PREFLIGHT._validate_wms_url(valid.replace("geo.api.vlaanderen.be", "example.com"))
with pytest.raises(RuntimeError, match="not an exact"):
PREFLIGHT._validate_wms_url(valid.replace("GetCapabilities", "GetMap"), request_name="GetCapabilities")
def test_bbox_matches_acquisition_bounds_and_grid_is_bounded() -> None:
extent = [22000.0, 150000.0, 259000.0, 245000.0]
selection = PREFLIGHT.validate_bbox([5.110, 51.180, 5.117, 51.185], extent)
samples = PREFLIGHT._sample_grid(selection)
assert len(samples) == 20
assert len(samples) <= PREFLIGHT.MAX_SAMPLE_COUNT
with pytest.raises(RuntimeError, match="at least"):
PREFLIGHT.validate_bbox([5.110, 51.180, 5.1105, 51.1805], extent)
with pytest.raises(RuntimeError, match="may not exceed"):
PREFLIGHT.validate_bbox([5.05, 51.15, 5.25, 51.33], extent)
def test_operator_preflight_is_read_only_packaged_and_readiness_checked() -> None:
script = (SCRIPTS / "orthophoto_release_preflight.py").read_text(encoding="utf-8")
dockerfile = (ROOT / "deploy" / "unraid" / "Dockerfile.all-in-one").read_text(encoding="utf-8")
readiness = (SCRIPTS / "run_readiness_check.sh").read_text(encoding="utf-8")
assert "GetMap" not in script
assert "GetCoverage" not in script
assert "/datasets/upload" not in script
assert "INSERT INTO" not in script
assert "COPY scripts/orthophoto_release_preflight.py" in dockerfile
assert "py_compile scripts/orthophoto_release_preflight.py" in readiness
+1
View File
@@ -92,6 +92,7 @@ COPY scripts/provision_mol_bwk_natura2000.py /app/scripts/provision_mol_bwk_natu
COPY scripts/provision_regional_bwk_natura2000.py /app/scripts/provision_regional_bwk_natura2000.py
COPY scripts/provision_agricultural_parcel_history.py /app/scripts/provision_agricultural_parcel_history.py
COPY scripts/manage_alz_agriculture_release.py /app/scripts/manage_alz_agriculture_release.py
COPY scripts/orthophoto_release_preflight.py /app/scripts/orthophoto_release_preflight.py
COPY scripts/provision_buildings_addresses_register.py /app/scripts/provision_buildings_addresses_register.py
COPY scripts/provision_regional_timeseries.py /app/scripts/provision_regional_timeseries.py
COPY scripts/geographic_scopes.py /app/scripts/geographic_scopes.py
+10
View File
@@ -466,6 +466,16 @@ revalidates the current catalog and delegates to the existing Dataset upload
contract. No ALZ release endpoint, background task or provider URL parameter
is added; v1/v2 campaign snapshots remain non-importable.
Current-orthophoto release preflight also remains outside the HTTP request
cycle in `scripts/orthophoto_release_preflight.py`. It composes the existing
product-registry and source-catalog envelopes with allowlisted WMS
`GetCapabilities`, WCS `DescribeCoverage` and bounded `Vliegdagcontour`
`GetFeatureInfo` evidence for one 128-1,024 m EPSG:4326 selection. The result
reports product variant, official `YYYY.NN` edition, exact raster-domain
containment, sampled flight dates/years, local comparison state and
`staging_permitted`. It performs no pixel request, upload, Job, Dataset write
or legacy metadata rewrite. This operator script adds no public API contract.
The endpoint accepts no arbitrary URL, feature query, area or layer. It does
not fetch vector features, raster pixels or models, create jobs/datasets, write
to PostGIS or trigger an import. The normal `source-freshness` endpoint remains
+37
View File
@@ -9715,3 +9715,40 @@ Live validation:
Next:
- Evaluate ALZ as the next governed edition probe only after its official
machine-readable release version and schema stability are verified.
## Sprint 230 - Governed orthophoto release preflight (2026-07-17)
Implemented:
- Added `scripts/orthophoto_release_preflight.py` as a read-only gate for one
bounded current-orthophoto selection in the approved Kempen regional scope.
- Bound the existing canonical product registry and source-catalog result to
the exact official WMS 1.3.0 capabilities hash, ISO metadata identifier and
`YYYY.NN` edition. Redirects, hosts, paths and response sizes fail closed.
- Added metadata-only WCS `DescribeCoverage` validation for `Ortho`,
EPSG:31370, the complete raster domain, 15 cm rectified grid, three bands and
TIFF native format. Selection limits remain identical to acquisition:
128-1,024 m per projected side.
- Added a maximum 64-point deterministic `Vliegdagcontour` grid with exact
GeoJSON layer/date/year validation and hashed sample evidence. WCS domain
containment proves raster-domain coverage; point samples remain honestly
labelled as flight-date evidence rather than polygon-union geometry.
- Kept all pixel requests, filesystem staging, Jobs, Dataset/PostGIS writes,
migrations and browser behavior out of this sprint. Local
`most_recent_at_*` acquisition markers remain non-comparable and blocked.
Validation so far:
- 42 focused preflight/runtime-packaging tests passed. Coverage includes
release ordering, legacy provenance, hash drift, product variant, official
host/path, queryable/GeoJSON capability, WCS identity/resolution, missing
flight coverage, flight-year mismatch, selection bounds and no-write rules.
- Target Python compilation and Ruff passed.
- A read-only compatibility run against Tower for bbox
`5.110,51.180,5.117,51.185` verified official edition `2025.04`, exact Ortho
WCS domain containment and 20/20 flight-day samples from 2025. It performed
zero pixel requests and correctly returned `staging_permitted=false` because
local version `most_recent_at_2026-07-15` is not an official edition.
Boundary:
- This sprint supplies preflight only. It deliberately does not promote or
backfill existing orthophotos. A future stage/review/apply coordinator must
revalidate and retain the exact preflight identity before creating a new
immutable raster Dataset with official `YYYY.NN` source version.
+8
View File
@@ -291,6 +291,14 @@ annual Dataset, DatasetVersion and vector_features records with
`source_version=<year>-definitive`. Earlier annual snapshots are retained and
provisional v1/v2 publications cannot create rows.
The current-orthophoto release preflight likewise adds no lifecycle table or
migration. It is read-only and creates neither Dataset nor Job. It compares the
existing local `source_version` with the official `YYYY.NN` catalog edition
and verifies WMS/WCS/flight-day evidence for one bounded selection. A later
governed pixel apply must still create an immutable raster Dataset plus
DatasetVersion through DatasetService and retain that exact edition/evidence;
direct metadata backfill of legacy `most_recent_at_*` rows is prohibited.
## Geometry normalization
- User-drawn polygons arrive as EPSG:4326.
+16 -1
View File
@@ -24,6 +24,21 @@ aanvraag. GeoIntel verzint geen historische pixelopnamedatum. Bronnen:
- https://www.vlaanderen.be/datavindplaats/catalogus/orthofotomozaiek-middenschalig-winteropnamen-kleur-meest-recent-vlaanderen
- https://www.vlaanderen.be/digitaal-vlaanderen/onze-diensten-en-platformen/luchtopnamen/gebruik-orthofotomozaieken
Voor een toekomstige rolling-releasebeslissing gebruikt
`scripts/orthophoto_release_preflight.py` uitsluitend metadata. Het bindt de
lokale `most_recent`-productvariant aan de officiële ISO-editie en exacte WMS-
capabilitieshash, controleert het EPSG:31370/15 cm/driebanden-rasterdomein via
WCS `DescribeCoverage` en bemonstert de querybare `Vliegdagcontour` op een
deterministisch raster met maximaal 128 m afstand. Er worden geen `GetMap`-,
`GetCoverage`- of uploadrequests uitgevoerd.
De WCS-domeincontrole bewijst dat de volledige begrensde selectie binnen het
officiële rasterdomein valt. De WMS publiceert geen vliegdagpolygonen als WFS;
het vluchtjaar blijft daarom expliciet puntbewijs en geen verzonnen polygon-
union. Een selectie met ontbrekende contourpunten, meerdere/afwijkende
vluchtjaren, gewijzigde service-identiteit of een niet-vergelijkbare lokale
`most_recent_at_*` marker is niet stagebaar.
Dit document verzamelt concrete databronnen voor GeoIntel Kempen.
## Cross-domain official area profile
@@ -174,7 +189,7 @@ gebeurd.
| GRB gebouwen/wegen/water/percelen | operationele, expliciete plan-stage-apply refresh met onveranderlijke snapshots | alleen een nieuw officieel gedateerd cataloguseditie na operatorbevestiging ophalen |
| Statbel bevolking | jaarlijkse, expliciete edities in één tijdreeks; officiële DCAT-releaseprobe | een nieuwe publicatie alleen na schema-, sectorgeometrie- en totalencontrole toevoegen |
| ALZ landbouwgebruikspercelen | definitieve jaarlijkse edities 2008-2025; expliciete publicatieprobe en plan-stage-review-apply promotie; metricvergelijking zonder objectlineage | alleen een nieuwere definitieve v3-editie na gestagede schema-/codelijst-/scopecontrole en benoemde review toevoegen |
| orthofoto | vaste lokale opname per expliciete analysezone; catalogusprobe is alleen een signaal | vluchtjaar, productvariant en dekking vergelijken voordat nieuwe pixels worden opgehaald |
| orthofoto | vaste lokale opname per expliciete analysezone; read-only releasepreflight voor variant, officiële editie, exact WCS-domein en begrensd vluchtjaarbewijs | eerst officiële lokale editieprovenance vastleggen; daarna pas een afzonderlijke menselijke pixel-stage/apply-flow bouwen |
| landgebruik, thematische rasters, DHMV en VMM-scenario's | vaste product-/scenario-edities, geen rolling snapshot | alleen een nieuwe gedocumenteerde producteditie als afzonderlijke Dataset verwerven |
| bodemkaart en historische kaarten | historische referentie-editie | niet als verouderde actuele bron labelen; alleen vervangen bij een officiële inhoudelijke heruitgave |
| BWK/Natura 2000 en gebouwen-/adressenregister | expliciete actuele snapshot met eigen methodologische betekenis | eerst een stabiele officiële editieprobe en bron-specifieke reconciliatiecontrole toevoegen |
+7
View File
@@ -274,6 +274,13 @@ layer, observation label, `observed_at`, optional `valid_from`/`valid_to`,
temporal granularity, request/spatial hash, attribution and a limitation that
states whether the product is annual, multi-year or merely most recent.
A future rolling `most_recent` import must also retain the exact official
`YYYY.NN` edition and the preflight identities for WMS capabilities, WCS
coverage description, selected EPSG:31370 domain and sampled flight year. A
legacy `most_recent_at_<date>` value is acquisition timing, not an official
edition, and cannot be promoted or compared as if it were one. The read-only
preflight creates no Dataset and does not retroactively rewrite that evidence.
### Hydrological station observations
Waterinfo observations are persisted as EPSG:4326 Point features, one station
+8
View File
@@ -147,6 +147,14 @@ product/layer, request/spatial hash, temporal validity and limitations are held
in source/provenance metadata. Browser PNG rendering is derived on request and
does not replace the stored GeoTIFF.
The orthophoto release preflight writes no source file, raster or database row.
Its JSON stdout may be retained by an operator as review evidence, but it is
not itself staging authorization. The report binds official WMS and WCS XML
hashes, the exact selected domain and hashed flight-day sample evidence. A
future pixel stage must persist and revalidate that identity separately before
DatasetService is called; existing `most_recent_at_*` raster metadata is not
silently rewritten.
DHMV II DTM/DSM outputs are also normal raster Dataset files. The provider WCS
returns multipart coverage data; GeoIntel retains response and extracted
coverage SHA256 values in provenance, then stores one normalized, compressed,
+5 -2
View File
@@ -690,5 +690,8 @@ This file now starts with the current implementation status. Older preparation/b
- [x] Document refresh readiness across the complete official-source portfolio.
- [x] Verify and add the governed ALZ catalog probe against the official
campaign-snapshot and definitive-archive publication contract.
- [ ] Keep orthophoto refresh manual until product variant, flight year and
complete selected-area coverage can be compared deterministically.
- [x] Add a read-only orthophoto preflight for product variant, official
edition, exact WCS selected-area domain and deterministic flight-year points.
- [ ] Keep orthophoto pixel refresh manual and blocked until a separate
plan-stage-review-apply flow can retain the passed preflight identity and
create a new immutable Dataset with official `YYYY.NN` source version.
+694
View File
@@ -0,0 +1,694 @@
#!/usr/bin/env python3
"""Read-only release preflight for the governed current orthophoto product.
The preflight does not request or stage raster pixels. It binds the canonical
GeoIntel catalog result to the current product registry and the exact official
WMS capabilities, then probes the queryable flight-day layer on a bounded,
deterministic grid. Local rolling markers remain non-comparable and blocked.
"""
from __future__ import annotations
import argparse
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone
from hashlib import sha256
import json
import math
import os
import re
import sys
from typing import Any, Callable
from urllib.error import HTTPError, URLError
from urllib.parse import parse_qs, urlencode, urlparse
from urllib.request import Request, urlopen
from xml.etree import ElementTree
from pyproj import Transformer
from geographic_scopes import GEOGRAPHIC_SCOPES
DEFAULT_API_URL = "http://127.0.0.1:8000/api/v1"
DEFAULT_SCOPE = "kempen-transport-region"
SOURCE_NAME = "digitaal_vlaanderen_orthophoto"
PRODUCT_KEY = "most_recent"
WMS_HOST = "geo.api.vlaanderen.be"
WMS_PATH = "/OMWRGBMRVL/wms"
WCS_PATH = "/OMWRGBMRVL/wcs"
CATALOG_URL = (
"https://www.vlaanderen.be/datavindplaats/catalogus/"
"orthofotomozaiek-middenschalig-winteropnamen-kleur-meest-recent-vlaanderen"
)
METADATA_IDENTIFIER = "f5304d6d-0dd4-43fd-a726-427af31e8d61"
EDITION_PATTERN = re.compile(r"^(20[0-9]{2})\.([0-9]{2})$")
SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$")
FLIGHT_YEAR_PATTERN = re.compile(r"(?:^|[^0-9])(20[0-9]{2})(?:$|[^0-9])")
MAX_CAPABILITIES_BYTES = 2 * 1024 * 1024
MAX_COVERAGE_DESCRIPTION_BYTES = 512 * 1024
MAX_FEATURE_INFO_BYTES = 128 * 1024
MAX_SAMPLE_SPACING_M = 128.0
MAX_SAMPLE_COUNT = 64
MIN_SIDE_M = 128.0
MAX_SIDE_M = 1024.0
FEATURE_INFO_SIZE = 1000
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description=(
"Read-only current-orthophoto release preflight. No raster pixels are fetched, staged or imported."
)
)
parser.add_argument("--project-id", required=True, help="GeoIntel project UUID for the governed scope")
parser.add_argument("--scope", choices=(DEFAULT_SCOPE,), default=DEFAULT_SCOPE)
parser.add_argument("--api-url", default=os.environ.get("GEOINTEL_API_URL", DEFAULT_API_URL))
parser.add_argument(
"--bbox",
nargs=4,
type=float,
metavar=("MIN_LON", "MIN_LAT", "MAX_LON", "MAX_LAT"),
required=True,
help="Exact EPSG:4326 selection; each projected side must be between 128 and 1024 metres",
)
parser.add_argument("--refresh-catalog", action="store_true", help="Bypass the short catalog cache")
parser.add_argument("--api-timeout", type=int, default=180)
parser.add_argument("--wms-timeout", type=int, default=30)
return parser.parse_args()
def api_data(api_url: str, path: str, timeout: int) -> dict[str, Any]:
endpoint = f"{api_url.rstrip('/')}/{path.lstrip('/')}"
request = Request(
endpoint,
headers={"Accept": "application/json", "User-Agent": "GeoIntel-orthophoto-preflight/1.0"},
)
try:
with urlopen(request, timeout=timeout) as response:
payload = json.load(response)
except HTTPError as exc:
body = exc.read().decode("utf-8", errors="replace")
raise RuntimeError(f"GeoIntel API returned HTTP {exc.code}: {body[-1000:]}") from exc
except URLError as exc:
raise RuntimeError(f"GeoIntel API is unreachable: {exc.reason}") from exc
if not isinstance(payload, dict) or not isinstance(payload.get("data"), dict):
raise RuntimeError("GeoIntel API response is not a canonical data envelope")
return payload["data"]
def validate_project_scope(args: argparse.Namespace, loader: Callable[[str, str, int], dict[str, Any]]) -> None:
project = loader(args.api_url, f"projects/{args.project_id}", args.api_timeout)
expected_name = GEOGRAPHIC_SCOPES[args.scope].project_name
if project.get("name") != expected_name:
raise RuntimeError(
f"Project {args.project_id} is '{project.get('name')}', but scope {args.scope} requires '{expected_name}'"
)
def _validate_service_url(url: str, *, path: str, request_name: str | None = None) -> str:
parsed = urlparse(url)
if (
parsed.scheme.lower() != "https"
or parsed.hostname != WMS_HOST
or parsed.port not in (None, 443)
or parsed.username
or parsed.password
or parsed.fragment
or parsed.path.rstrip("/").lower() != path.lower()
):
raise RuntimeError("Orthophoto service URL is outside the official allowlist")
if request_name is not None:
requests = parse_qs(parsed.query).get("REQUEST") or parse_qs(parsed.query).get("request") or []
if len(requests) != 1 or requests[0].lower() != request_name.lower():
raise RuntimeError(f"Orthophoto service URL is not an exact {request_name} request")
return f"https://{WMS_HOST}{path}"
def _validate_wms_url(url: str, *, request_name: str | None = None) -> str:
return _validate_service_url(url, path=WMS_PATH, request_name=request_name)
def _validate_wcs_url(url: str, *, request_name: str | None = None) -> str:
return _validate_service_url(url, path=WCS_PATH, request_name=request_name)
def _bounded_get(
url: str,
*,
timeout: int,
max_bytes: int,
accept: str,
opener: Callable[..., Any] | None = None,
validator: Callable[[str], str] = _validate_wms_url,
) -> tuple[bytes, str, str]:
request = Request(url, headers={"Accept": accept, "User-Agent": "GeoIntel-orthophoto-preflight/1.0"})
fetch = opener or urlopen
try:
response = fetch(request, timeout=timeout)
with response:
final_url = response.geturl()
validator(final_url)
content_type = str(response.headers.get("Content-Type") or "").split(";", 1)[0].strip().lower()
declared = response.headers.get("Content-Length")
try:
if declared and int(declared) > max_bytes:
raise RuntimeError("Official WMS response exceeds the configured preflight limit")
except ValueError as exc:
raise RuntimeError("Official WMS response has an invalid Content-Length") from exc
body = response.read(max_bytes + 1)
except HTTPError as exc:
raise RuntimeError(f"Official orthophoto WMS returned HTTP {exc.code}") from exc
except URLError as exc:
raise RuntimeError(f"Official orthophoto WMS is unreachable: {exc.reason}") from exc
if len(body) > max_bytes:
raise RuntimeError("Official WMS response exceeds the configured preflight limit")
return body, content_type, final_url
def _local_name(tag: str) -> str:
return tag.rsplit("}", 1)[-1]
def _children(element: ElementTree.Element, name: str) -> list[ElementTree.Element]:
return [child for child in element if _local_name(child.tag) == name]
def _child_text(element: ElementTree.Element, name: str) -> str | None:
for child in element:
if _local_name(child.tag) == name and child.text:
return child.text.strip()
return None
def parse_capabilities(body: bytes, expected_sha256: str) -> dict[str, Any]:
if not SHA256_PATTERN.fullmatch(expected_sha256):
raise RuntimeError("Canonical catalog response has no valid capabilities SHA-256")
actual_sha256 = sha256(body).hexdigest()
if actual_sha256 != expected_sha256:
raise RuntimeError("Official WMS capabilities changed after the canonical catalog check")
upper = body[:4096].upper()
if b"<!DOCTYPE" in upper or b"<!ENTITY" in upper:
raise RuntimeError("Official WMS capabilities contains a prohibited XML declaration")
try:
root = ElementTree.fromstring(body)
except ElementTree.ParseError as exc:
raise RuntimeError("Official WMS capabilities is invalid XML") from exc
if _local_name(root.tag) != "WMS_Capabilities" or root.attrib.get("version") != "1.3.0":
raise RuntimeError("Official orthophoto service is not WMS 1.3.0")
capability = next((node for node in root if _local_name(node.tag) == "Capability"), None)
root_layer = (
next((node for node in capability if _local_name(node.tag) == "Layer"), None)
if capability is not None
else None
)
if capability is None or root_layer is None:
raise RuntimeError("Official WMS capabilities has no root layer")
layers: dict[str, ElementTree.Element] = {}
for node in root.iter():
if _local_name(node.tag) != "Layer":
continue
name = _child_text(node, "Name")
if name:
layers[name] = node
if set(("Ortho", "Vliegdagcontour")) - set(layers):
raise RuntimeError("Official WMS capabilities does not contain both governed orthophoto layers")
if layers["Vliegdagcontour"].attrib.get("queryable") != "1":
raise RuntimeError("Official Vliegdagcontour layer is not queryable")
formats: set[str] = set()
for request_node in capability:
if _local_name(request_node.tag) != "Request":
continue
for operation in request_node:
if _local_name(operation.tag) == "GetFeatureInfo":
formats.update(
(item.text or "").strip().lower()
for item in operation
if _local_name(item.tag) == "Format" and item.text
)
if "application/geo+json" not in formats:
raise RuntimeError("Official WMS does not advertise GeoJSON feature info")
bbox = None
for node in _children(root_layer, "BoundingBox"):
if node.attrib.get("CRS") == "EPSG:31370":
try:
bbox = [float(node.attrib[key]) for key in ("minx", "miny", "maxx", "maxy")]
except (KeyError, ValueError) as exc:
raise RuntimeError("Official WMS EPSG:31370 extent is invalid") from exc
break
if bbox is None or bbox[0] >= bbox[2] or bbox[1] >= bbox[3]:
raise RuntimeError("Official WMS does not advertise a valid EPSG:31370 extent")
metadata_identifiers: set[str] = set()
for layer_name in ("Ortho", "Vliegdagcontour"):
for node in layers[layer_name].iter():
if _local_name(node.tag) != "OnlineResource":
continue
href = next((value for key, value in node.attrib.items() if _local_name(key) == "href"), "")
if METADATA_IDENTIFIER in href:
metadata_identifiers.add(METADATA_IDENTIFIER)
if metadata_identifiers != {METADATA_IDENTIFIER}:
raise RuntimeError("Governed orthophoto layers do not share the expected ISO metadata identity")
return {
"service_version": "1.3.0",
"capabilities_sha256": actual_sha256,
"layers": ["Ortho", "Vliegdagcontour"],
"vliegdagcontour_queryable": True,
"feature_info_format": "application/geo+json",
"extent_epsg31370": bbox,
"metadata_identifier": METADATA_IDENTIFIER,
}
def parse_coverage_description(body: bytes) -> dict[str, Any]:
upper = body[:4096].upper()
if b"<!DOCTYPE" in upper or b"<!ENTITY" in upper:
raise RuntimeError("Official WCS coverage description contains a prohibited XML declaration")
try:
root = ElementTree.fromstring(body)
except ElementTree.ParseError as exc:
raise RuntimeError("Official WCS coverage description is invalid XML") from exc
coverage_ids = [
(node.text or "").strip()
for node in root.iter()
if _local_name(node.tag) == "CoverageId" and node.text
]
if coverage_ids != ["Ortho"]:
raise RuntimeError("Official WCS coverage description is not the governed Ortho coverage")
envelope = next((node for node in root.iter() if _local_name(node.tag) == "Envelope"), None)
if envelope is None or not str(envelope.attrib.get("srsName") or "").endswith("/31370"):
raise RuntimeError("Official WCS Ortho domain is not EPSG:31370")
try:
lower = [float(value) for value in (_child_text(envelope, "lowerCorner") or "").split()]
upper_corner = [float(value) for value in (_child_text(envelope, "upperCorner") or "").split()]
except ValueError as exc:
raise RuntimeError("Official WCS Ortho domain corners are invalid") from exc
if len(lower) != 2 or len(upper_corner) != 2 or lower[0] >= upper_corner[0] or lower[1] >= upper_corner[1]:
raise RuntimeError("Official WCS Ortho domain corners are invalid")
grid = next((node for node in root.iter() if _local_name(node.tag) == "RectifiedGrid"), None)
if grid is None or grid.attrib.get("dimension") != "2":
raise RuntimeError("Official WCS Ortho domain is not a two-dimensional rectified grid")
offsets: list[list[float]] = []
for node in grid.iter():
if _local_name(node.tag) == "offsetVector" and node.text:
try:
offsets.append([float(value) for value in node.text.split()])
except ValueError as exc:
raise RuntimeError("Official WCS Ortho grid resolution is invalid") from exc
if (
len(offsets) != 2
or any(len(vector) != 2 for vector in offsets)
or not math.isclose(offsets[0][0], 0.15, abs_tol=1e-9)
or not math.isclose(offsets[0][1], 0.0, abs_tol=1e-9)
or not math.isclose(offsets[1][0], 0.0, abs_tol=1e-9)
or not math.isclose(offsets[1][1], -0.15, abs_tol=1e-9)
):
raise RuntimeError("Official WCS Ortho grid is no longer the governed 15 cm product")
field_count = sum(_local_name(node.tag) == "field" for node in root.iter())
subtype = next(
((node.text or "").strip() for node in root.iter() if _local_name(node.tag) == "CoverageSubtype"),
"",
)
native_format = next(
((node.text or "").strip() for node in root.iter() if _local_name(node.tag) == "nativeFormat"),
"",
)
if field_count != 3 or subtype != "RectifiedGridCoverage" or native_format != "image/tiff":
raise RuntimeError("Official WCS Ortho range or native format changed")
return {
"coverage_id": "Ortho",
"crs": "EPSG:31370",
"extent_epsg31370": [lower[0], lower[1], upper_corner[0], upper_corner[1]],
"native_resolution_m": 0.15,
"band_count": 3,
"native_format": "image/tiff",
"coverage_description_sha256": sha256(body).hexdigest(),
}
def validate_bbox(values: list[float], advertised_extent: list[float]) -> dict[str, Any]:
if len(values) != 4 or not all(math.isfinite(value) for value in values):
raise RuntimeError("Selection bbox must contain four finite EPSG:4326 values")
min_x, min_y, max_x, max_y = values
if min_x >= max_x or min_y >= max_y or not (-180 <= min_x < max_x <= 180) or not (-90 <= min_y < max_y <= 90):
raise RuntimeError("Selection bbox is not a valid EPSG:4326 rectangle")
transformer = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
lambert = [float(value) for value in transformer.transform_bounds(min_x, min_y, max_x, max_y, densify_pts=21)]
width_m = lambert[2] - lambert[0]
height_m = lambert[3] - lambert[1]
if width_m < MIN_SIDE_M or height_m < MIN_SIDE_M:
raise RuntimeError(f"Orthophoto preflight selection must be at least {MIN_SIDE_M:.0f} by {MIN_SIDE_M:.0f} metres")
if width_m > MAX_SIDE_M or height_m > MAX_SIDE_M:
raise RuntimeError(f"Orthophoto preflight selection may not exceed {MAX_SIDE_M:.0f} by {MAX_SIDE_M:.0f} metres")
if not (
advertised_extent[0] <= lambert[0]
and advertised_extent[1] <= lambert[1]
and lambert[2] <= advertised_extent[2]
and lambert[3] <= advertised_extent[3]
):
raise RuntimeError("Selection is outside the official WMS advertised EPSG:31370 extent")
return {
"bbox_epsg4326": [min_x, min_y, max_x, max_y],
"bbox_epsg31370": lambert,
"width_m": width_m,
"height_m": height_m,
}
def validate_product(items: list[dict[str, Any]]) -> dict[str, Any]:
matches = [item for item in items if item.get("key") == PRODUCT_KEY]
if len(matches) != 1:
raise RuntimeError("Orthophoto product registry does not contain exactly one most_recent product")
product = matches[0]
expected = {
"temporal_granularity": "snapshot",
"native_resolution_m": 0.15,
"supports_detection": True,
"color_mode": "rgb",
"catalog_url": CATALOG_URL,
}
if any(product.get(key) != value for key, value in expected.items()):
raise RuntimeError("Current orthophoto product variant no longer matches the governed registry contract")
return {"key": PRODUCT_KEY, **expected, "display_name": product.get("display_name")}
def catalog_decision(item: dict[str, Any]) -> dict[str, Any]:
remote_version = str(item.get("remote_version") or "")
remote_match = EDITION_PATTERN.fullmatch(remote_version)
if (
item.get("source_name") != SOURCE_NAME
or item.get("status") != "available"
or item.get("reachable") is not True
or set(item.get("matched_layers") or []) != {"Ortho", "Vliegdagcontour"}
or item.get("missing_layers") not in (None, [])
or item.get("metadata_identifier") != METADATA_IDENTIFIER
or not SHA256_PATTERN.fullmatch(str(item.get("capabilities_sha256") or ""))
or not remote_match
or remote_version not in str(item.get("remote_title") or "")
):
raise RuntimeError("Canonical catalog report does not contain one complete governed orthophoto release")
local_version = str(item.get("local_source_version") or "")
local_match = EDITION_PATTERN.fullmatch(local_version)
if not local_version:
status = "not_loaded"
expected_comparison = "no_local_data"
elif not local_match:
status = "blocked_local_version"
expected_comparison = "not_comparable"
else:
local_order = tuple(int(value) for value in local_match.groups())
remote_order = tuple(int(value) for value in remote_match.groups())
if local_order == remote_order:
status = "current"
expected_comparison = "same"
elif local_order < remote_order:
status = "update_available"
expected_comparison = "different"
else:
status = "blocked_remote_older"
expected_comparison = "different"
if item.get("comparison_status") != expected_comparison:
raise RuntimeError("Canonical catalog local/remote comparison is internally inconsistent")
return {
"status": status,
"remote_edition": remote_version,
"remote_year": int(remote_match.group(1)),
"local_source_version": local_version or None,
"comparison_status": expected_comparison,
"metadata_identifier": METADATA_IDENTIFIER,
"metadata_url": item.get("metadata_url"),
"remote_title": item.get("remote_title"),
"remote_modified_at": item.get("remote_modified_at"),
"remote_published_at": item.get("remote_published_at"),
"catalog_checked_at": item.get("checked_at"),
"capabilities_url": item.get("endpoint_url"),
"capabilities_sha256": item.get("capabilities_sha256"),
}
def _sample_grid(selection: dict[str, Any]) -> list[dict[str, Any]]:
count_x = max(1, math.ceil(selection["width_m"] / MAX_SAMPLE_SPACING_M))
count_y = max(1, math.ceil(selection["height_m"] / MAX_SAMPLE_SPACING_M))
if count_x * count_y > MAX_SAMPLE_COUNT:
raise RuntimeError("Deterministic flight-day grid exceeds the fixed preflight limit")
min_x, min_y, max_x, max_y = selection["bbox_epsg31370"]
samples: list[dict[str, Any]] = []
for row in range(count_y):
fraction_y = (row + 0.5) / count_y
for column in range(count_x):
fraction_x = (column + 0.5) / count_x
samples.append(
{
"column": column,
"row": row,
"i": min(FEATURE_INFO_SIZE - 1, int(fraction_x * FEATURE_INFO_SIZE)),
"j": min(FEATURE_INFO_SIZE - 1, int((1.0 - fraction_y) * FEATURE_INFO_SIZE)),
"x": min_x + fraction_x * (max_x - min_x),
"y": min_y + fraction_y * (max_y - min_y),
}
)
return samples
def _flight_sample(
base_url: str,
selection: dict[str, Any],
sample: dict[str, Any],
*,
timeout: int,
opener: Callable[..., Any] | None,
) -> dict[str, Any]:
params = {
"SERVICE": "WMS",
"VERSION": "1.3.0",
"REQUEST": "GetFeatureInfo",
"LAYERS": "Vliegdagcontour",
"QUERY_LAYERS": "Vliegdagcontour",
"STYLES": "",
"CRS": "EPSG:31370",
"BBOX": ",".join(f"{value:.3f}" for value in selection["bbox_epsg31370"]),
"WIDTH": str(FEATURE_INFO_SIZE),
"HEIGHT": str(FEATURE_INFO_SIZE),
"I": str(sample["i"]),
"J": str(sample["j"]),
"INFO_FORMAT": "application/geo+json",
"FEATURE_COUNT": "10",
"FORMAT": "image/png",
}
body, content_type, _ = _bounded_get(
f"{base_url}?{urlencode(params)}",
timeout=timeout,
max_bytes=MAX_FEATURE_INFO_BYTES,
accept="application/geo+json",
opener=opener,
)
if content_type not in {"application/geo+json", "application/json"}:
raise RuntimeError("Vliegdagcontour feature-info response is not GeoJSON")
try:
payload = json.loads(body)
except (UnicodeDecodeError, json.JSONDecodeError) as exc:
raise RuntimeError("Vliegdagcontour feature-info response is invalid JSON") from exc
features = payload.get("features") if isinstance(payload, dict) else None
if not isinstance(payload, dict) or payload.get("type") != "FeatureCollection" or not isinstance(features, list) or not features:
raise RuntimeError("Vliegdagcontour has no coverage at one deterministic selection sample")
dates: set[str] = set()
feature_ids: set[str] = set()
years: set[int] = set()
for feature in features:
if not isinstance(feature, dict) or feature.get("layerName") != "Vliegdagcontour":
raise RuntimeError("Vliegdagcontour feature-info returned an unexpected layer")
properties = feature.get("properties")
if not isinstance(properties, dict):
raise RuntimeError("Vliegdagcontour feature-info has no properties")
date_value = str(properties.get("OpnDatum") or "").strip()
match = FLIGHT_YEAR_PATTERN.search(date_value)
if not match:
raise RuntimeError("Vliegdagcontour feature-info has no parseable acquisition year")
dates.add(date_value)
years.add(int(match.group(1)))
if properties.get("FID") is not None:
feature_ids.add(str(properties["FID"]))
return {
"column": sample["column"],
"row": sample["row"],
"x": round(sample["x"], 3),
"y": round(sample["y"], 3),
"dates": sorted(dates),
"years": sorted(years),
"feature_ids": sorted(feature_ids),
"response_sha256": sha256(body).hexdigest(),
}
def verify_flight_coverage(
base_url: str,
selection: dict[str, Any],
*,
timeout: int,
opener: Callable[..., Any] | None = None,
) -> dict[str, Any]:
samples = _sample_grid(selection)
worker_count = min(8, len(samples))
with ThreadPoolExecutor(max_workers=worker_count) as executor:
evidence = list(
executor.map(
lambda sample: _flight_sample(
base_url,
selection,
sample,
timeout=timeout,
opener=opener,
),
samples,
)
)
years = sorted({year for item in evidence for year in item["years"]})
dates = sorted({date for item in evidence for date in item["dates"]})
feature_ids = sorted({feature_id for item in evidence for feature_id in item["feature_ids"]})
canonical = json.dumps(evidence, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8")
count_x = len({item["column"] for item in evidence})
count_y = len({item["row"] for item in evidence})
return {
"status": "passed",
"mode": "official_queryable_flight_day_grid",
"sample_count": len(evidence),
"grid_columns": count_x,
"grid_rows": count_y,
"maximum_sample_spacing_m": MAX_SAMPLE_SPACING_M,
"covered_sample_count": len(evidence),
"sample_coverage_ratio": 1.0,
"flight_dates": dates,
"flight_years": years,
"feature_ids": feature_ids,
"sample_evidence_sha256": sha256(canonical).hexdigest(),
"claim_boundary": (
"Every deterministic grid sample is covered by the official queryable flight-day layer. "
"The WMS exposes no vector geometry, so this is bounded point evidence and not a polygon-union proof."
),
}
def run_preflight(
args: argparse.Namespace,
*,
loader: Callable[[str, str, int], dict[str, Any]] = api_data,
opener: Callable[..., Any] | None = None,
) -> dict[str, Any]:
validate_project_scope(args, loader)
products = loader(
args.api_url,
f"projects/{args.project_id}/datasets/orthophoto/products",
args.api_timeout,
)
product = validate_product(products.get("items") or [])
refresh = "true" if args.refresh_catalog else "false"
report = loader(
args.api_url,
f"projects/{args.project_id}/datasets/source-catalog-probes?refresh={refresh}",
args.api_timeout,
)
matches = [item for item in report.get("items") or [] if item.get("source_name") == SOURCE_NAME]
if len(matches) != 1:
raise RuntimeError("Source catalog report did not contain exactly one orthophoto contract")
release = catalog_decision(matches[0])
capabilities_url = str(release["capabilities_url"] or "")
base_url = _validate_wms_url(capabilities_url, request_name="GetCapabilities")
capabilities_body, content_type, _ = _bounded_get(
capabilities_url,
timeout=args.wms_timeout,
max_bytes=MAX_CAPABILITIES_BYTES,
accept="application/xml,text/xml",
opener=opener,
)
if content_type and "xml" not in content_type and "text" not in content_type:
raise RuntimeError("Official WMS capabilities response is not XML")
capabilities = parse_capabilities(capabilities_body, release["capabilities_sha256"])
coverage_description_url = (
f"https://{WMS_HOST}{WCS_PATH}?"
+ urlencode(
{
"SERVICE": "WCS",
"VERSION": "2.0.1",
"REQUEST": "DescribeCoverage",
"COVERAGEID": "Ortho",
}
)
)
_validate_wcs_url(coverage_description_url, request_name="DescribeCoverage")
coverage_body, coverage_content_type, _ = _bounded_get(
coverage_description_url,
timeout=args.wms_timeout,
max_bytes=MAX_COVERAGE_DESCRIPTION_BYTES,
accept="application/xml,text/xml",
opener=opener,
validator=_validate_wcs_url,
)
if coverage_content_type and "xml" not in coverage_content_type and "text" not in coverage_content_type:
raise RuntimeError("Official WCS coverage description response is not XML")
coverage_description = parse_coverage_description(coverage_body)
selection = validate_bbox(list(args.bbox), coverage_description["extent_epsg31370"])
validate_bbox(list(args.bbox), capabilities["extent_epsg31370"])
coverage = verify_flight_coverage(
base_url,
selection,
timeout=args.wms_timeout,
opener=opener,
)
flight_year_matches = coverage["flight_years"] == [release["remote_year"]]
release_actionable = release["status"] in {"not_loaded", "update_available"}
staging_permitted = release_actionable and flight_year_matches
if release["status"] == "blocked_local_version":
next_action = "establish_official_local_edition_before_staging"
elif release["status"] == "current":
next_action = "none_current"
elif release["status"] == "blocked_remote_older":
next_action = "investigate_catalog_regression"
elif not flight_year_matches:
next_action = "split_or_review_selection_flight_years"
else:
next_action = "governed_pixel_stage"
return {
"schema_version": 1,
"status": "passed",
"generated_at": datetime.now(timezone.utc).isoformat(),
"project_id": args.project_id,
"scope": args.scope,
"product": product,
"release": release,
"capabilities": capabilities,
"coverage_domain": {
**coverage_description,
"selected_area_fully_inside_domain": True,
"pixel_data_requested": False,
},
"selection": {
"bbox_epsg4326": selection["bbox_epsg4326"],
"bbox_epsg31370": [round(value, 3) for value in selection["bbox_epsg31370"]],
"width_m": round(selection["width_m"], 3),
"height_m": round(selection["height_m"], 3),
},
"flight_day_coverage": coverage,
"flight_year_matches_release": flight_year_matches,
"staging_permitted": staging_permitted,
"next_action": next_action,
"pixel_requests_performed": 0,
"datasets_mutated": 0,
"automatic_staging": False,
"automatic_import": False,
}
def main() -> int:
try:
report = run_preflight(parse_args())
print(json.dumps(report, ensure_ascii=False, indent=2))
return 0
except (RuntimeError, ValueError) as exc:
print(json.dumps({"status": "error", "error": str(exc)}, ensure_ascii=False, indent=2), file=sys.stderr)
return 1
if __name__ == "__main__":
raise SystemExit(main())
+1
View File
@@ -55,6 +55,7 @@ ${PYTHON_BIN} -m py_compile scripts/provision_mol_bwk_natura2000.py
${PYTHON_BIN} -m py_compile scripts/provision_regional_bwk_natura2000.py
${PYTHON_BIN} -m py_compile scripts/provision_agricultural_parcel_history.py
${PYTHON_BIN} -m py_compile scripts/manage_alz_agriculture_release.py
${PYTHON_BIN} -m py_compile scripts/orthophoto_release_preflight.py
${PYTHON_BIN} -m py_compile scripts/provision_buildings_addresses_register.py
${PYTHON_BIN} -m py_compile scripts/provision_mol_dhmv.py
${PYTHON_BIN} -m py_compile scripts/provision_regional_dhmv.py