495 lines
18 KiB
Python
495 lines
18 KiB
Python
"""Prepare explicit real operator samples for GeoIntel detection QA.
|
|
|
|
This script downloads small orthophoto and GRB building reference pairs from
|
|
Digitaal Vlaanderen for documented Kempen AOIs. It is an operator/runtime
|
|
helper, not an application provider integration: no GeoIntel API route calls it
|
|
and no production data is fetched silently by the app.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
from dataclasses import dataclass, replace
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
WMS_URL = "https://geo.api.vlaanderen.be/omwrgbmrvl/wms"
|
|
GRB_GBG_URL = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items"
|
|
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data")
|
|
requests: Any = None
|
|
rasterio: Any = None
|
|
Transformer: Any = None
|
|
MemoryFile: Any = None
|
|
from_bounds: Any = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class OperatorSample:
|
|
slug: str
|
|
display_name: str
|
|
center_lon: float
|
|
center_lat: float
|
|
half_size_m: float = 250.0
|
|
width: int = 512
|
|
height: int = 512
|
|
sample_role: str = "reference"
|
|
allow_empty_reference: bool = False
|
|
|
|
|
|
SAMPLES: dict[str, OperatorSample] = {
|
|
"geel": OperatorSample(
|
|
slug="geel",
|
|
display_name="Geel center",
|
|
center_lon=4.991,
|
|
center_lat=51.162,
|
|
),
|
|
"mol": OperatorSample(
|
|
slug="mol",
|
|
display_name="Mol center",
|
|
center_lon=5.1167,
|
|
center_lat=51.1919,
|
|
),
|
|
"turnhout": OperatorSample(
|
|
slug="turnhout",
|
|
display_name="Turnhout center",
|
|
center_lon=4.9488,
|
|
center_lat=51.3225,
|
|
),
|
|
"herentals": OperatorSample(
|
|
slug="herentals",
|
|
display_name="Herentals center",
|
|
center_lon=4.8339,
|
|
center_lat=51.1766,
|
|
half_size_m=220.0,
|
|
),
|
|
"balen": OperatorSample(
|
|
slug="balen",
|
|
display_name="Balen center",
|
|
center_lon=5.1703,
|
|
center_lat=51.1688,
|
|
half_size_m=220.0,
|
|
),
|
|
"retie": OperatorSample(
|
|
slug="retie",
|
|
display_name="Retie center",
|
|
center_lon=5.0827,
|
|
center_lat=51.2665,
|
|
half_size_m=220.0,
|
|
),
|
|
"westerlo": OperatorSample(
|
|
slug="westerlo",
|
|
display_name="Westerlo center",
|
|
center_lon=4.9158,
|
|
center_lat=51.0909,
|
|
half_size_m=220.0,
|
|
),
|
|
"postel_bos": OperatorSample(
|
|
slug="postel_bos",
|
|
display_name="Postel forest background candidate",
|
|
center_lon=5.16,
|
|
center_lat=51.305,
|
|
half_size_m=260.0,
|
|
sample_role="background_candidate",
|
|
allow_empty_reference=True,
|
|
),
|
|
"lommel_heide": OperatorSample(
|
|
slug="lommel_heide",
|
|
display_name="Lommel forest background candidate",
|
|
center_lon=5.287,
|
|
center_lat=51.249,
|
|
half_size_m=260.0,
|
|
sample_role="background_candidate",
|
|
allow_empty_reference=True,
|
|
),
|
|
"kasterlee_bos": OperatorSample(
|
|
slug="kasterlee_bos",
|
|
display_name="Kasterlee forest background candidate",
|
|
center_lon=4.965,
|
|
center_lat=51.273,
|
|
half_size_m=260.0,
|
|
sample_role="background_candidate",
|
|
allow_empty_reference=True,
|
|
),
|
|
"dessel_heide": OperatorSample(
|
|
slug="dessel_heide",
|
|
display_name="Dessel heath background candidate",
|
|
center_lon=5.092,
|
|
center_lat=51.235,
|
|
half_size_m=260.0,
|
|
sample_role="background_candidate",
|
|
allow_empty_reference=True,
|
|
),
|
|
"ravels_bos": OperatorSample(
|
|
slug="ravels_bos",
|
|
display_name="Ravels forest background candidate",
|
|
center_lon=4.977,
|
|
center_lat=51.384,
|
|
half_size_m=260.0,
|
|
sample_role="background_candidate",
|
|
allow_empty_reference=True,
|
|
),
|
|
"meerhout_bos": OperatorSample(
|
|
slug="meerhout_bos",
|
|
display_name="Meerhout forest background candidate",
|
|
center_lon=5.069,
|
|
center_lat=51.115,
|
|
half_size_m=260.0,
|
|
sample_role="background_candidate",
|
|
allow_empty_reference=True,
|
|
),
|
|
"geel_bel": OperatorSample(
|
|
slug="geel_bel",
|
|
display_name="Geel-Bel rural background candidate",
|
|
center_lon=5.046,
|
|
center_lat=51.137,
|
|
half_size_m=260.0,
|
|
sample_role="background_candidate",
|
|
allow_empty_reference=True,
|
|
),
|
|
"arendonk_heide": OperatorSample(
|
|
slug="arendonk_heide",
|
|
display_name="Arendonk heath background candidate",
|
|
center_lon=5.238,
|
|
center_lat=51.334,
|
|
half_size_m=260.0,
|
|
sample_role="background_candidate",
|
|
allow_empty_reference=True,
|
|
),
|
|
"herenthout_bos": OperatorSample(
|
|
slug="herenthout_bos",
|
|
display_name="Herenthout forest background candidate",
|
|
center_lon=4.781,
|
|
center_lat=51.143,
|
|
half_size_m=260.0,
|
|
sample_role="background_candidate",
|
|
allow_empty_reference=True,
|
|
),
|
|
}
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description="Prepare real Digitaal Vlaanderen orthophoto/GRB building samples for GeoIntel operator QA.",
|
|
)
|
|
parser.add_argument(
|
|
"--output-dir",
|
|
type=Path,
|
|
default=Path(os.environ.get("OPERATOR_DATA_DIR", DEFAULT_OUTPUT_DIR)),
|
|
help="Directory for generated GeoTIFF, GeoJSON and manifest files.",
|
|
)
|
|
parser.add_argument(
|
|
"--samples",
|
|
default=",".join(SAMPLES),
|
|
help="Comma/space separated sample slugs to prepare. Defaults to all documented samples.",
|
|
)
|
|
parser.add_argument(
|
|
"--force",
|
|
action="store_true",
|
|
help="Refetch and overwrite sample files. By default existing raster/reference pairs are reused.",
|
|
)
|
|
parser.add_argument(
|
|
"--manifest-name",
|
|
default="operator_samples_manifest.json",
|
|
help="Manifest filename written inside output-dir.",
|
|
)
|
|
parser.add_argument(
|
|
"--width",
|
|
type=int,
|
|
default=int(os.environ.get("OPERATOR_SAMPLE_WIDTH", "512")),
|
|
help="Orthophoto WMS output width in pixels. Use larger values for operator training datasets.",
|
|
)
|
|
parser.add_argument(
|
|
"--height",
|
|
type=int,
|
|
default=int(os.environ.get("OPERATOR_SAMPLE_HEIGHT", "512")),
|
|
help="Orthophoto WMS output height in pixels. Use larger values for operator training datasets.",
|
|
)
|
|
parser.add_argument(
|
|
"--half-size-scale",
|
|
type=float,
|
|
default=float(os.environ.get("OPERATOR_SAMPLE_HALF_SIZE_SCALE", "1")),
|
|
help="Multiplier applied to each documented AOI half-size in meters.",
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
def ensure_gis_dependencies() -> None:
|
|
global MemoryFile, Transformer, from_bounds, rasterio, requests
|
|
try:
|
|
import requests as requests_module
|
|
import rasterio as rasterio_module
|
|
from pyproj import Transformer as transformer_class
|
|
from rasterio.io import MemoryFile as memory_file_class
|
|
from rasterio.transform import from_bounds as from_bounds_function
|
|
except Exception as exc: # pragma: no cover - exercised only in runtime envs.
|
|
raise SystemExit(
|
|
"prepare_operator_real_data_samples.py requires requests, rasterio and pyproj. "
|
|
"Run it inside the GeoIntel all-in-one container or an equivalent GIS Python environment."
|
|
) from exc
|
|
requests = requests_module
|
|
rasterio = rasterio_module
|
|
Transformer = transformer_class
|
|
MemoryFile = memory_file_class
|
|
from_bounds = from_bounds_function
|
|
|
|
|
|
def selected_samples(raw: str) -> list[OperatorSample]:
|
|
slugs = [value.strip().lower() for value in raw.replace(",", " ").split() if value.strip()]
|
|
if not slugs:
|
|
raise SystemExit("--samples must include at least one sample slug")
|
|
unknown = [slug for slug in slugs if slug not in SAMPLES]
|
|
if unknown:
|
|
raise SystemExit(f"Unknown sample slug(s): {', '.join(unknown)}. Known: {', '.join(SAMPLES)}")
|
|
return [SAMPLES[slug] for slug in slugs]
|
|
|
|
|
|
def apply_sample_overrides(
|
|
sample: OperatorSample,
|
|
*,
|
|
width: int,
|
|
height: int,
|
|
half_size_scale: float,
|
|
) -> OperatorSample:
|
|
if width <= 0 or height <= 0:
|
|
raise SystemExit("--width and --height must be positive integers")
|
|
if half_size_scale <= 0:
|
|
raise SystemExit("--half-size-scale must be greater than zero")
|
|
return replace(
|
|
sample,
|
|
width=width,
|
|
height=height,
|
|
half_size_m=sample.half_size_m * half_size_scale,
|
|
)
|
|
|
|
|
|
def sample_bounds(sample: OperatorSample) -> tuple[tuple[float, float, float, float], list[float]]:
|
|
lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
|
wgs84 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
|
|
center_x, center_y = lambert.transform(sample.center_lon, sample.center_lat)
|
|
minx = center_x - sample.half_size_m
|
|
miny = center_y - sample.half_size_m
|
|
maxx = center_x + sample.half_size_m
|
|
maxy = center_y + sample.half_size_m
|
|
corners = [
|
|
wgs84.transform(x, y)
|
|
for x, y in ((minx, miny), (minx, maxy), (maxx, miny), (maxx, maxy))
|
|
]
|
|
lon_values = [point[0] for point in corners]
|
|
lat_values = [point[1] for point in corners]
|
|
return (minx, miny, maxx, maxy), [
|
|
min(lon_values),
|
|
min(lat_values),
|
|
max(lon_values),
|
|
max(lat_values),
|
|
]
|
|
|
|
|
|
def prepared_url(url: str, params: dict[str, str]) -> str:
|
|
return requests.Request("GET", url, params=params).prepare().url
|
|
|
|
|
|
def raster_summary(path: Path) -> dict[str, Any]:
|
|
with rasterio.open(path) as ds:
|
|
return {
|
|
"path": str(path),
|
|
"crs": str(ds.crs),
|
|
"bounds": list(ds.bounds),
|
|
"width": ds.width,
|
|
"height": ds.height,
|
|
"count": ds.count,
|
|
"dtypes": list(ds.dtypes),
|
|
}
|
|
|
|
|
|
def geojson_feature_count(path: Path) -> int:
|
|
payload = json.loads(path.read_text(encoding="utf-8-sig"))
|
|
return len(payload.get("features") or [])
|
|
|
|
|
|
def sample_artifact_paths(sample: OperatorSample, output_dir: Path) -> tuple[Path, Path]:
|
|
ortho_path = output_dir / f"{sample.slug}_orthophoto_wms_{sample.width}.tif"
|
|
reference_path = output_dir / f"{sample.slug}_grb_gbg_buildings.geojson"
|
|
return ortho_path, reference_path
|
|
|
|
|
|
def fetch_orthophoto(sample: OperatorSample, ortho_path: Path, lambert_bbox: tuple[float, float, float, float]) -> str:
|
|
minx, miny, maxx, maxy = lambert_bbox
|
|
wms_params = {
|
|
"SERVICE": "WMS",
|
|
"VERSION": "1.3.0",
|
|
"REQUEST": "GetMap",
|
|
"LAYERS": "Ortho",
|
|
"STYLES": "",
|
|
"FORMAT": "image/tiff",
|
|
"CRS": "EPSG:31370",
|
|
"BBOX": f"{minx},{miny},{maxx},{maxy}",
|
|
"WIDTH": str(sample.width),
|
|
"HEIGHT": str(sample.height),
|
|
}
|
|
response = requests.get(WMS_URL, params=wms_params, timeout=120)
|
|
response.raise_for_status()
|
|
content_type = response.headers.get("content-type", "")
|
|
if "image" not in content_type.lower() and "tiff" not in content_type.lower():
|
|
raise SystemExit(f"Orthophoto WMS did not return an image for {sample.slug}: {content_type}")
|
|
|
|
with MemoryFile(response.content) as memfile:
|
|
with memfile.open() as src:
|
|
image = src.read()
|
|
profile = src.profile.copy()
|
|
profile.update(
|
|
driver="GTiff",
|
|
width=src.width,
|
|
height=src.height,
|
|
count=src.count,
|
|
dtype=src.dtypes[0],
|
|
crs="EPSG:31370",
|
|
transform=from_bounds(minx, miny, maxx, maxy, src.width, src.height),
|
|
compress="deflate",
|
|
tiled=False,
|
|
)
|
|
with rasterio.open(ortho_path, "w", **profile) as dst:
|
|
dst.write(image)
|
|
dst.update_tags(
|
|
source="Digitaal Vlaanderen OMWRGBMRVL WMS Ortho layer",
|
|
source_url=prepared_url(WMS_URL, wms_params),
|
|
attribution="Bron: Orthofotomozaiek Vlaanderen, Digitaal Vlaanderen",
|
|
aoi=f"{sample.display_name} sample AOI for GeoIntel operator validation",
|
|
)
|
|
return prepared_url(WMS_URL, wms_params)
|
|
|
|
|
|
def fetch_reference(sample: OperatorSample, reference_path: Path, geo_bbox: list[float]) -> tuple[str, int]:
|
|
ogc_params = {
|
|
"f": "application/geo+json",
|
|
"limit": "1000",
|
|
"bbox": ",".join(f"{value:.8f}" for value in geo_bbox),
|
|
}
|
|
response = requests.get(GRB_GBG_URL, params=ogc_params, timeout=120)
|
|
response.raise_for_status()
|
|
reference = response.json()
|
|
features = reference.get("features") or []
|
|
if not features and not sample.allow_empty_reference:
|
|
raise SystemExit(f"GRB GBG returned no building features for {sample.slug} bbox {geo_bbox}")
|
|
|
|
reference["name"] = f"GRB GBG buildings - {sample.display_name} sample AOI"
|
|
reference["source"] = "Digitaal Vlaanderen GRB OGC API Features collection GBG"
|
|
reference["source_url"] = prepared_url(GRB_GBG_URL, ogc_params)
|
|
reference["attribution"] = "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen"
|
|
reference["bbox"] = geo_bbox
|
|
reference["sample_slug"] = sample.slug
|
|
reference["sample_role"] = sample.sample_role
|
|
reference["allow_empty_reference"] = sample.allow_empty_reference
|
|
for feature in features:
|
|
props = feature.setdefault("properties", {})
|
|
props.setdefault("source_name", "grb")
|
|
props.setdefault("reference_layer_name", "buildings")
|
|
props.setdefault("sample_slug", sample.slug)
|
|
props.setdefault("sample_role", sample.sample_role)
|
|
|
|
reference_path.write_text(json.dumps(reference, ensure_ascii=False), encoding="utf-8")
|
|
return prepared_url(GRB_GBG_URL, ogc_params), len(features)
|
|
|
|
|
|
def prepare_sample(sample: OperatorSample, output_dir: Path, force: bool) -> dict[str, Any]:
|
|
ortho_path, reference_path = sample_artifact_paths(sample, output_dir)
|
|
lambert_bbox, geo_bbox = sample_bounds(sample)
|
|
skip_existing = ortho_path.exists() and reference_path.exists() and not force
|
|
|
|
source_urls: dict[str, str | None] = {"orthophoto": None, "reference": None}
|
|
if not skip_existing:
|
|
source_urls["orthophoto"] = fetch_orthophoto(sample, ortho_path, lambert_bbox)
|
|
source_urls["reference"], reference_feature_count = fetch_reference(sample, reference_path, geo_bbox)
|
|
else:
|
|
reference_feature_count = geojson_feature_count(reference_path)
|
|
|
|
return {
|
|
"sample_slug": sample.slug,
|
|
"display_name": sample.display_name,
|
|
"center_lon": sample.center_lon,
|
|
"center_lat": sample.center_lat,
|
|
"half_size_m": sample.half_size_m,
|
|
"width": sample.width,
|
|
"height": sample.height,
|
|
"sample_role": sample.sample_role,
|
|
"allow_empty_reference": sample.allow_empty_reference,
|
|
"raster_path": str(ortho_path),
|
|
"reference_path": str(reference_path),
|
|
"reference_feature_count": reference_feature_count,
|
|
"raster": raster_summary(ortho_path),
|
|
"wgs84_bbox": geo_bbox,
|
|
"epsg31370_bbox": list(lambert_bbox),
|
|
"skip_existing": skip_existing,
|
|
"source_urls": source_urls,
|
|
"attribution": {
|
|
"orthophoto": "Bron: Orthofotomozaiek Vlaanderen, Digitaal Vlaanderen",
|
|
"reference": "Bron: Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen",
|
|
},
|
|
}
|
|
|
|
|
|
def write_readme(output_dir: Path, samples: list[dict[str, Any]]) -> None:
|
|
lines = [
|
|
"# GeoIntel operator real-data samples",
|
|
"",
|
|
"Generated for runtime validation, not committed to the GeoIntel repository.",
|
|
"",
|
|
"Sources:",
|
|
"- Orthophoto rasters: Digitaal Vlaanderen OMWRGBMRVL WMS `Ortho` layer.",
|
|
"- Reference buildings: Digitaal Vlaanderen GRB OGC API Features `GBG` collection.",
|
|
"- Attribution: Bron: Orthofotomozaiek Vlaanderen / Grootschalig Referentie Bestand Vlaanderen, Digitaal Vlaanderen.",
|
|
"",
|
|
"Samples:",
|
|
]
|
|
for sample in samples:
|
|
lines.append(
|
|
f"- `{sample['sample_slug']}`: `{Path(sample['raster_path']).name}` and "
|
|
f"`{Path(sample['reference_path']).name}`, "
|
|
f"{sample['reference_feature_count']} reference features, role `{sample['sample_role']}`."
|
|
)
|
|
lines.append("")
|
|
lines.append("Purpose: configured-YOLO detection + persisted QA/QC validation with operator-provided files.")
|
|
(output_dir / "README.md").write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
ensure_gis_dependencies()
|
|
output_dir: Path = args.output_dir
|
|
output_dir.mkdir(parents=True, exist_ok=True)
|
|
samples = [
|
|
prepare_sample(
|
|
apply_sample_overrides(
|
|
sample,
|
|
width=args.width,
|
|
height=args.height,
|
|
half_size_scale=args.half_size_scale,
|
|
),
|
|
output_dir,
|
|
force=args.force,
|
|
)
|
|
for sample in selected_samples(args.samples)
|
|
]
|
|
write_readme(output_dir, samples)
|
|
|
|
manifest = {
|
|
"schema_version": 1,
|
|
"description": "GeoIntel operator real-data samples for configured-YOLO QA validation.",
|
|
"output_dir": str(output_dir),
|
|
"sample_width": args.width,
|
|
"sample_height": args.height,
|
|
"half_size_scale": args.half_size_scale,
|
|
"samples": samples,
|
|
}
|
|
manifest_path = output_dir / args.manifest_name
|
|
manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True), encoding="utf-8")
|
|
print(json.dumps({"status": "ok", "manifest_path": str(manifest_path), "samples": samples}, indent=2, ensure_ascii=False))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|