Build governed Belgian training corpus pipeline
This commit is contained in:
Jens
@@ -0,0 +1,218 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Normalize governed Belgian building references for detector training.
|
||||
|
||||
The script never asserts semantic parity between providers. It emits one
|
||||
canonical training class while retaining provider-native identifiers,
|
||||
properties and an explicit accept/reject decision for every source feature.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import rasterio
|
||||
from pyproj import Transformer
|
||||
from shapely.geometry import mapping, shape
|
||||
from shapely.ops import transform as shapely_transform
|
||||
from shapely.validation import make_valid
|
||||
|
||||
SUPPORTED_SOURCES = {"grb", "spw_picc", "urbis"}
|
||||
EXCLUDED_TOKENS = {
|
||||
"canopy": ("canopy", "afdak", "auvent"),
|
||||
"ruin": ("ruin", "ruine", "ruïne"),
|
||||
"underground": ("underground", "ondergronds", "souterrain"),
|
||||
}
|
||||
|
||||
|
||||
def _feature_id(feature: dict[str, Any], index: int) -> str:
|
||||
properties = feature.get("properties") or {}
|
||||
for key in ("source_feature_id", "OBJECTID", "INSPIRE_ID", "id", "gml_id"):
|
||||
value = properties.get(key)
|
||||
if value not in (None, ""):
|
||||
return str(value)
|
||||
if feature.get("id") not in (None, ""):
|
||||
return str(feature["id"])
|
||||
return f"row-{index}"
|
||||
|
||||
|
||||
def _source_class(properties: dict[str, Any]) -> str | None:
|
||||
for key in ("source_class", "TYPE", "type", "OBJTYPE", "nature", "NATURE", "class"):
|
||||
value = properties.get(key)
|
||||
if value not in (None, ""):
|
||||
return str(value)
|
||||
return None
|
||||
|
||||
|
||||
def _semantic_exclusion(properties: dict[str, Any]) -> str | None:
|
||||
haystack = " ".join(str(value).lower() for value in properties.values() if value is not None)
|
||||
for reason, tokens in EXCLUDED_TOKENS.items():
|
||||
if any(token in haystack for token in tokens):
|
||||
return f"excluded_{reason}"
|
||||
return None
|
||||
|
||||
|
||||
def _polygonal(geometry: Any) -> Any | None:
|
||||
if geometry.geom_type in {"Polygon", "MultiPolygon"}:
|
||||
return geometry
|
||||
if geometry.geom_type == "GeometryCollection":
|
||||
polygons = [part for part in geometry.geoms if part.geom_type in {"Polygon", "MultiPolygon"}]
|
||||
if not polygons:
|
||||
return None
|
||||
from shapely.ops import unary_union
|
||||
|
||||
return unary_union(polygons)
|
||||
return None
|
||||
|
||||
|
||||
def normalize(
|
||||
*,
|
||||
reference_path: Path,
|
||||
raster_path: Path,
|
||||
source_name: str,
|
||||
min_label_px: float,
|
||||
imagery_observed_at: str | None,
|
||||
reference_observed_at: str | None,
|
||||
) -> tuple[dict[str, Any], dict[str, Any]]:
|
||||
if source_name not in SUPPORTED_SOURCES:
|
||||
raise SystemExit(f"Unsupported governed building source: {source_name}")
|
||||
payload = json.loads(reference_path.read_text(encoding="utf-8-sig"))
|
||||
if payload.get("type") != "FeatureCollection" or not isinstance(payload.get("features"), list):
|
||||
raise SystemExit("Reference must be a GeoJSON FeatureCollection")
|
||||
|
||||
accepted: list[dict[str, Any]] = []
|
||||
decisions: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
counts: Counter[str] = Counter()
|
||||
with rasterio.open(raster_path) as raster:
|
||||
if raster.crs is None:
|
||||
raise SystemExit("Raster CRS is required")
|
||||
transformer = Transformer.from_crs("EPSG:4326", raster.crs, always_xy=True)
|
||||
for index, feature in enumerate(payload["features"]):
|
||||
properties = dict(feature.get("properties") or {})
|
||||
source_feature_id = _feature_id(feature, index)
|
||||
decision = {
|
||||
"source_name": source_name,
|
||||
"source_feature_id": source_feature_id,
|
||||
"source_class": _source_class(properties),
|
||||
"accepted": False,
|
||||
"reason": None,
|
||||
"geometry_repaired": False,
|
||||
}
|
||||
try:
|
||||
geometry = shape(feature.get("geometry"))
|
||||
except Exception:
|
||||
decision["reason"] = "invalid_geometry_unreadable"
|
||||
decisions.append(decision)
|
||||
counts[decision["reason"]] += 1
|
||||
continue
|
||||
if not geometry.is_valid:
|
||||
geometry = make_valid(geometry)
|
||||
decision["geometry_repaired"] = True
|
||||
geometry = _polygonal(geometry)
|
||||
if geometry is None or geometry.is_empty or not geometry.is_valid:
|
||||
decision["reason"] = "invalid_geometry_unrepairable"
|
||||
elif (reason := _semantic_exclusion(properties)) is not None:
|
||||
decision["reason"] = reason
|
||||
else:
|
||||
metric = shapely_transform(transformer.transform, geometry)
|
||||
min_x, min_y, max_x, max_y = metric.bounds
|
||||
pixel_width = (max_x - min_x) / abs(float(raster.transform.a))
|
||||
pixel_height = (max_y - min_y) / abs(float(raster.transform.e))
|
||||
decision["pixel_width"] = pixel_width
|
||||
decision["pixel_height"] = pixel_height
|
||||
if pixel_width < min_label_px or pixel_height < min_label_px:
|
||||
decision["reason"] = "below_resolvable_pixel_size"
|
||||
else:
|
||||
digest = hashlib.sha256(geometry.normalize().wkb).hexdigest()
|
||||
if digest in seen:
|
||||
decision["reason"] = "duplicate_geometry"
|
||||
else:
|
||||
seen.add(digest)
|
||||
decision["accepted"] = True
|
||||
decision["reason"] = "accepted"
|
||||
normalized_properties = dict(properties)
|
||||
normalized_properties.update(
|
||||
{
|
||||
"canonical_class": "building",
|
||||
"source_name": source_name,
|
||||
"reference_layer_name": "buildings",
|
||||
"source_feature_id": source_feature_id,
|
||||
"source_class": decision["source_class"],
|
||||
"label_decision": "accepted",
|
||||
"geometry_repaired": decision["geometry_repaired"],
|
||||
}
|
||||
)
|
||||
accepted.append(
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": f"{source_name}:{source_feature_id}",
|
||||
"properties": normalized_properties,
|
||||
"geometry": mapping(geometry),
|
||||
}
|
||||
)
|
||||
decisions.append(decision)
|
||||
counts[str(decision["reason"])] += 1
|
||||
|
||||
temporal_mismatch_days = None
|
||||
if imagery_observed_at and reference_observed_at:
|
||||
imagery_date = datetime.fromisoformat(imagery_observed_at.replace("Z", "+00:00"))
|
||||
reference_date = datetime.fromisoformat(reference_observed_at.replace("Z", "+00:00"))
|
||||
temporal_mismatch_days = abs((imagery_date - reference_date).days)
|
||||
normalized = {
|
||||
"type": "FeatureCollection",
|
||||
"name": f"canonical-building-{source_name}",
|
||||
"features": accepted,
|
||||
}
|
||||
audit = {
|
||||
"schema_version": 1,
|
||||
"canonical_class": "building",
|
||||
"source_name": source_name,
|
||||
"reference_path": str(reference_path),
|
||||
"raster_path": str(raster_path),
|
||||
"min_label_px": min_label_px,
|
||||
"imagery_observed_at": imagery_observed_at,
|
||||
"reference_observed_at": reference_observed_at,
|
||||
"temporal_mismatch_days": temporal_mismatch_days,
|
||||
"input_feature_count": len(payload["features"]),
|
||||
"accepted_feature_count": len(accepted),
|
||||
"decision_counts": dict(sorted(counts.items())),
|
||||
"decisions": decisions,
|
||||
}
|
||||
return normalized, audit
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--reference", type=Path, required=True)
|
||||
parser.add_argument("--raster", type=Path, required=True)
|
||||
parser.add_argument("--source-name", choices=sorted(SUPPORTED_SOURCES), required=True)
|
||||
parser.add_argument("--output-reference", type=Path, required=True)
|
||||
parser.add_argument("--output-audit", type=Path, required=True)
|
||||
parser.add_argument("--min-label-px", type=float, default=3.0)
|
||||
parser.add_argument("--imagery-observed-at")
|
||||
parser.add_argument("--reference-observed-at")
|
||||
args = parser.parse_args()
|
||||
normalized, audit = normalize(
|
||||
reference_path=args.reference,
|
||||
raster_path=args.raster,
|
||||
source_name=args.source_name,
|
||||
min_label_px=args.min_label_px,
|
||||
imagery_observed_at=args.imagery_observed_at,
|
||||
reference_observed_at=args.reference_observed_at,
|
||||
)
|
||||
args.output_reference.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output_audit.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output_reference.write_text(json.dumps(normalized, ensure_ascii=False), encoding="utf-8")
|
||||
args.output_audit.write_text(json.dumps(audit, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(json.dumps({key: value for key, value in audit.items() if key != "decisions"}, ensure_ascii=False))
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user