diff --git a/backend/tests/test_building_label_normalization.py b/backend/tests/test_building_label_normalization.py new file mode 100644 index 00000000..f8e9c5b8 --- /dev/null +++ b/backend/tests/test_building_label_normalization.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +import numpy as np +import rasterio +from rasterio.transform import from_origin + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "normalize_belgium_building_labels.py" +SPEC = importlib.util.spec_from_file_location("normalize_buildings", SCRIPT) +assert SPEC and SPEC.loader +module = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(module) + + +def test_normalizer_retains_native_identity_and_records_rejections(tmp_path: Path) -> None: + raster_path = tmp_path / "image.tif" + with rasterio.open( + raster_path, + "w", + driver="GTiff", + width=100, + height=100, + count=3, + dtype="uint8", + crs="EPSG:4326", + transform=from_origin(4.0, 51.0, 0.001, 0.001), + ) as dataset: + dataset.write(np.zeros((3, 100, 100), dtype="uint8")) + valid = { + "type": "Feature", + "id": "native-1", + "properties": {"TYPE": "main building"}, + "geometry": {"type": "Polygon", "coordinates": [[[4.01, 50.99], [4.02, 50.99], [4.02, 50.98], [4.01, 50.98], [4.01, 50.99]]]}, + } + duplicate = json.loads(json.dumps(valid)) + duplicate["id"] = "native-2" + canopy = json.loads(json.dumps(valid)) + canopy["id"] = "native-3" + canopy["properties"]["TYPE"] = "canopy" + tiny = json.loads(json.dumps(valid)) + tiny["id"] = "native-4" + tiny["geometry"] = {"type": "Polygon", "coordinates": [[[4.03, 50.97], [4.031, 50.97], [4.031, 50.969], [4.03, 50.969], [4.03, 50.97]]]} + reference_path = tmp_path / "reference.geojson" + reference_path.write_text(json.dumps({"type": "FeatureCollection", "features": [valid, duplicate, canopy, tiny]}), encoding="utf-8") + + normalized, audit = module.normalize( + reference_path=reference_path, + raster_path=raster_path, + source_name="urbis", + min_label_px=3, + imagery_observed_at="2026-01-01T00:00:00Z", + reference_observed_at="2025-12-01T00:00:00Z", + ) + + assert len(normalized["features"]) == 1 + properties = normalized["features"][0]["properties"] + assert properties["canonical_class"] == "building" + assert properties["source_name"] == "urbis" + assert properties["source_feature_id"] == "native-1" + assert properties["source_class"] == "main building" + assert audit["decision_counts"] == { + "accepted": 1, + "below_resolvable_pixel_size": 1, + "duplicate_geometry": 1, + "excluded_canopy": 1, + } + assert audit["temporal_mismatch_days"] == 31 diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index 49307055..32d9c6b2 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -120,6 +120,8 @@ COPY scripts/provision_regional_grb_context.py /app/scripts/provision_regional_g COPY scripts/audit_source_freshness.py /app/scripts/audit_source_freshness.py COPY scripts/manage_grb_refresh.py /app/scripts/manage_grb_refresh.py COPY scripts/export_operator_yolo_tile_dataset.py /app/scripts/export_operator_yolo_tile_dataset.py +COPY scripts/normalize_belgium_building_labels.py /app/scripts/normalize_belgium_building_labels.py +COPY scripts/assemble_belgium_building_corpus.py /app/scripts/assemble_belgium_building_corpus.py COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py COPY scripts/train_operator_yolo_detector.sh /app/scripts/train_operator_yolo_detector.sh diff --git a/scripts/assemble_belgium_building_corpus.py b/scripts/assemble_belgium_building_corpus.py new file mode 100644 index 00000000..fff3e01e --- /dev/null +++ b/scripts/assemble_belgium_building_corpus.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Assemble immutable detector input pairs from persisted governed Datasets.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import shutil +from pathlib import Path +from typing import Any +from uuid import UUID + +from app.db.session import SessionLocal +from app.models import Dataset + +from normalize_belgium_building_labels import normalize + +REGION_SOURCES = { + "flanders": ("digitaal_vlaanderen_orthophoto", "grb"), + "wallonia": ("spw_orthophoto", "spw_picc"), + "brussels": ("urbis_orthophoto", "urbis"), +} +SPLITS = {"train", "val", "calibration", "test", "background-test"} + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _dataset_path(dataset: Dataset) -> Path: + if not dataset.storage_path: + raise SystemExit(f"Dataset {dataset.id} has no persisted storage path") + path = Path(dataset.storage_path) + if not path.is_file(): + raise SystemExit(f"Dataset {dataset.id} artifact is unreadable: {path}") + return path + + +def _validate_pair(sample: dict[str, Any], raster: Dataset, reference: Dataset) -> tuple[str, str]: + region = str(sample.get("region") or "").lower() + if region not in REGION_SOURCES: + raise SystemExit(f"Unsupported region for {sample.get('sample_slug')}: {region}") + expected_raster, expected_reference = REGION_SOURCES[region] + if raster.source_name != expected_raster: + raise SystemExit(f"Raster provider mismatch for {sample['sample_slug']}: {raster.source_name}") + if reference.source_name != expected_reference or reference.reference_layer_name != "buildings": + raise SystemExit(f"Reference provider/layer mismatch for {sample['sample_slug']}") + split = str(sample.get("split") or "") + if split not in SPLITS: + raise SystemExit(f"Unsupported split for {sample['sample_slug']}: {split}") + if raster.status != "ready" or reference.status != "ready": + raise SystemExit(f"Dataset pair is not ready for {sample['sample_slug']}") + return region, expected_reference + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--spec", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--version", default="building-be-v1") + parser.add_argument("--min-label-px", type=float, default=3.0) + parser.add_argument("--freeze", action="store_true") + args = parser.parse_args() + + spec = json.loads(args.spec.read_text(encoding="utf-8-sig")) + samples = spec.get("samples") + if not isinstance(samples, list) or not samples: + raise SystemExit("Corpus spec must contain at least one sample") + output_dir = args.output_dir.resolve() + if output_dir.exists() and any(output_dir.iterdir()): + raise SystemExit(f"Refusing to overwrite non-empty corpus directory: {output_dir}") + output_dir.mkdir(parents=True, exist_ok=True) + pairs_dir = output_dir / "pairs" + pairs_dir.mkdir() + + manifest_samples: list[dict[str, Any]] = [] + seen_slugs: set[str] = set() + with SessionLocal() as db: + for sample in samples: + slug = str(sample.get("sample_slug") or "").strip() + if not slug or slug in seen_slugs: + raise SystemExit(f"Missing or duplicate sample_slug: {slug}") + seen_slugs.add(slug) + raster = db.get(Dataset, UUID(str(sample["raster_dataset_id"]))) + reference = db.get(Dataset, UUID(str(sample["reference_dataset_id"]))) + if raster is None or reference is None: + raise SystemExit(f"Persisted Dataset pair not found for {slug}") + region, reference_source = _validate_pair(sample, raster, reference) + raster_source = _dataset_path(raster) + reference_source_path = _dataset_path(reference) + sample_dir = pairs_dir / slug + sample_dir.mkdir() + raster_target = sample_dir / "image.tif" + normalized_target = sample_dir / "buildings.normalized.geojson" + audit_target = sample_dir / "label-audit.json" + shutil.copyfile(raster_source, raster_target) + normalized, audit = normalize( + reference_path=reference_source_path, + raster_path=raster_target, + source_name=reference_source, + min_label_px=args.min_label_px, + imagery_observed_at=raster.observed_at.isoformat() if raster.observed_at else None, + reference_observed_at=reference.observed_at.isoformat() if reference.observed_at else None, + ) + normalized_target.write_text(json.dumps(normalized, ensure_ascii=False), encoding="utf-8") + audit_target.write_text(json.dumps(audit, ensure_ascii=False, indent=2), encoding="utf-8") + manifest_samples.append( + { + "sample_slug": slug, + "sample_role": sample.get("sample_role", "positive"), + "region": region, + "context": sample.get("context"), + "split": sample["split"], + "raster_path": str(raster_target), + "reference_path": str(normalized_target), + "reference_source": reference_source, + "reference_layer": "buildings", + "reference_feature_count": audit["accepted_feature_count"], + "raster_dataset_id": str(raster.id), + "reference_dataset_id": str(reference.id), + "raster_sha256": sha256(raster_target), + "reference_sha256": sha256(normalized_target), + "label_audit_sha256": sha256(audit_target), + "bbox_epsg4326": (raster.source_metadata or {}).get("bbox_epsg4326"), + } + ) + manifest = { + "schema_version": 1, + "dataset_version": args.version, + "immutable": bool(args.freeze), + "samples": manifest_samples, + } + manifest_path = output_dir / "operator_samples_manifest.json" + manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8") + freeze = { + "dataset_version": args.version, + "manifest_sha256": sha256(manifest_path), + "sample_count": len(manifest_samples), + "immutable": bool(args.freeze), + } + (output_dir / "corpus-freeze.json").write_text(json.dumps(freeze, indent=2), encoding="utf-8") + print(json.dumps(freeze)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/export_operator_yolo_tile_dataset.py b/scripts/export_operator_yolo_tile_dataset.py index 3025f09b..1dbccef6 100644 --- a/scripts/export_operator_yolo_tile_dataset.py +++ b/scripts/export_operator_yolo_tile_dataset.py @@ -484,6 +484,8 @@ def export_sample_tiles( ) -> list[dict[str, Any]]: sample_slug = str(sample["sample_slug"]) sample_role = str(sample.get("sample_role") or "reference") + sample_reference_source = str(sample.get("reference_source") or reference_source).strip().lower() + sample_reference_layer = str(sample.get("reference_layer") or reference_layer).strip().lower() background_category = background_category_for_sample(sample) recommended_split = str(sample.get("recommended_split") or "") split = "val" if sample_slug.lower() in val_slugs else "train" @@ -500,8 +502,8 @@ def export_sample_tiles( reference_path, dataset, min_label_px=min_label_px, - reference_source=reference_source, - reference_layer=reference_layer, + reference_source=sample_reference_source, + reference_layer=sample_reference_layer, ) for tile_index, tile_window in enumerate(iter_tile_windows(dataset.width, dataset.height, tile_size, stride)): labels = labels_for_tile( diff --git a/scripts/normalize_belgium_building_labels.py b/scripts/normalize_belgium_building_labels.py new file mode 100644 index 00000000..4379f364 --- /dev/null +++ b/scripts/normalize_belgium_building_labels.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Normalize governed Belgian building references for detector training. + +The script never asserts semantic parity between providers. It emits one +canonical training class while retaining provider-native identifiers, +properties and an explicit accept/reject decision for every source feature. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from collections import Counter +from datetime import datetime +from pathlib import Path +from typing import Any + +import rasterio +from pyproj import Transformer +from shapely.geometry import mapping, shape +from shapely.ops import transform as shapely_transform +from shapely.validation import make_valid + +SUPPORTED_SOURCES = {"grb", "spw_picc", "urbis"} +EXCLUDED_TOKENS = { + "canopy": ("canopy", "afdak", "auvent"), + "ruin": ("ruin", "ruine", "ruïne"), + "underground": ("underground", "ondergronds", "souterrain"), +} + + +def _feature_id(feature: dict[str, Any], index: int) -> str: + properties = feature.get("properties") or {} + for key in ("source_feature_id", "OBJECTID", "INSPIRE_ID", "id", "gml_id"): + value = properties.get(key) + if value not in (None, ""): + return str(value) + if feature.get("id") not in (None, ""): + return str(feature["id"]) + return f"row-{index}" + + +def _source_class(properties: dict[str, Any]) -> str | None: + for key in ("source_class", "TYPE", "type", "OBJTYPE", "nature", "NATURE", "class"): + value = properties.get(key) + if value not in (None, ""): + return str(value) + return None + + +def _semantic_exclusion(properties: dict[str, Any]) -> str | None: + haystack = " ".join(str(value).lower() for value in properties.values() if value is not None) + for reason, tokens in EXCLUDED_TOKENS.items(): + if any(token in haystack for token in tokens): + return f"excluded_{reason}" + return None + + +def _polygonal(geometry: Any) -> Any | None: + if geometry.geom_type in {"Polygon", "MultiPolygon"}: + return geometry + if geometry.geom_type == "GeometryCollection": + polygons = [part for part in geometry.geoms if part.geom_type in {"Polygon", "MultiPolygon"}] + if not polygons: + return None + from shapely.ops import unary_union + + return unary_union(polygons) + return None + + +def normalize( + *, + reference_path: Path, + raster_path: Path, + source_name: str, + min_label_px: float, + imagery_observed_at: str | None, + reference_observed_at: str | None, +) -> tuple[dict[str, Any], dict[str, Any]]: + if source_name not in SUPPORTED_SOURCES: + raise SystemExit(f"Unsupported governed building source: {source_name}") + payload = json.loads(reference_path.read_text(encoding="utf-8-sig")) + if payload.get("type") != "FeatureCollection" or not isinstance(payload.get("features"), list): + raise SystemExit("Reference must be a GeoJSON FeatureCollection") + + accepted: list[dict[str, Any]] = [] + decisions: list[dict[str, Any]] = [] + seen: set[str] = set() + counts: Counter[str] = Counter() + with rasterio.open(raster_path) as raster: + if raster.crs is None: + raise SystemExit("Raster CRS is required") + transformer = Transformer.from_crs("EPSG:4326", raster.crs, always_xy=True) + for index, feature in enumerate(payload["features"]): + properties = dict(feature.get("properties") or {}) + source_feature_id = _feature_id(feature, index) + decision = { + "source_name": source_name, + "source_feature_id": source_feature_id, + "source_class": _source_class(properties), + "accepted": False, + "reason": None, + "geometry_repaired": False, + } + try: + geometry = shape(feature.get("geometry")) + except Exception: + decision["reason"] = "invalid_geometry_unreadable" + decisions.append(decision) + counts[decision["reason"]] += 1 + continue + if not geometry.is_valid: + geometry = make_valid(geometry) + decision["geometry_repaired"] = True + geometry = _polygonal(geometry) + if geometry is None or geometry.is_empty or not geometry.is_valid: + decision["reason"] = "invalid_geometry_unrepairable" + elif (reason := _semantic_exclusion(properties)) is not None: + decision["reason"] = reason + else: + metric = shapely_transform(transformer.transform, geometry) + min_x, min_y, max_x, max_y = metric.bounds + pixel_width = (max_x - min_x) / abs(float(raster.transform.a)) + pixel_height = (max_y - min_y) / abs(float(raster.transform.e)) + decision["pixel_width"] = pixel_width + decision["pixel_height"] = pixel_height + if pixel_width < min_label_px or pixel_height < min_label_px: + decision["reason"] = "below_resolvable_pixel_size" + else: + digest = hashlib.sha256(geometry.normalize().wkb).hexdigest() + if digest in seen: + decision["reason"] = "duplicate_geometry" + else: + seen.add(digest) + decision["accepted"] = True + decision["reason"] = "accepted" + normalized_properties = dict(properties) + normalized_properties.update( + { + "canonical_class": "building", + "source_name": source_name, + "reference_layer_name": "buildings", + "source_feature_id": source_feature_id, + "source_class": decision["source_class"], + "label_decision": "accepted", + "geometry_repaired": decision["geometry_repaired"], + } + ) + accepted.append( + { + "type": "Feature", + "id": f"{source_name}:{source_feature_id}", + "properties": normalized_properties, + "geometry": mapping(geometry), + } + ) + decisions.append(decision) + counts[str(decision["reason"])] += 1 + + temporal_mismatch_days = None + if imagery_observed_at and reference_observed_at: + imagery_date = datetime.fromisoformat(imagery_observed_at.replace("Z", "+00:00")) + reference_date = datetime.fromisoformat(reference_observed_at.replace("Z", "+00:00")) + temporal_mismatch_days = abs((imagery_date - reference_date).days) + normalized = { + "type": "FeatureCollection", + "name": f"canonical-building-{source_name}", + "features": accepted, + } + audit = { + "schema_version": 1, + "canonical_class": "building", + "source_name": source_name, + "reference_path": str(reference_path), + "raster_path": str(raster_path), + "min_label_px": min_label_px, + "imagery_observed_at": imagery_observed_at, + "reference_observed_at": reference_observed_at, + "temporal_mismatch_days": temporal_mismatch_days, + "input_feature_count": len(payload["features"]), + "accepted_feature_count": len(accepted), + "decision_counts": dict(sorted(counts.items())), + "decisions": decisions, + } + return normalized, audit + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--reference", type=Path, required=True) + parser.add_argument("--raster", type=Path, required=True) + parser.add_argument("--source-name", choices=sorted(SUPPORTED_SOURCES), required=True) + parser.add_argument("--output-reference", type=Path, required=True) + parser.add_argument("--output-audit", type=Path, required=True) + parser.add_argument("--min-label-px", type=float, default=3.0) + parser.add_argument("--imagery-observed-at") + parser.add_argument("--reference-observed-at") + args = parser.parse_args() + normalized, audit = normalize( + reference_path=args.reference, + raster_path=args.raster, + source_name=args.source_name, + min_label_px=args.min_label_px, + imagery_observed_at=args.imagery_observed_at, + reference_observed_at=args.reference_observed_at, + ) + args.output_reference.parent.mkdir(parents=True, exist_ok=True) + args.output_audit.parent.mkdir(parents=True, exist_ok=True) + args.output_reference.write_text(json.dumps(normalized, ensure_ascii=False), encoding="utf-8") + args.output_audit.write_text(json.dumps(audit, ensure_ascii=False, indent=2), encoding="utf-8") + print(json.dumps({key: value for key, value in audit.items() if key != "decisions"}, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())