Build governed Belgian training corpus pipeline

backend/tests/test_building_label_normalization.py

@@ -0,0 +1,71 @@
+from __future__ import annotations
+
+import importlib.util
+import json
+from pathlib import Path
+
+import numpy as np
+import rasterio
+from rasterio.transform import from_origin
+
+ROOT = Path(__file__).resolve().parents[2]
+SCRIPT = ROOT / "scripts" / "normalize_belgium_building_labels.py"
+SPEC = importlib.util.spec_from_file_location("normalize_buildings", SCRIPT)
+assert SPEC and SPEC.loader
+module = importlib.util.module_from_spec(SPEC)
+SPEC.loader.exec_module(module)
+
+
+def test_normalizer_retains_native_identity_and_records_rejections(tmp_path: Path) -> None:
+    raster_path = tmp_path / "image.tif"
+    with rasterio.open(
+        raster_path,
+        "w",
+        driver="GTiff",
+        width=100,
+        height=100,
+        count=3,
+        dtype="uint8",
+        crs="EPSG:4326",
+        transform=from_origin(4.0, 51.0, 0.001, 0.001),
+    ) as dataset:
+        dataset.write(np.zeros((3, 100, 100), dtype="uint8"))
+    valid = {
+        "type": "Feature",
+        "id": "native-1",
+        "properties": {"TYPE": "main building"},
+        "geometry": {"type": "Polygon", "coordinates": [[[4.01, 50.99], [4.02, 50.99], [4.02, 50.98], [4.01, 50.98], [4.01, 50.99]]]},
+    }
+    duplicate = json.loads(json.dumps(valid))
+    duplicate["id"] = "native-2"
+    canopy = json.loads(json.dumps(valid))
+    canopy["id"] = "native-3"
+    canopy["properties"]["TYPE"] = "canopy"
+    tiny = json.loads(json.dumps(valid))
+    tiny["id"] = "native-4"
+    tiny["geometry"] = {"type": "Polygon", "coordinates": [[[4.03, 50.97], [4.031, 50.97], [4.031, 50.969], [4.03, 50.969], [4.03, 50.97]]]}
+    reference_path = tmp_path / "reference.geojson"
+    reference_path.write_text(json.dumps({"type": "FeatureCollection", "features": [valid, duplicate, canopy, tiny]}), encoding="utf-8")
+
+    normalized, audit = module.normalize(
+        reference_path=reference_path,
+        raster_path=raster_path,
+        source_name="urbis",
+        min_label_px=3,
+        imagery_observed_at="2026-01-01T00:00:00Z",
+        reference_observed_at="2025-12-01T00:00:00Z",
+    )
+
+    assert len(normalized["features"]) == 1
+    properties = normalized["features"][0]["properties"]
+    assert properties["canonical_class"] == "building"
+    assert properties["source_name"] == "urbis"
+    assert properties["source_feature_id"] == "native-1"
+    assert properties["source_class"] == "main building"
+    assert audit["decision_counts"] == {
+        "accepted": 1,
+        "below_resolvable_pixel_size": 1,
+        "duplicate_geometry": 1,
+        "excluded_canopy": 1,
+    }
+    assert audit["temporal_mismatch_days"] == 31

deploy/unraid/Dockerfile.all-in-one

@@ -120,6 +120,8 @@ COPY scripts/provision_regional_grb_context.py /app/scripts/provision_regional_g
 COPY scripts/audit_source_freshness.py /app/scripts/audit_source_freshness.py
 COPY scripts/manage_grb_refresh.py /app/scripts/manage_grb_refresh.py
 COPY scripts/export_operator_yolo_tile_dataset.py /app/scripts/export_operator_yolo_tile_dataset.py
+COPY scripts/normalize_belgium_building_labels.py /app/scripts/normalize_belgium_building_labels.py
+COPY scripts/assemble_belgium_building_corpus.py /app/scripts/assemble_belgium_building_corpus.py
 COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py
 COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py
 COPY scripts/train_operator_yolo_detector.sh /app/scripts/train_operator_yolo_detector.sh

scripts/assemble_belgium_building_corpus.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python3
+"""Assemble immutable detector input pairs from persisted governed Datasets."""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+import shutil
+from pathlib import Path
+from typing import Any
+from uuid import UUID
+
+from app.db.session import SessionLocal
+from app.models import Dataset
+
+from normalize_belgium_building_labels import normalize
+
+REGION_SOURCES = {
+    "flanders": ("digitaal_vlaanderen_orthophoto", "grb"),
+    "wallonia": ("spw_orthophoto", "spw_picc"),
+    "brussels": ("urbis_orthophoto", "urbis"),
+}
+SPLITS = {"train", "val", "calibration", "test", "background-test"}
+
+
+def sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def _dataset_path(dataset: Dataset) -> Path:
+    if not dataset.storage_path:
+        raise SystemExit(f"Dataset {dataset.id} has no persisted storage path")
+    path = Path(dataset.storage_path)
+    if not path.is_file():
+        raise SystemExit(f"Dataset {dataset.id} artifact is unreadable: {path}")
+    return path
+
+
+def _validate_pair(sample: dict[str, Any], raster: Dataset, reference: Dataset) -> tuple[str, str]:
+    region = str(sample.get("region") or "").lower()
+    if region not in REGION_SOURCES:
+        raise SystemExit(f"Unsupported region for {sample.get('sample_slug')}: {region}")
+    expected_raster, expected_reference = REGION_SOURCES[region]
+    if raster.source_name != expected_raster:
+        raise SystemExit(f"Raster provider mismatch for {sample['sample_slug']}: {raster.source_name}")
+    if reference.source_name != expected_reference or reference.reference_layer_name != "buildings":
+        raise SystemExit(f"Reference provider/layer mismatch for {sample['sample_slug']}")
+    split = str(sample.get("split") or "")
+    if split not in SPLITS:
+        raise SystemExit(f"Unsupported split for {sample['sample_slug']}: {split}")
+    if raster.status != "ready" or reference.status != "ready":
+        raise SystemExit(f"Dataset pair is not ready for {sample['sample_slug']}")
+    return region, expected_reference
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--spec", type=Path, required=True)
+    parser.add_argument("--output-dir", type=Path, required=True)
+    parser.add_argument("--version", default="building-be-v1")
+    parser.add_argument("--min-label-px", type=float, default=3.0)
+    parser.add_argument("--freeze", action="store_true")
+    args = parser.parse_args()
+
+    spec = json.loads(args.spec.read_text(encoding="utf-8-sig"))
+    samples = spec.get("samples")
+    if not isinstance(samples, list) or not samples:
+        raise SystemExit("Corpus spec must contain at least one sample")
+    output_dir = args.output_dir.resolve()
+    if output_dir.exists() and any(output_dir.iterdir()):
+        raise SystemExit(f"Refusing to overwrite non-empty corpus directory: {output_dir}")
+    output_dir.mkdir(parents=True, exist_ok=True)
+    pairs_dir = output_dir / "pairs"
+    pairs_dir.mkdir()
+
+    manifest_samples: list[dict[str, Any]] = []
+    seen_slugs: set[str] = set()
+    with SessionLocal() as db:
+        for sample in samples:
+            slug = str(sample.get("sample_slug") or "").strip()
+            if not slug or slug in seen_slugs:
+                raise SystemExit(f"Missing or duplicate sample_slug: {slug}")
+            seen_slugs.add(slug)
+            raster = db.get(Dataset, UUID(str(sample["raster_dataset_id"])))
+            reference = db.get(Dataset, UUID(str(sample["reference_dataset_id"])))
+            if raster is None or reference is None:
+                raise SystemExit(f"Persisted Dataset pair not found for {slug}")
+            region, reference_source = _validate_pair(sample, raster, reference)
+            raster_source = _dataset_path(raster)
+            reference_source_path = _dataset_path(reference)
+            sample_dir = pairs_dir / slug
+            sample_dir.mkdir()
+            raster_target = sample_dir / "image.tif"
+            normalized_target = sample_dir / "buildings.normalized.geojson"
+            audit_target = sample_dir / "label-audit.json"
+            shutil.copyfile(raster_source, raster_target)
+            normalized, audit = normalize(
+                reference_path=reference_source_path,
+                raster_path=raster_target,
+                source_name=reference_source,
+                min_label_px=args.min_label_px,
+                imagery_observed_at=raster.observed_at.isoformat() if raster.observed_at else None,
+                reference_observed_at=reference.observed_at.isoformat() if reference.observed_at else None,
+            )
+            normalized_target.write_text(json.dumps(normalized, ensure_ascii=False), encoding="utf-8")
+            audit_target.write_text(json.dumps(audit, ensure_ascii=False, indent=2), encoding="utf-8")
+            manifest_samples.append(
+                {
+                    "sample_slug": slug,
+                    "sample_role": sample.get("sample_role", "positive"),
+                    "region": region,
+                    "context": sample.get("context"),
+                    "split": sample["split"],
+                    "raster_path": str(raster_target),
+                    "reference_path": str(normalized_target),
+                    "reference_source": reference_source,
+                    "reference_layer": "buildings",
+                    "reference_feature_count": audit["accepted_feature_count"],
+                    "raster_dataset_id": str(raster.id),
+                    "reference_dataset_id": str(reference.id),
+                    "raster_sha256": sha256(raster_target),
+                    "reference_sha256": sha256(normalized_target),
+                    "label_audit_sha256": sha256(audit_target),
+                    "bbox_epsg4326": (raster.source_metadata or {}).get("bbox_epsg4326"),
+                }
+            )
+    manifest = {
+        "schema_version": 1,
+        "dataset_version": args.version,
+        "immutable": bool(args.freeze),
+        "samples": manifest_samples,
+    }
+    manifest_path = output_dir / "operator_samples_manifest.json"
+    manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
+    freeze = {
+        "dataset_version": args.version,
+        "manifest_sha256": sha256(manifest_path),
+        "sample_count": len(manifest_samples),
+        "immutable": bool(args.freeze),
+    }
+    (output_dir / "corpus-freeze.json").write_text(json.dumps(freeze, indent=2), encoding="utf-8")
+    print(json.dumps(freeze))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/export_operator_yolo_tile_dataset.py

@@ -484,6 +484,8 @@ def export_sample_tiles(
 ) -> list[dict[str, Any]]:
     sample_slug = str(sample["sample_slug"])
     sample_role = str(sample.get("sample_role") or "reference")
+    sample_reference_source = str(sample.get("reference_source") or reference_source).strip().lower()
+    sample_reference_layer = str(sample.get("reference_layer") or reference_layer).strip().lower()
     background_category = background_category_for_sample(sample)
     recommended_split = str(sample.get("recommended_split") or "")
     split = "val" if sample_slug.lower() in val_slugs else "train"
@@ -500,8 +502,8 @@ def export_sample_tiles(
             reference_path,
             dataset,
             min_label_px=min_label_px,
-            reference_source=reference_source,
+            reference_source=sample_reference_source,
-            reference_layer=reference_layer,
+            reference_layer=sample_reference_layer,
         )
         for tile_index, tile_window in enumerate(iter_tile_windows(dataset.width, dataset.height, tile_size, stride)):
             labels = labels_for_tile(

scripts/normalize_belgium_building_labels.py

@@ -0,0 +1,218 @@
+#!/usr/bin/env python3
+"""Normalize governed Belgian building references for detector training.
+
+The script never asserts semantic parity between providers. It emits one
+canonical training class while retaining provider-native identifiers,
+properties and an explicit accept/reject decision for every source feature.
+"""
+
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from collections import Counter
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+import rasterio
+from pyproj import Transformer
+from shapely.geometry import mapping, shape
+from shapely.ops import transform as shapely_transform
+from shapely.validation import make_valid
+
+SUPPORTED_SOURCES = {"grb", "spw_picc", "urbis"}
+EXCLUDED_TOKENS = {
+    "canopy": ("canopy", "afdak", "auvent"),
+    "ruin": ("ruin", "ruine", "ruïne"),
+    "underground": ("underground", "ondergronds", "souterrain"),
+}
+
+
+def _feature_id(feature: dict[str, Any], index: int) -> str:
+    properties = feature.get("properties") or {}
+    for key in ("source_feature_id", "OBJECTID", "INSPIRE_ID", "id", "gml_id"):
+        value = properties.get(key)
+        if value not in (None, ""):
+            return str(value)
+    if feature.get("id") not in (None, ""):
+        return str(feature["id"])
+    return f"row-{index}"
+
+
+def _source_class(properties: dict[str, Any]) -> str | None:
+    for key in ("source_class", "TYPE", "type", "OBJTYPE", "nature", "NATURE", "class"):
+        value = properties.get(key)
+        if value not in (None, ""):
+            return str(value)
+    return None
+
+
+def _semantic_exclusion(properties: dict[str, Any]) -> str | None:
+    haystack = " ".join(str(value).lower() for value in properties.values() if value is not None)
+    for reason, tokens in EXCLUDED_TOKENS.items():
+        if any(token in haystack for token in tokens):
+            return f"excluded_{reason}"
+    return None
+
+
+def _polygonal(geometry: Any) -> Any | None:
+    if geometry.geom_type in {"Polygon", "MultiPolygon"}:
+        return geometry
+    if geometry.geom_type == "GeometryCollection":
+        polygons = [part for part in geometry.geoms if part.geom_type in {"Polygon", "MultiPolygon"}]
+        if not polygons:
+            return None
+        from shapely.ops import unary_union
+
+        return unary_union(polygons)
+    return None
+
+
+def normalize(
+    *,
+    reference_path: Path,
+    raster_path: Path,
+    source_name: str,
+    min_label_px: float,
+    imagery_observed_at: str | None,
+    reference_observed_at: str | None,
+) -> tuple[dict[str, Any], dict[str, Any]]:
+    if source_name not in SUPPORTED_SOURCES:
+        raise SystemExit(f"Unsupported governed building source: {source_name}")
+    payload = json.loads(reference_path.read_text(encoding="utf-8-sig"))
+    if payload.get("type") != "FeatureCollection" or not isinstance(payload.get("features"), list):
+        raise SystemExit("Reference must be a GeoJSON FeatureCollection")
+
+    accepted: list[dict[str, Any]] = []
+    decisions: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    counts: Counter[str] = Counter()
+    with rasterio.open(raster_path) as raster:
+        if raster.crs is None:
+            raise SystemExit("Raster CRS is required")
+        transformer = Transformer.from_crs("EPSG:4326", raster.crs, always_xy=True)
+        for index, feature in enumerate(payload["features"]):
+            properties = dict(feature.get("properties") or {})
+            source_feature_id = _feature_id(feature, index)
+            decision = {
+                "source_name": source_name,
+                "source_feature_id": source_feature_id,
+                "source_class": _source_class(properties),
+                "accepted": False,
+                "reason": None,
+                "geometry_repaired": False,
+            }
+            try:
+                geometry = shape(feature.get("geometry"))
+            except Exception:
+                decision["reason"] = "invalid_geometry_unreadable"
+                decisions.append(decision)
+                counts[decision["reason"]] += 1
+                continue
+            if not geometry.is_valid:
+                geometry = make_valid(geometry)
+                decision["geometry_repaired"] = True
+            geometry = _polygonal(geometry)
+            if geometry is None or geometry.is_empty or not geometry.is_valid:
+                decision["reason"] = "invalid_geometry_unrepairable"
+            elif (reason := _semantic_exclusion(properties)) is not None:
+                decision["reason"] = reason
+            else:
+                metric = shapely_transform(transformer.transform, geometry)
+                min_x, min_y, max_x, max_y = metric.bounds
+                pixel_width = (max_x - min_x) / abs(float(raster.transform.a))
+                pixel_height = (max_y - min_y) / abs(float(raster.transform.e))
+                decision["pixel_width"] = pixel_width
+                decision["pixel_height"] = pixel_height
+                if pixel_width < min_label_px or pixel_height < min_label_px:
+                    decision["reason"] = "below_resolvable_pixel_size"
+                else:
+                    digest = hashlib.sha256(geometry.normalize().wkb).hexdigest()
+                    if digest in seen:
+                        decision["reason"] = "duplicate_geometry"
+                    else:
+                        seen.add(digest)
+                        decision["accepted"] = True
+                        decision["reason"] = "accepted"
+                        normalized_properties = dict(properties)
+                        normalized_properties.update(
+                            {
+                                "canonical_class": "building",
+                                "source_name": source_name,
+                                "reference_layer_name": "buildings",
+                                "source_feature_id": source_feature_id,
+                                "source_class": decision["source_class"],
+                                "label_decision": "accepted",
+                                "geometry_repaired": decision["geometry_repaired"],
+                            }
+                        )
+                        accepted.append(
+                            {
+                                "type": "Feature",
+                                "id": f"{source_name}:{source_feature_id}",
+                                "properties": normalized_properties,
+                                "geometry": mapping(geometry),
+                            }
+                        )
+            decisions.append(decision)
+            counts[str(decision["reason"])] += 1
+
+    temporal_mismatch_days = None
+    if imagery_observed_at and reference_observed_at:
+        imagery_date = datetime.fromisoformat(imagery_observed_at.replace("Z", "+00:00"))
+        reference_date = datetime.fromisoformat(reference_observed_at.replace("Z", "+00:00"))
+        temporal_mismatch_days = abs((imagery_date - reference_date).days)
+    normalized = {
+        "type": "FeatureCollection",
+        "name": f"canonical-building-{source_name}",
+        "features": accepted,
+    }
+    audit = {
+        "schema_version": 1,
+        "canonical_class": "building",
+        "source_name": source_name,
+        "reference_path": str(reference_path),
+        "raster_path": str(raster_path),
+        "min_label_px": min_label_px,
+        "imagery_observed_at": imagery_observed_at,
+        "reference_observed_at": reference_observed_at,
+        "temporal_mismatch_days": temporal_mismatch_days,
+        "input_feature_count": len(payload["features"]),
+        "accepted_feature_count": len(accepted),
+        "decision_counts": dict(sorted(counts.items())),
+        "decisions": decisions,
+    }
+    return normalized, audit
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--reference", type=Path, required=True)
+    parser.add_argument("--raster", type=Path, required=True)
+    parser.add_argument("--source-name", choices=sorted(SUPPORTED_SOURCES), required=True)
+    parser.add_argument("--output-reference", type=Path, required=True)
+    parser.add_argument("--output-audit", type=Path, required=True)
+    parser.add_argument("--min-label-px", type=float, default=3.0)
+    parser.add_argument("--imagery-observed-at")
+    parser.add_argument("--reference-observed-at")
+    args = parser.parse_args()
+    normalized, audit = normalize(
+        reference_path=args.reference,
+        raster_path=args.raster,
+        source_name=args.source_name,
+        min_label_px=args.min_label_px,
+        imagery_observed_at=args.imagery_observed_at,
+        reference_observed_at=args.reference_observed_at,
+    )
+    args.output_reference.parent.mkdir(parents=True, exist_ok=True)
+    args.output_audit.parent.mkdir(parents=True, exist_ok=True)
+    args.output_reference.write_text(json.dumps(normalized, ensure_ascii=False), encoding="utf-8")
+    args.output_audit.write_text(json.dumps(audit, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(json.dumps({key: value for key, value in audit.items() if key != "decisions"}, ensure_ascii=False))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())