Provision balanced Belgian training portfolio

backend/tests/test_belgium_training_portfolio.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+import importlib.util
+import sys
+from collections import Counter
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[2]
+SPEC = importlib.util.spec_from_file_location(
+    "building_portfolio", ROOT / "scripts" / "provision_belgium_building_training_portfolio.py"
+)
+assert SPEC and SPEC.loader
+module = importlib.util.module_from_spec(SPEC)
+sys.modules[SPEC.name] = module
+SPEC.loader.exec_module(module)
+
+
+def test_portfolio_covers_every_region_split_and_context_family() -> None:
+    assert len({aoi.slug for aoi in module.AOIS}) == len(module.AOIS)
+    counts = Counter((aoi.region, aoi.split) for aoi in module.AOIS)
+    for region in module.REGION_CONTRACT:
+        assert counts[(region, "train")] >= 4
+        assert counts[(region, "val")] >= 2
+        assert counts[(region, "calibration")] >= 2
+        assert counts[(region, "test")] >= 2
+        assert counts[(region, "background-test")] >= 2
+
+
+def test_portfolio_bbox_is_metric_sized() -> None:
+    bbox = module.bbox_for_center(4.35, 50.85, 256.0)
+    to_metric = module.Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
+    bounds = to_metric.transform_bounds(bbox["min_x"], bbox["min_y"], bbox["max_x"], bbox["max_y"])
+    assert 255 <= bounds[2] - bounds[0] <= 258
+    assert 255 <= bounds[3] - bounds[1] <= 258

deploy/unraid/Dockerfile.all-in-one

@@ -122,6 +122,7 @@ COPY scripts/manage_grb_refresh.py /app/scripts/manage_grb_refresh.py
 COPY scripts/export_operator_yolo_tile_dataset.py /app/scripts/export_operator_yolo_tile_dataset.py
 COPY scripts/normalize_belgium_building_labels.py /app/scripts/normalize_belgium_building_labels.py
 COPY scripts/assemble_belgium_building_corpus.py /app/scripts/assemble_belgium_building_corpus.py
+COPY scripts/provision_belgium_building_training_portfolio.py /app/scripts/provision_belgium_building_training_portfolio.py
 COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py
 COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py
 COPY scripts/train_operator_yolo_detector.sh /app/scripts/train_operator_yolo_detector.sh

scripts/provision_belgium_building_training_portfolio.py

@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+"""Provision a reproducible, region-balanced Belgian building corpus portfolio."""
+
+from __future__ import annotations
+
+import argparse
+import json
+from dataclasses import asdict, dataclass
+from pathlib import Path
+from typing import Any
+
+import requests
+from pyproj import Transformer
+
+
+@dataclass(frozen=True)
+class Aoi:
+    slug: str
+    region: str
+    context: str
+    split: str
+    lon: float
+    lat: float
+    sample_role: str = "positive"
+    require_empty: bool = False
+
+
+AOIS = (
+    # Flanders: all split roles plus authoritative empty/hard contexts.
+    Aoi("antwerp-core-train", "flanders", "dense-urban", "train", 4.402, 51.219),
+    Aoi("ghent-core-train", "flanders", "dense-urban", "train", 3.725, 51.052),
+    Aoi("genk-industry-train", "flanders", "industrial", "train", 5.500, 50.965),
+    Aoi("flanders-farms-train", "flanders", "rural-farms", "train", 4.850, 50.900),
+    Aoi("bruges-val", "flanders", "historic-urban", "val", 3.224, 51.209),
+    Aoi("turnhout-val", "flanders", "suburban", "val", 4.944, 51.322),
+    Aoi("hasselt-cal", "flanders", "suburban", "calibration", 5.340, 50.930),
+    Aoi("kortrijk-cal", "flanders", "urban-industrial", "calibration", 3.265, 50.828),
+    Aoi("leuven-test", "flanders", "urban", "test", 4.700, 50.880),
+    Aoi("sint-niklaas-test", "flanders", "ribbon-development", "test", 4.143, 51.165),
+    Aoi("kempen-forest-bg", "flanders", "forest-heath", "background-test", 5.180, 51.300, "background_candidate", True),
+    Aoi("antwerp-port-bg", "flanders", "port-hard-negative", "background-test", 4.380, 51.280, "background_candidate", True),
+    # Wallonia.
+    Aoi("liege-core-train", "wallonia", "dense-urban", "train", 5.570, 50.640),
+    Aoi("charleroi-core-train", "wallonia", "dense-urban", "train", 4.440, 50.410),
+    Aoi("seraing-industry-train", "wallonia", "industrial-valley", "train", 5.500, 50.600),
+    Aoi("namur-residential-train", "wallonia", "residential", "train", 4.870, 50.470),
+    Aoi("tournai-val", "wallonia", "historic-urban", "val", 3.389, 50.606),
+    Aoi("arlon-val", "wallonia", "small-city", "val", 5.817, 49.683),
+    Aoi("verviers-cal", "wallonia", "suburban", "calibration", 5.860, 50.590),
+    Aoi("dinant-cal", "wallonia", "valley-town", "calibration", 4.912, 50.260),
+    Aoi("mons-test", "wallonia", "urban", "test", 3.950, 50.450),
+    Aoi("bastogne-test", "wallonia", "rural-town", "test", 5.720, 50.000),
+    Aoi("wallonia-rural-bg", "wallonia", "open-rural", "background-test", 5.000, 50.300, "background_candidate", True),
+    Aoi("ardennes-forest-hard", "wallonia", "forest-hard-negative", "background-test", 5.600, 50.100, "background_candidate"),
+    # Brussels.
+    Aoi("brussels-center-train", "brussels", "dense-urban", "train", 4.352, 50.847),
+    Aoi("anderlecht-industry-train", "brussels", "industrial", "train", 4.320, 50.880),
+    Aoi("uccle-residential-train", "brussels", "detached-residential", "train", 4.350, 50.795),
+    Aoi("schaerbeek-train", "brussels", "dense-residential", "train", 4.380, 50.865),
+    Aoi("woluwe-val", "brussels", "suburban", "val", 4.430, 50.845),
+    Aoi("molenbeek-val", "brussels", "mixed-urban", "val", 4.325, 50.855),
+    Aoi("brussels-park-cal", "brussels", "park-edge", "calibration", 4.380, 50.820),
+    Aoi("brussels-canal-cal", "brussels", "canal-industry", "calibration", 4.340, 50.870),
+    Aoi("brussels-rail-test", "brussels", "rail-context", "test", 4.330, 50.840),
+    Aoi("jette-test", "brussels", "residential-park", "test", 4.325, 50.880),
+    Aoi("sonian-forest-hard", "brussels", "forest-hard-negative", "background-test", 4.410, 50.770, "background_candidate"),
+    Aoi("bois-cambre-hard", "brussels", "park-hard-negative", "background-test", 4.375, 50.795, "background_candidate"),
+)
+
+REGION_CONTRACT = {
+    "flanders": {
+        "area_id": "2a1a064e-cfd6-46f4-981a-d6ffa0ed5b5e",
+        "orthophoto_product": "most_recent",
+        "reference_path": "datasets/grb/acquire",
+        "reference_product": "buildings",
+    },
+    "wallonia": {
+        "area_id": "e5fd742a-ca18-4520-abe0-d28416aa2ece",
+        "orthophoto_product": "wallonia_latest",
+        "reference_path": "datasets/official-vector/acquire",
+        "reference_product": "spw_picc_buildings",
+    },
+    "brussels": {
+        "area_id": "304b413d-d81c-40fc-922e-49907b8efaaa",
+        "orthophoto_product": "brussels_latest",
+        "reference_path": "datasets/official-vector/acquire",
+        "reference_product": "urbis_buildings",
+    },
+}
+
+
+def bbox_for_center(lon: float, lat: float, side_m: float) -> dict[str, Any]:
+    to_metric = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
+    to_wgs84 = Transformer.from_crs("EPSG:31370", "EPSG:4326", always_xy=True)
+    x, y = to_metric.transform(lon, lat)
+    half = side_m / 2.0
+    min_lon, min_lat, max_lon, max_lat = to_wgs84.transform_bounds(x - half, y - half, x + half, y + half)
+    return {"min_x": min_lon, "min_y": min_lat, "max_x": max_lon, "max_y": max_lat, "crs": "EPSG:4326"}
+
+
+def post(session: requests.Session, url: str, payload: dict[str, Any]) -> dict[str, Any]:
+    response = session.post(url, json=payload, timeout=180)
+    response.raise_for_status()
+    body = response.json()
+    if body.get("error"):
+        raise RuntimeError(f"{body['error']}: {body.get('message')}")
+    job = body["data"]
+    if job.get("status") != "success":
+        raise RuntimeError(f"Job failed: {job}")
+    return job
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--project-id", required=True)
+    parser.add_argument("--base-url", default="http://127.0.0.1:8000/api/v1")
+    parser.add_argument("--output-spec", type=Path, required=True)
+    parser.add_argument("--side-m", type=float, default=256.0)
+    parser.add_argument("--resolution-m", type=float, default=0.25)
+    parser.add_argument("--force-refresh", action="store_true")
+    args = parser.parse_args()
+    session = requests.Session()
+    samples: list[dict[str, Any]] = []
+    for aoi in AOIS:
+        contract = REGION_CONTRACT[aoi.region]
+        bbox = bbox_for_center(aoi.lon, aoi.lat, args.side_m)
+        common = {"bbox": bbox, "area_id": contract["area_id"], "force_refresh": args.force_refresh}
+        image_job = post(
+            session,
+            f"{args.base_url}/projects/{args.project_id}/datasets/orthophoto/acquire",
+            {**common, "product_key": contract["orthophoto_product"], "resolution_m": args.resolution_m},
+        )
+        reference_job = post(
+            session,
+            f"{args.base_url}/projects/{args.project_id}/{contract['reference_path']}",
+            {**common, "product_key": contract["reference_product"]},
+        )
+        feature_count = int(reference_job["result_json"]["feature_count"])
+        if aoi.require_empty and feature_count != 0:
+            raise RuntimeError(f"Pure-background AOI {aoi.slug} contains {feature_count} reference buildings")
+        samples.append(
+            {
+                **asdict(aoi),
+                "bbox_epsg4326": [bbox["min_x"], bbox["min_y"], bbox["max_x"], bbox["max_y"]],
+                "raster_dataset_id": image_job["output_dataset_id"],
+                "reference_dataset_id": reference_job["output_dataset_id"],
+                "provider_reference_feature_count": feature_count,
+            }
+        )
+        print(f"{aoi.slug}: {feature_count} reference buildings", flush=True)
+    payload = {
+        "schema_version": 1,
+        "side_m": args.side_m,
+        "resolution_m": args.resolution_m,
+        "samples": samples,
+    }
+    args.output_spec.parent.mkdir(parents=True, exist_ok=True)
+    args.output_spec.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(json.dumps({"status": "ok", "sample_count": len(samples), "output_spec": str(args.output_spec)}))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())