Complete Belgian building corpus v2 workflow

scripts/provision_belgium_building_training_portfolio.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 
 import argparse
 import json
-from dataclasses import asdict, dataclass
+from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
@@ -31,6 +31,8 @@ AOIS = (
     Aoi("ghent-core-train", "flanders", "dense-urban", "train", 3.725, 51.052),
     Aoi("genk-industry-train", "flanders", "industrial", "train", 5.500, 50.965),
     Aoi("flanders-farms-train", "flanders", "rural-farms", "train", 4.850, 50.900),
+    Aoi("kalmthout-heath-train-bg", "flanders", "heath-negative", "train", 4.450, 51.390, "background_candidate"),
+    Aoi("limburg-forest-train-bg", "flanders", "forest-negative", "train", 5.550, 51.050, "background_candidate"),
     Aoi("bruges-val", "flanders", "historic-urban", "val", 3.224, 51.209),
     Aoi("turnhout-val", "flanders", "suburban", "val", 4.944, 51.322),
     Aoi("hasselt-cal", "flanders", "suburban", "calibration", 5.340, 50.930),
@@ -44,6 +46,8 @@ AOIS = (
     Aoi("charleroi-core-train", "wallonia", "dense-urban", "train", 4.440, 50.410),
     Aoi("seraing-industry-train", "wallonia", "industrial-valley", "train", 5.500, 50.600),
     Aoi("namur-residential-train", "wallonia", "residential", "train", 4.870, 50.470),
+    Aoi("ardennes-forest-train-bg", "wallonia", "forest-negative", "train", 5.700, 50.200, "background_candidate"),
+    Aoi("wallonia-quarry-train-hard", "wallonia", "quarry-hard-negative", "train", 5.130, 50.530, "background_candidate"),
     Aoi("tournai-val", "wallonia", "historic-urban", "val", 3.389, 50.606),
     Aoi("arlon-val", "wallonia", "small-city", "val", 5.817, 49.683),
     Aoi("verviers-cal", "wallonia", "suburban", "calibration", 5.860, 50.590),
@@ -57,13 +61,15 @@ AOIS = (
     Aoi("anderlecht-industry-train", "brussels", "industrial", "train", 4.320, 50.880),
     Aoi("uccle-residential-train", "brussels", "detached-residential", "train", 4.350, 50.795),
     Aoi("schaerbeek-train", "brussels", "dense-residential", "train", 4.380, 50.865),
+    Aoi("brussels-rail-train-hard", "brussels", "rail-hard-negative", "train", 4.345, 50.875, "background_candidate"),
+    Aoi("brussels-park-train-hard", "brussels", "park-hard-negative", "train", 4.400, 50.820, "background_candidate"),
     Aoi("woluwe-val", "brussels", "suburban", "val", 4.430, 50.845),
     Aoi("molenbeek-val", "brussels", "mixed-urban", "val", 4.325, 50.855),
     Aoi("brussels-park-cal", "brussels", "park-edge", "calibration", 4.380, 50.820),
     Aoi("brussels-canal-cal", "brussels", "canal-industry", "calibration", 4.340, 50.870),
     Aoi("brussels-rail-test", "brussels", "rail-context", "test", 4.330, 50.840),
     Aoi("jette-test", "brussels", "residential-park", "test", 4.325, 50.880),
-    Aoi("sonian-forest-hard", "brussels", "forest-hard-negative", "background-test", 4.410, 50.770, "background_candidate"),
+    Aoi("sonian-forest-hard", "brussels", "forest-hard-negative", "background-test", 4.420, 50.790, "background_candidate"),
     Aoi("bois-cambre-hard", "brussels", "park-hard-negative", "background-test", 4.375, 50.795, "background_candidate"),
 )
 
@@ -100,7 +106,8 @@ def bbox_for_center(lon: float, lat: float, side_m: float) -> dict[str, Any]:
 
 def post(session: requests.Session, url: str, payload: dict[str, Any]) -> dict[str, Any]:
     response = session.post(url, json=payload, timeout=180)
-    response.raise_for_status()
+    if not response.ok:
+        raise RuntimeError(f"Provider workflow HTTP {response.status_code}: {response.text[:1000]}")
     body = response.json()
     if body.get("error"):
         raise RuntimeError(f"{body['error']}: {body.get('message')}")
@@ -121,7 +128,17 @@ def main() -> int:
     args = parser.parse_args()
     session = requests.Session()
     samples: list[dict[str, Any]] = []
+    if args.output_spec.is_file():
+        existing = json.loads(args.output_spec.read_text(encoding="utf-8-sig"))
+        samples = list(existing.get("samples") or [])
+        for sample in samples:
+            if "sample_slug" not in sample and sample.get("slug"):
+                sample["sample_slug"] = sample.pop("slug")
+    completed = {str(sample.get("sample_slug") or sample.get("slug")) for sample in samples}
     for aoi in AOIS:
+        if aoi.slug in completed:
+            print(f"{aoi.slug}: checkpoint reused", flush=True)
+            continue
         contract = REGION_CONTRACT[aoi.region]
         bbox = bbox_for_center(aoi.lon, aoi.lat, args.side_m)
         common = {"bbox": bbox, "area_id": contract["area_id"], "force_refresh": args.force_refresh}
@@ -140,18 +157,33 @@ def main() -> int:
             raise RuntimeError(f"Pure-background AOI {aoi.slug} contains {feature_count} reference buildings")
         samples.append(
             {
-                **asdict(aoi),
+                "sample_slug": aoi.slug,
+                "region": aoi.region,
+                "context": aoi.context,
+                "split": aoi.split,
+                "sample_role": aoi.sample_role,
+                "require_empty": aoi.require_empty,
                 "bbox_epsg4326": [bbox["min_x"], bbox["min_y"], bbox["max_x"], bbox["max_y"]],
                 "raster_dataset_id": image_job["output_dataset_id"],
                 "reference_dataset_id": reference_job["output_dataset_id"],
                 "provider_reference_feature_count": feature_count,
             }
         )
+        checkpoint = {
+            "schema_version": 1,
+            "status": "in_progress",
+            "side_m": args.side_m,
+            "resolution_m": args.resolution_m,
+            "samples": samples,
+        }
+        args.output_spec.parent.mkdir(parents=True, exist_ok=True)
+        args.output_spec.write_text(json.dumps(checkpoint, ensure_ascii=False, indent=2), encoding="utf-8")
         print(f"{aoi.slug}: {feature_count} reference buildings", flush=True)
     payload = {
         "schema_version": 1,
         "side_m": args.side_m,
         "resolution_m": args.resolution_m,
+        "status": "complete",
         "samples": samples,
     }
     args.output_spec.parent.mkdir(parents=True, exist_ok=True)