evaluate models on fresh regional calibration AOIs

scripts/provision_belgium_building_training_portfolio.py

@@ -267,6 +267,62 @@ REGION_CONTRACT = {
     },
 }
 
+NON_PROTECTED_SPLITS = {"train", "val", "calibration"}
+
+
+def load_custom_aois(path: Path) -> tuple[Aoi, ...]:
+    payload = json.loads(path.read_text(encoding="utf-8-sig"))
+    rows = payload.get("aois")
+    if not isinstance(rows, list) or not rows:
+        raise SystemExit("--aoi-spec requires a non-empty aois list")
+    result: list[Aoi] = []
+    seen: set[str] = set()
+    for row in rows:
+        if not isinstance(row, dict):
+            raise SystemExit("--aoi-spec entries must be objects")
+        slug = str(row.get("slug") or "").strip()
+        region = str(row.get("region") or "").strip().lower()
+        split = str(row.get("split") or "").strip().lower()
+        context = str(row.get("context") or "").strip()
+        role = str(row.get("sample_role") or "positive").strip()
+        if not slug or slug in seen:
+            raise SystemExit(f"Custom AOI slug is missing or duplicated: {slug}")
+        if region not in REGION_CONTRACT:
+            raise SystemExit(f"Unsupported custom AOI region for {slug}: {region}")
+        if split not in NON_PROTECTED_SPLITS:
+            raise SystemExit(
+                f"Custom AOI {slug} uses protected/unsupported split {split!r}; "
+                "fresh provisioning is limited to train, val and calibration"
+            )
+        if not context:
+            raise SystemExit(f"Custom AOI context is missing: {slug}")
+        try:
+            lon = float(row["lon"])
+            lat = float(row["lat"])
+        except (KeyError, TypeError, ValueError) as exc:
+            raise SystemExit(f"Custom AOI coordinates are invalid: {slug}") from exc
+        if not 2.4 <= lon <= 6.5 or not 49.4 <= lat <= 51.7:
+            raise SystemExit(f"Custom AOI is outside the Belgium workbench: {slug}")
+        if role not in {"positive", "background_candidate"}:
+            raise SystemExit(f"Unsupported custom AOI sample_role for {slug}: {role}")
+        require_empty = bool(row.get("require_empty", False))
+        if require_empty and role != "background_candidate":
+            raise SystemExit(f"Only background candidates may require empty labels: {slug}")
+        seen.add(slug)
+        result.append(
+            Aoi(
+                slug,
+                region,
+                context,
+                split,
+                lon,
+                lat,
+                role,
+                require_empty,
+            )
+        )
+    return tuple(result)
+
 
 def bbox_for_center(lon: float, lat: float, side_m: float) -> dict[str, Any]:
     to_metric = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
@@ -298,6 +354,14 @@ def main() -> int:
     parser.add_argument("--side-m", type=float, default=256.0)
     parser.add_argument("--resolution-m", type=float, default=0.25)
     parser.add_argument("--force-refresh", action="store_true")
+    parser.add_argument(
+        "--aoi-spec",
+        type=Path,
+        help=(
+            "Optional JSON file containing fresh non-protected AOIs. Custom "
+            "specs cannot provision test or background-test roles."
+        ),
+    )
     parser.add_argument(
         "--only-slug",
         action="append",
@@ -310,11 +374,16 @@ def main() -> int:
         help="Read an operator session token from a local mode-0600 file instead of exposing it on the command line.",
     )
     args = parser.parse_args()
-    known_slugs = {aoi.slug for aoi in AOIS}
+    if args.aoi_spec and args.only_slug:
+        parser.error("--aoi-spec and --only-slug are mutually exclusive")
+    available_aois = load_custom_aois(args.aoi_spec) if args.aoi_spec else AOIS
+    known_slugs = {aoi.slug for aoi in available_aois}
     unknown_slugs = sorted(set(args.only_slug) - known_slugs)
     if unknown_slugs:
         raise SystemExit(f"Unknown --only-slug value(s): {', '.join(unknown_slugs)}")
-    selected_aois = tuple(aoi for aoi in AOIS if not args.only_slug or aoi.slug in args.only_slug)
+    selected_aois = tuple(
+        aoi for aoi in available_aois if not args.only_slug or aoi.slug in args.only_slug
+    )
     session = requests.Session()
     if args.session_token_file:
         token = args.session_token_file.read_text(encoding="utf-8").strip()