evaluate models on fresh regional calibration AOIs

scripts/assemble_belgium_building_corpus.py

@@ -62,6 +62,7 @@ def _validate_pair(
     reference: Dataset,
     *,
     fixture_mode: bool,
+    evaluation_only_pending_regional_contracts: bool,
 ) -> tuple[str, str, dict[str, Any]]:
     region = str(sample.get("region") or "").lower()
     if region not in REGION_SOURCES:
@@ -89,9 +90,22 @@ def _validate_pair(
                 for reason in eligibility[role]["reasons"]
             }
         )
-        raise SystemExit(
-            f"Dataset pair is not eligible for training for {sample['sample_slug']}: {', '.join(reasons)}"
+        allowed_pending = bool(
+            evaluation_only_pending_regional_contracts
+            and split == "calibration"
+            and region in {"wallonia", "brussels"}
+            and set(reasons) == {"reference_building_validation_not_primary"}
         )
+        if not allowed_pending:
+            raise SystemExit(
+                f"Dataset pair is not eligible for training for {sample['sample_slug']}: {', '.join(reasons)}"
+            )
+        eligibility["evaluation_only_exception"] = {
+            "allowed": True,
+            "reason": "regional_building_authority_contract_pending",
+            "training_allowed": False,
+            "release_claim_allowed": False,
+        }
     return region, expected_reference, eligibility
 
 
@@ -130,6 +144,15 @@ def main() -> int:
     parser.add_argument("--min-label-px", type=float, default=3.0)
     parser.add_argument("--merge-touching-roofs", action="store_true")
     parser.add_argument("--freeze", action="store_true")
+    parser.add_argument(
+        "--evaluation-only-pending-regional-contracts",
+        action="store_true",
+        help=(
+            "Allow calibration-only PICC/UrbIS pairs whose sole training gate "
+            "failure is a pending regional authority contract. Writes an "
+            "explicit NO_TRAINING marker and never makes a release claim."
+        ),
+    )
     parser.add_argument(
         "--fixture-mode",
         action="store_true",
@@ -168,6 +191,9 @@ def main() -> int:
                 raster,
                 reference,
                 fixture_mode=args.fixture_mode,
+                evaluation_only_pending_regional_contracts=(
+                    args.evaluation_only_pending_regional_contracts
+                ),
             )
             raster_source = _dataset_path(raster)
             reference_source_path = _dataset_path(reference)
@@ -223,9 +249,18 @@ def main() -> int:
         "immutable": bool(args.freeze),
         "training_eligibility": {
             "policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION,
-            "status": "eligible",
+            "status": (
+                "not_eligible_evaluation_only"
+                if args.evaluation_only_pending_regional_contracts
+                else "eligible"
+            ),
             "fixture_mode": bool(args.fixture_mode),
         },
+        "purpose": (
+            "non_protected_diagnostic_evaluation"
+            if args.evaluation_only_pending_regional_contracts
+            else "training_corpus"
+        ),
         "samples": manifest_samples,
     }
     manifest_path = output_dir / "operator_samples_manifest.json"
@@ -244,8 +279,21 @@ def main() -> int:
         "immutable": bool(args.freeze),
         "training_eligibility_policy": TRAINING_ELIGIBILITY_POLICY_VERSION,
         "fixture_mode": bool(args.fixture_mode),
+        "training_allowed": not args.evaluation_only_pending_regional_contracts,
+        "release_claim_allowed": False,
     }
     (output_dir / "corpus-freeze.json").write_text(json.dumps(freeze, indent=2), encoding="utf-8")
+    if args.evaluation_only_pending_regional_contracts:
+        marker = {
+            "schema_version": 1,
+            "reason": "evaluation_only_pending_regional_authority_contracts",
+            "training_allowed": False,
+            "release_claim_allowed": False,
+            "manifest_sha256": freeze["manifest_sha256"],
+        }
+        (output_dir / "NO_TRAINING.json").write_text(
+            json.dumps(marker, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+        )
     print(json.dumps(freeze))
     return 0