evaluate models on fresh regional calibration AOIs

tests/test_evaluate_yolo_checkpoint_matrix.py

@@ -6,6 +6,7 @@ from scripts.evaluate_yolo_checkpoint_matrix import (
     dataset_overlap_evidence,
     main,
     model_lineage_independence_evidence,
+    validate_pure_background_prefixes,
     write_blocked_manifest,
 )
 
@@ -196,3 +197,32 @@ def test_cli_blocks_lineage_validation_exposure_before_model_resolution(
     payload = json.loads(output.read_text(encoding="utf-8"))
     assert payload["status"] == "blocked_model_lineage_sample_exposure"
     assert payload["model_loading_attempted"] is False
+
+
+def test_pure_background_prefix_rejects_nonempty_labels(tmp_path: Path) -> None:
+    image_dir = tmp_path / "images" / "val"
+    label_dir = tmp_path / "labels" / "val"
+    image_dir.mkdir(parents=True)
+    label_dir.mkdir(parents=True)
+    image = image_dir / "sparse_bg_0001.png"
+    image.write_bytes(b"image")
+    (label_dir / "sparse_bg_0001.txt").write_text(
+        "0 0.5 0.5 0.1 0.1\n", encoding="utf-8"
+    )
+
+    import pytest
+
+    with pytest.raises(ValueError, match="non-empty labels"):
+        validate_pure_background_prefixes([image], ("sparse_bg",))
+
+
+def test_pure_background_prefix_accepts_empty_labels(tmp_path: Path) -> None:
+    image_dir = tmp_path / "images" / "val"
+    label_dir = tmp_path / "labels" / "val"
+    image_dir.mkdir(parents=True)
+    label_dir.mkdir(parents=True)
+    image = image_dir / "pure_bg_0001.png"
+    image.write_bytes(b"image")
+    (label_dir / "pure_bg_0001.txt").write_text("", encoding="utf-8")
+
+    assert validate_pure_background_prefixes([image], ("pure_bg",)) == [image]

tests/test_export_yolo_diagnostic_evaluation_tiles.py

@@ -0,0 +1,74 @@
+import json
+from pathlib import Path
+
+import pytest
+
+from scripts.export_yolo_diagnostic_evaluation_tiles import (
+    is_canonical_evaluation_window,
+    validate_diagnostic_manifest,
+)
+
+
+def _write_manifest(tmp_path: Path, *, split: str = "calibration") -> Path:
+    manifest = tmp_path / "operator_samples_manifest.json"
+    manifest.write_text(
+        json.dumps(
+            {
+                "purpose": "non_protected_diagnostic_evaluation",
+                "training_eligibility": {
+                    "status": "not_eligible_evaluation_only"
+                },
+                "samples": [
+                    {
+                        "sample_slug": "fresh-aoi",
+                        "split": split,
+                        "sample_role": "positive",
+                    }
+                ],
+            }
+        ),
+        encoding="utf-8",
+    )
+    import hashlib
+
+    digest = hashlib.sha256(manifest.read_bytes()).hexdigest()
+    (tmp_path / "NO_TRAINING.json").write_text(
+        json.dumps({"training_allowed": False, "manifest_sha256": digest}),
+        encoding="utf-8",
+    )
+    return manifest
+
+
+def test_validate_diagnostic_manifest_accepts_bound_calibration(tmp_path: Path) -> None:
+    manifest = _write_manifest(tmp_path)
+
+    payload = validate_diagnostic_manifest(manifest)
+
+    assert payload["samples"][0]["sample_slug"] == "fresh-aoi"
+
+
+def test_validate_diagnostic_manifest_rejects_training_split(tmp_path: Path) -> None:
+    manifest = _write_manifest(tmp_path, split="train")
+
+    with pytest.raises(ValueError, match="calibration-only"):
+        validate_diagnostic_manifest(manifest)
+
+
+def test_validate_diagnostic_manifest_rejects_unbound_marker(tmp_path: Path) -> None:
+    manifest = _write_manifest(tmp_path)
+    (tmp_path / "NO_TRAINING.json").write_text(
+        json.dumps({"training_allowed": False, "manifest_sha256": "0" * 64}),
+        encoding="utf-8",
+    )
+
+    with pytest.raises(ValueError, match="not bound"):
+        validate_diagnostic_manifest(manifest)
+
+
+def test_canonical_evaluation_window_rejects_overlapping_edge_cover() -> None:
+    assert is_canonical_evaluation_window(
+        {"row_off": 512, "col_off": 512, "height": 512, "width": 512}, 512
+    )
+    assert not is_canonical_evaluation_window(
+        {"row_off": 521, "col_off": 512, "height": 512, "width": 512}, 512
+    )

tests/test_provision_belgium_building_training_portfolio.py

@@ -1,4 +1,9 @@
-from scripts.provision_belgium_building_training_portfolio import AOIS
+import json
+from pathlib import Path
+
+import pytest
+
+from scripts.provision_belgium_building_training_portfolio import AOIS, load_custom_aois
 
 
 V63_CANDIDATES = {
@@ -32,3 +37,57 @@ def test_v66_lowrise_candidates_are_flemish_train_role_inputs() -> None:
     assert all(aoi.region == "flanders" for aoi in candidates.values())
     assert all(aoi.split == "train" for aoi in candidates.values())
     assert all("lowrise" in aoi.context for aoi in candidates.values())
+
+
+def test_custom_aoi_spec_loads_non_protected_calibration(tmp_path: Path) -> None:
+    spec = tmp_path / "aois.json"
+    spec.write_text(
+        json.dumps(
+            {
+                "aois": [
+                    {
+                        "slug": "fresh-rural-cal",
+                        "region": "wallonia",
+                        "context": "rural-lowrise",
+                        "split": "calibration",
+                        "lon": 5.25,
+                        "lat": 50.25,
+                    }
+                ]
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    result = load_custom_aois(spec)
+
+    assert len(result) == 1
+    assert result[0].slug == "fresh-rural-cal"
+    assert result[0].split == "calibration"
+
+
+@pytest.mark.parametrize("split", ["test", "background-test"])
+def test_custom_aoi_spec_rejects_protected_splits(
+    tmp_path: Path, split: str
+) -> None:
+    spec = tmp_path / "aois.json"
+    spec.write_text(
+        json.dumps(
+            {
+                "aois": [
+                    {
+                        "slug": "forbidden",
+                        "region": "flanders",
+                        "context": "urban",
+                        "split": split,
+                        "lon": 4.5,
+                        "lat": 51.0,
+                    }
+                ]
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    with pytest.raises(SystemExit, match="protected/unsupported split"):
+        load_custom_aois(spec)