block training-seen checkpoint evaluation

tests/test_evaluate_yolo_checkpoint_matrix.py

@@ -1,7 +1,11 @@
 import json
 from pathlib import Path
 
-from scripts.evaluate_yolo_checkpoint_matrix import dataset_overlap_evidence
+from scripts.evaluate_yolo_checkpoint_matrix import (
+    dataset_overlap_evidence,
+    training_sample_independence_evidence,
+    write_blocked_manifest,
+)
 
 
 def test_dataset_overlap_evidence_marks_repeated_validation_rows(
@@ -45,3 +49,107 @@ def test_nonoverlap_does_not_overclaim_statistical_independence(
     assert evidence["validation_tiles_non_overlapping"] is True
     assert evidence["statistical_independence_established"] is False
     assert "not established" in evidence["interpretation"]
+
+
+def _write_summary(path: Path, rows: list[tuple[str, str]]) -> None:
+    path.write_text(
+        json.dumps(
+            {
+                "tile_size": 512,
+                "stride": 512,
+                "tiles": [
+                    {"sample_slug": sample_slug, "split": split}
+                    for sample_slug, split in rows
+                ],
+            }
+        ),
+        encoding="utf-8",
+    )
+
+
+def test_training_sample_independence_detects_overlap_for_each_corpus(
+    tmp_path: Path,
+) -> None:
+    evaluation = tmp_path / "evaluation"
+    evaluation.mkdir()
+    dataset_yaml = evaluation / "dataset.yaml"
+    dataset_yaml.write_text("path: .\nval: images/val\n", encoding="utf-8")
+    _write_summary(
+        evaluation / "yolo_tile_dataset_summary.json",
+        [("turnhout", "val"), ("postel_bos", "val")],
+    )
+    active = tmp_path / "active.json"
+    challenger = tmp_path / "challenger.json"
+    _write_summary(active, [("postel_bos", "train"), ("mol", "train")])
+    _write_summary(
+        challenger, [("postel_bos", "train"), ("dessel", "train")]
+    )
+
+    evidence = training_sample_independence_evidence(
+        dataset_yaml, [active, challenger]
+    )
+
+    assert evidence["status"] == "overlap"
+    assert evidence["overlapping_evaluation_samples"] == ["postel_bos"]
+    assert evidence["independent_for_all_supplied_training_corpora"] is False
+    assert [
+        row["overlapping_evaluation_samples"] for row in evidence["training_corpora"]
+    ] == [["postel_bos"], ["postel_bos"]]
+
+
+def test_training_sample_independence_accepts_disjoint_samples(tmp_path: Path) -> None:
+    evaluation = tmp_path / "evaluation"
+    evaluation.mkdir()
+    dataset_yaml = evaluation / "dataset.yaml"
+    dataset_yaml.write_text("path: .\nval: images/val\n", encoding="utf-8")
+    _write_summary(
+        evaluation / "yolo_tile_dataset_summary.json", [("turnhout", "val")]
+    )
+    training = tmp_path / "training.json"
+    _write_summary(training, [("mol", "train")])
+
+    evidence = training_sample_independence_evidence(dataset_yaml, [training])
+
+    assert evidence["status"] == "independent"
+    assert evidence["overlapping_evaluation_samples"] == []
+    assert evidence["independent_for_all_supplied_training_corpora"] is True
+
+
+def test_training_sample_independence_is_unavailable_without_training_corpus(
+    tmp_path: Path,
+) -> None:
+    evaluation = tmp_path / "evaluation"
+    evaluation.mkdir()
+    dataset_yaml = evaluation / "dataset.yaml"
+    dataset_yaml.write_text("path: .\nval: images/val\n", encoding="utf-8")
+    _write_summary(
+        evaluation / "yolo_tile_dataset_summary.json", [("turnhout", "val")]
+    )
+
+    evidence = training_sample_independence_evidence(dataset_yaml, [])
+
+    assert evidence["status"] == "unavailable"
+    assert evidence["overlapping_evaluation_samples"] == []
+    assert evidence["independent_for_all_supplied_training_corpora"] is False
+
+
+def test_blocked_manifest_records_that_models_and_gpu_were_not_used(
+    tmp_path: Path,
+) -> None:
+    dataset_yaml = tmp_path / "dataset.yaml"
+    dataset_yaml.write_text("path: .\nval: images/val\n", encoding="utf-8")
+    output = tmp_path / "evidence" / "matrix.json"
+
+    write_blocked_manifest(
+        output,
+        dataset_yaml,
+        {
+            "status": "overlap",
+            "independent_for_all_supplied_training_corpora": False,
+        },
+    )
+
+    payload = json.loads(output.read_text(encoding="utf-8"))
+    assert payload["status"] == "blocked_training_sample_overlap"
+    assert payload["model_loading_attempted"] is False
+    assert payload["gpu_inference_attempted"] is False