block checkpoint evaluation on ancestral exposure

scripts/evaluate_yolo_checkpoint_matrix.py

@@ -111,8 +111,8 @@ def _sample_slugs(payload: dict[str, Any], *, split: str | None) -> set[str]:
     return samples
 
 
-def training_sample_independence_evidence(
-    dataset_yaml: Path, training_summaries: list[Path]
+def model_lineage_independence_evidence(
+    dataset_yaml: Path, lineage_summaries: list[Path]
 ) -> dict[str, Any]:
     evaluation_summary = dataset_yaml.parent / "yolo_tile_dataset_summary.json"
     if not evaluation_summary.is_file():
@@ -120,7 +120,7 @@ def training_sample_independence_evidence(
             "status": "unavailable",
             "reason": "evaluation dataset summary is unavailable",
             "evaluation_summary_path": str(evaluation_summary),
-            "independent_for_all_supplied_training_corpora": False,
+            "independent_for_all_supplied_lineage_corpora": False,
         }
 
     try:
@@ -134,18 +134,23 @@ def training_sample_independence_evidence(
             "reason": str(exc),
             "evaluation_summary_path": str(evaluation_summary),
             "evaluation_summary_sha256": sha256_file(evaluation_summary),
-            "independent_for_all_supplied_training_corpora": False,
+            "independent_for_all_supplied_lineage_corpora": False,
         }
 
     rows: list[dict[str, Any]] = []
     union_overlap: set[str] = set()
-    for raw_summary in training_summaries:
+    for raw_summary in lineage_summaries:
         summary = raw_summary.expanduser().resolve(strict=True)
         try:
             payload = json.loads(summary.read_text(encoding="utf-8"))
-            training_samples = _sample_slugs(payload, split="train")
-            if not training_samples:
-                raise ValueError("training summary contains no training samples")
+            all_samples = _sample_slugs(payload, split=None)
+            if not all_samples:
+                raise ValueError("lineage summary contains no samples")
+            roles_by_sample: dict[str, set[str]] = {}
+            for tile in payload["tiles"]:
+                sample_slug = str(tile["sample_slug"]).strip()
+                split = str(tile.get("split") or "unknown").strip()
+                roles_by_sample.setdefault(sample_slug, set()).add(split)
         except (json.JSONDecodeError, OSError, ValueError) as exc:
             return {
                 "status": "invalid",
@@ -153,32 +158,35 @@ def training_sample_independence_evidence(
                 "evaluation_summary_path": str(evaluation_summary),
                 "evaluation_summary_sha256": sha256_file(evaluation_summary),
                 "evaluation_samples": sorted(evaluation_samples),
-                "independent_for_all_supplied_training_corpora": False,
+                "independent_for_all_supplied_lineage_corpora": False,
             }
-        overlap = evaluation_samples & training_samples
+        overlap = evaluation_samples & all_samples
         union_overlap.update(overlap)
         rows.append(
             {
-                "training_summary_path": str(summary),
-                "training_summary_sha256": sha256_file(summary),
-                "training_sample_count": len(training_samples),
+                "lineage_summary_path": str(summary),
+                "lineage_summary_sha256": sha256_file(summary),
+                "lineage_sample_count": len(all_samples),
                 "overlapping_evaluation_samples": sorted(overlap),
+                "exposure_roles": {
+                    sample: sorted(roles_by_sample[sample]) for sample in sorted(overlap)
+                },
             }
         )
 
     if not rows:
         return {
             "status": "unavailable",
-            "reason": "no training summaries were supplied",
+            "reason": "no complete model-lineage summaries were supplied",
             "evaluation_summary_path": str(evaluation_summary),
             "evaluation_summary_sha256": sha256_file(evaluation_summary),
             "evaluation_samples": sorted(evaluation_samples),
-            "training_corpora": [],
+            "lineage_corpora": [],
             "overlapping_evaluation_samples": [],
-            "independent_for_all_supplied_training_corpora": False,
+            "independent_for_all_supplied_lineage_corpora": False,
             "interpretation": (
-                "Training/evaluation independence cannot be established "
-                "without exact training summaries."
+                "Model-lineage/evaluation independence cannot be established "
+                "without every ancestral corpus summary."
             ),
         }
 
@@ -188,29 +196,37 @@ def training_sample_independence_evidence(
         "evaluation_summary_path": str(evaluation_summary),
         "evaluation_summary_sha256": sha256_file(evaluation_summary),
         "evaluation_samples": sorted(evaluation_samples),
-        "training_corpora": rows,
+        "lineage_corpora": rows,
         "overlapping_evaluation_samples": sorted(union_overlap),
-        "independent_for_all_supplied_training_corpora": independent,
+        "independent_for_all_supplied_lineage_corpora": independent,
         "interpretation": (
-            "No evaluation AOI occurs in the train split of any supplied corpus."
+            "No evaluation AOI occurs in any split of any supplied ancestral corpus."
             if independent
-            else "At least one evaluation AOI occurs in a supplied train split; "
-            "the matrix is blocked before model loading."
+            else "At least one evaluation AOI was exposed in an ancestral train, "
+            "validation, calibration or other split; the matrix is blocked before "
+            "model loading."
         ),
     }
 
 
+def training_sample_independence_evidence(
+    dataset_yaml: Path, training_summaries: list[Path]
+) -> dict[str, Any]:
+    """Backward-compatible alias; evidence now checks every lineage split."""
+    return model_lineage_independence_evidence(dataset_yaml, training_summaries)
+
+
 def write_blocked_manifest(
     output: Path, dataset_yaml: Path, evidence: dict[str, Any]
 ) -> None:
     payload = {
-        "schema_version": 2,
+        "schema_version": 3,
         "generated_at": datetime.now(UTC).isoformat(),
-        "status": "blocked_training_sample_overlap",
+        "status": "blocked_model_lineage_sample_exposure",
         "claim_boundary": "No checkpoint ranking or release claim is permitted.",
         "dataset_yaml": str(dataset_yaml),
         "dataset_yaml_sha256": sha256_file(dataset_yaml),
-        "training_sample_independence_evidence": evidence,
+        "model_lineage_independence_evidence": evidence,
         "model_loading_attempted": False,
         "gpu_inference_attempted": False,
     }
@@ -261,8 +277,21 @@ def main() -> int:
     parser.add_argument("--output", type=Path, required=True)
     parser.add_argument("--background-prefix", action="append", required=True)
     parser.add_argument("--background-confidence", type=float, default=0.15)
-    parser.add_argument("--training-summary", type=Path, action="append", default=[])
-    parser.add_argument("--require-training-sample-independence", action="store_true")
+    parser.add_argument(
+        "--lineage-summary",
+        "--training-summary",
+        dest="lineage_summary",
+        type=Path,
+        action="append",
+        default=[],
+        help="tile summary for every corpus in the complete model ancestry",
+    )
+    parser.add_argument(
+        "--require-lineage-sample-independence",
+        "--require-training-sample-independence",
+        dest="require_lineage_sample_independence",
+        action="store_true",
+    )
     parser.add_argument("--device", default="cuda:0")
     parser.add_argument("--imgsz", type=int, default=640)
     parser.add_argument("--batch", type=int, default=8)
@@ -275,17 +304,19 @@ def main() -> int:
     dataset_yaml = args.dataset_yaml.expanduser().resolve(strict=True)
     images = validation_images(dataset_yaml)
     overlap_evidence = dataset_overlap_evidence(dataset_yaml)
-    independence_evidence = training_sample_independence_evidence(
-        dataset_yaml, args.training_summary
+    independence_evidence = model_lineage_independence_evidence(
+        dataset_yaml, args.lineage_summary
     )
-    if args.require_training_sample_independence and not args.training_summary:
+    if args.require_lineage_sample_independence and not args.lineage_summary:
         parser.error(
-            "--require-training-sample-independence requires at least one "
-            "--training-summary"
+            "--require-lineage-sample-independence requires every ancestral "
+            "--lineage-summary"
         )
     if (
-        args.require_training_sample_independence
-        and not independence_evidence["independent_for_all_supplied_training_corpora"]
+        args.require_lineage_sample_independence
+        and not independence_evidence[
+            "independent_for_all_supplied_lineage_corpora"
+        ]
     ):
         write_blocked_manifest(args.output, dataset_yaml, independence_evidence)
         return 3
@@ -374,14 +405,14 @@ def main() -> int:
         reverse=True,
     )
     payload = {
-        "schema_version": 2,
+        "schema_version": 3,
         "generated_at": datetime.now(UTC).isoformat(),
         "status": "ok" if successful else "failed",
         "claim_boundary": "Non-protected validation ranking only; no test, challenge or promotion claim.",
         "dataset_yaml": str(dataset_yaml),
         "dataset_yaml_sha256": sha256_file(dataset_yaml),
         "dataset_overlap_evidence": overlap_evidence,
-        "training_sample_independence_evidence": independence_evidence,
+        "model_lineage_independence_evidence": independence_evidence,
         "validation_image_count": len(images),
         "pure_background_prefixes": list(prefixes),
         "pure_background_confidence": args.background_confidence,