block training-seen checkpoint evaluation

scripts/README.md

@@ -5,6 +5,13 @@ weights on one declared, non-protected `val` split using CUDA. It records exact
 model and dataset hashes, standard Ultralytics detection metrics and a separate
 pure-background detection count. The output claim is validation ranking only;
 the script neither reads protected test data nor promotes a model.
+Governed comparisons must pass every candidate's exact tile-summary with
+`--training-summary` and enable `--require-training-sample-independence`. If
+any validation AOI occurs in any supplied train split, the script writes a
+blocked manifest and exits before importing PyTorch, loading a model or using
+the GPU. A background detection count from a training-seen AOI is only a
+regression check and must never be presented as independent background
+evidence.
 
 `render_operator_yolo_label_qa_contact_sheets.py` paginates complete visual
 reviews with `--tiles-per-sheet` (default `64`). This keeps large corpora

scripts/evaluate_yolo_checkpoint_matrix.py

@@ -94,6 +94,132 @@ def dataset_overlap_evidence(dataset_yaml: Path) -> dict[str, Any]:
     }
 
 
+def _sample_slugs(payload: dict[str, Any], *, split: str | None) -> set[str]:
+    tiles = payload.get("tiles")
+    if not isinstance(tiles, list):
+        raise ValueError("dataset summary requires a tiles list")
+    samples: set[str] = set()
+    for tile in tiles:
+        if not isinstance(tile, dict):
+            raise ValueError("dataset summary tiles must contain mappings")
+        if split is not None and tile.get("split") != split:
+            continue
+        sample_slug = tile.get("sample_slug")
+        if not isinstance(sample_slug, str) or not sample_slug.strip():
+            raise ValueError("every selected tile requires a sample_slug")
+        samples.add(sample_slug.strip())
+    return samples
+
+
+def training_sample_independence_evidence(
+    dataset_yaml: Path, training_summaries: list[Path]
+) -> dict[str, Any]:
+    evaluation_summary = dataset_yaml.parent / "yolo_tile_dataset_summary.json"
+    if not evaluation_summary.is_file():
+        return {
+            "status": "unavailable",
+            "reason": "evaluation dataset summary is unavailable",
+            "evaluation_summary_path": str(evaluation_summary),
+            "independent_for_all_supplied_training_corpora": False,
+        }
+
+    try:
+        evaluation_payload = json.loads(evaluation_summary.read_text(encoding="utf-8"))
+        evaluation_samples = _sample_slugs(evaluation_payload, split="val")
+        if not evaluation_samples:
+            raise ValueError("evaluation summary contains no validation samples")
+    except (json.JSONDecodeError, OSError, ValueError) as exc:
+        return {
+            "status": "invalid",
+            "reason": str(exc),
+            "evaluation_summary_path": str(evaluation_summary),
+            "evaluation_summary_sha256": sha256_file(evaluation_summary),
+            "independent_for_all_supplied_training_corpora": False,
+        }
+
+    rows: list[dict[str, Any]] = []
+    union_overlap: set[str] = set()
+    for raw_summary in training_summaries:
+        summary = raw_summary.expanduser().resolve(strict=True)
+        try:
+            payload = json.loads(summary.read_text(encoding="utf-8"))
+            training_samples = _sample_slugs(payload, split="train")
+            if not training_samples:
+                raise ValueError("training summary contains no training samples")
+        except (json.JSONDecodeError, OSError, ValueError) as exc:
+            return {
+                "status": "invalid",
+                "reason": f"{summary}: {exc}",
+                "evaluation_summary_path": str(evaluation_summary),
+                "evaluation_summary_sha256": sha256_file(evaluation_summary),
+                "evaluation_samples": sorted(evaluation_samples),
+                "independent_for_all_supplied_training_corpora": False,
+            }
+        overlap = evaluation_samples & training_samples
+        union_overlap.update(overlap)
+        rows.append(
+            {
+                "training_summary_path": str(summary),
+                "training_summary_sha256": sha256_file(summary),
+                "training_sample_count": len(training_samples),
+                "overlapping_evaluation_samples": sorted(overlap),
+            }
+        )
+
+    if not rows:
+        return {
+            "status": "unavailable",
+            "reason": "no training summaries were supplied",
+            "evaluation_summary_path": str(evaluation_summary),
+            "evaluation_summary_sha256": sha256_file(evaluation_summary),
+            "evaluation_samples": sorted(evaluation_samples),
+            "training_corpora": [],
+            "overlapping_evaluation_samples": [],
+            "independent_for_all_supplied_training_corpora": False,
+            "interpretation": (
+                "Training/evaluation independence cannot be established "
+                "without exact training summaries."
+            ),
+        }
+
+    independent = not union_overlap
+    return {
+        "status": "independent" if independent else "overlap",
+        "evaluation_summary_path": str(evaluation_summary),
+        "evaluation_summary_sha256": sha256_file(evaluation_summary),
+        "evaluation_samples": sorted(evaluation_samples),
+        "training_corpora": rows,
+        "overlapping_evaluation_samples": sorted(union_overlap),
+        "independent_for_all_supplied_training_corpora": independent,
+        "interpretation": (
+            "No evaluation AOI occurs in the train split of any supplied corpus."
+            if independent
+            else "At least one evaluation AOI occurs in a supplied train split; "
+            "the matrix is blocked before model loading."
+        ),
+    }
+
+
+def write_blocked_manifest(
+    output: Path, dataset_yaml: Path, evidence: dict[str, Any]
+) -> None:
+    payload = {
+        "schema_version": 2,
+        "generated_at": datetime.now(UTC).isoformat(),
+        "status": "blocked_training_sample_overlap",
+        "claim_boundary": "No checkpoint ranking or release claim is permitted.",
+        "dataset_yaml": str(dataset_yaml),
+        "dataset_yaml_sha256": sha256_file(dataset_yaml),
+        "training_sample_independence_evidence": evidence,
+        "model_loading_attempted": False,
+        "gpu_inference_attempted": False,
+    }
+    output.parent.mkdir(parents=True, exist_ok=True)
+    output.write_text(
+        json.dumps(payload, indent=2, sort_keys=True) + "\n", encoding="utf-8"
+    )
+
+
 def metric_value(metrics: Any, attribute: str) -> float:
     value = getattr(metrics.box, attribute)
     return float(value)
@@ -135,6 +261,8 @@ def main() -> int:
     parser.add_argument("--output", type=Path, required=True)
     parser.add_argument("--background-prefix", action="append", required=True)
     parser.add_argument("--background-confidence", type=float, default=0.15)
+    parser.add_argument("--training-summary", type=Path, action="append", default=[])
+    parser.add_argument("--require-training-sample-independence", action="store_true")
     parser.add_argument("--device", default="cuda:0")
     parser.add_argument("--imgsz", type=int, default=640)
     parser.add_argument("--batch", type=int, default=8)
@@ -147,6 +275,20 @@ def main() -> int:
     dataset_yaml = args.dataset_yaml.expanduser().resolve(strict=True)
     images = validation_images(dataset_yaml)
     overlap_evidence = dataset_overlap_evidence(dataset_yaml)
+    independence_evidence = training_sample_independence_evidence(
+        dataset_yaml, args.training_summary
+    )
+    if args.require_training_sample_independence and not args.training_summary:
+        parser.error(
+            "--require-training-sample-independence requires at least one "
+            "--training-summary"
+        )
+    if (
+        args.require_training_sample_independence
+        and not independence_evidence["independent_for_all_supplied_training_corpora"]
+    ):
+        write_blocked_manifest(args.output, dataset_yaml, independence_evidence)
+        return 3
     prefixes = tuple(value.casefold() for value in args.background_prefix)
 
     import torch
@@ -232,13 +374,14 @@ def main() -> int:
         reverse=True,
     )
     payload = {
-        "schema_version": 1,
+        "schema_version": 2,
         "generated_at": datetime.now(UTC).isoformat(),
         "status": "ok" if successful else "failed",
         "claim_boundary": "Non-protected validation ranking only; no test, challenge or promotion claim.",
         "dataset_yaml": str(dataset_yaml),
         "dataset_yaml_sha256": sha256_file(dataset_yaml),
         "dataset_overlap_evidence": overlap_evidence,
+        "training_sample_independence_evidence": independence_evidence,
         "validation_image_count": len(images),
         "pure_background_prefixes": list(prefixes),
         "pure_background_confidence": args.background_confidence,