feat(provenance): govern source snapshots and data inputs

scripts/build_failure_driven_yolo_sampling.py

@@ -8,10 +8,25 @@ import hashlib
 import json
 import math
 import re
+import sys
 from collections import Counter
 from pathlib import Path
 from typing import Any
 
+SCRIPT_DIR = Path(__file__).resolve().parent
+if str(SCRIPT_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPT_DIR))
+
+from training_dataset_eligibility import (  # noqa: E402
+    TrainingEligibilityError,
+    assert_frozen_manifest_training_eligible,
+)
+from training_release_manifest import (  # noqa: E402
+    TrainingReleaseError,
+    assert_yolo_summary_bound_to_embedded_training_release,
+    create_training_release_manifest,
+)
+
 
 PRECISION_NEGATIVE_CONTEXTS = {
     "coastal-urban": {"port-hard-negative", "dunes-negative"},
@@ -80,6 +95,17 @@ def build_sampling(
 ) -> tuple[list[str], dict[str, Any]]:
     if assessment.get("status") != "continue_training_loop":
         raise ValueError("Failure-driven sampling requires a failed assessment")
+    protected_feedback = [
+        role
+        for role in ("test", "background")
+        if assessment.get(role) is not None
+    ]
+    if protected_feedback:
+        raise ValueError(
+            "Failure-driven sampling is prohibited after protected "
+            + "/".join(protected_feedback)
+            + " evidence was opened"
+        )
     if min(
         positive_repeat,
         negative_repeat,
@@ -97,9 +123,9 @@ def build_sampling(
 
     samples = {item["sample_slug"]: item for item in manifest["samples"]}
     gates = assessment["gates"]
-    evaluation = assessment.get("test") or assessment.get("calibration")
+    evaluation = assessment.get("calibration")
     if not evaluation or "regions" not in evaluation:
-        raise ValueError("Assessment has no regional calibration or test evidence")
+        raise ValueError("Assessment has no regional calibration evidence")
     regions = evaluation["regions"]
     weak_recall_regions = {
         region
@@ -246,7 +272,7 @@ def build_sampling(
         "schema_version": 1,
         "status": "ok",
         "strategy": "failed-region-positive-and-hard-negative-repeat",
-        "failure_evidence_source": "test" if assessment.get("test") else "calibration",
+        "failure_evidence_source": "calibration",
         "weak_recall_regions": sorted(weak_recall_regions),
         "weak_precision_regions": sorted(weak_precision_regions),
         "recall_dominant_regions": sorted(recall_dominant_regions),
@@ -290,6 +316,11 @@ def main() -> int:
     parser.add_argument("--corpus-manifest", type=Path, required=True)
     parser.add_argument("--assessment", type=Path, required=True)
     parser.add_argument("--output-dir", type=Path, required=True)
+    parser.add_argument(
+        "--review-audit",
+        type=Path,
+        help="Passed corpus audit containing accepted human-review evidence for the frozen corpus.",
+    )
     parser.add_argument("--positive-repeat", type=int, default=3)
     parser.add_argument("--negative-repeat", type=int, default=4)
     parser.add_argument("--precision-positive-repeat", type=int, default=1)
@@ -299,8 +330,26 @@ def main() -> int:
     parser.add_argument("--sampling-round", type=int)
     parser.add_argument("--precision-guard-band", type=float, default=0.03)
     parser.add_argument("--recall-guard-band", type=float, default=0.03)
+    parser.add_argument(
+        "--fixture-mode",
+        action="store_true",
+        help="Accept only an explicitly fixture-only corpus manifest; never use for operational sampling.",
+    )
     args = parser.parse_args()
 
+    try:
+        assert_frozen_manifest_training_eligible(
+            args.corpus_manifest,
+            fixture_mode=args.fixture_mode,
+            verify_live=True,
+        )
+        source_release = assert_yolo_summary_bound_to_embedded_training_release(
+            summary_path=args.summary,
+            corpus_manifest=args.corpus_manifest,
+            fixture_mode=args.fixture_mode,
+        )
+    except (TrainingEligibilityError, TrainingReleaseError) as exc:
+        raise SystemExit(str(exc)) from exc
     summary = json.loads(args.summary.read_text(encoding="utf-8"))
     manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
     assessment = json.loads(args.assessment.read_text(encoding="utf-8"))
@@ -325,7 +374,7 @@ def main() -> int:
     args.output_dir.mkdir(parents=True, exist_ok=True)
     train_list = args.output_dir / "train-failure-driven.txt"
     train_list.write_text("\n".join(paths) + "\n", encoding="utf-8")
-    source_yaml = args.summary.parent / "dataset.yaml"
+    source_yaml = Path(str(source_release["dataset_yaml"]["path"]))
     val_source = dataset_validation_source(source_yaml)
     dataset_yaml = args.output_dir / "dataset.yaml"
     dataset_yaml.write_text(
@@ -335,6 +384,15 @@ def main() -> int:
         "names:\n  0: building\n",
         encoding="utf-8",
     )
+    try:
+        release_paths = create_training_release_manifest(
+            train_yaml=dataset_yaml,
+            corpus_manifest=args.corpus_manifest,
+            review_audit_path=args.review_audit,
+            fixture_mode=args.fixture_mode,
+        )
+    except TrainingReleaseError as exc:
+        raise SystemExit(str(exc)) from exc
     metadata.update(
         {
             "summary": str(args.summary),
@@ -346,6 +404,11 @@ def main() -> int:
             "source_dataset_yaml": str(source_yaml),
             "train_list": str(train_list),
             "dataset_yaml": str(dataset_yaml),
+            "training_release_manifest": str(release_paths["release_manifest"]),
+            "training_release_manifest_sha256": file_sha256(release_paths["release_manifest"]),
+            "training_release_freeze": str(release_paths["release_freeze"]),
+            "training_asset_manifest": str(release_paths["asset_manifest"]),
+            "fixture_mode": bool(args.fixture_mode),
         }
     )
     output = args.output_dir / "failure-driven-sampling.json"