feat(provenance): govern source snapshots and data inputs

scripts/export_operator_yolo_tile_dataset.py

@@ -18,6 +18,19 @@ import sys
 from pathlib import Path
 from typing import Any, Iterable
 
+SCRIPT_DIR = Path(__file__).resolve().parent
+if str(SCRIPT_DIR) not in sys.path:
+    sys.path.insert(0, str(SCRIPT_DIR))
+
+from training_dataset_eligibility import (  # noqa: E402
+    TrainingEligibilityError,
+    assert_frozen_manifest_training_eligible,
+)
+from training_release_manifest import (  # noqa: E402
+    TrainingReleaseError,
+    create_training_release_manifest,
+)
+
 
 DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/operator_samples_manifest.json")
 DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-tile-dataset")
@@ -45,6 +58,14 @@ Transformer: Any = None
 Image: Any = None
 
 
+def file_sha256(path: Path) -> str:
+    digest = hashlib.sha256()
+    with path.open("rb") as stream:
+        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
 @dataclass(frozen=True)
 class TileWindow:
     row_off: int
@@ -87,6 +108,12 @@ def parse_args() -> argparse.Namespace:
         default=os.environ.get("OPERATOR_YOLO_REFERENCE_SOURCE", DEFAULT_REFERENCE_SOURCE),
         help="Required source_name in reference GeoJSON features.",
     )
+    parser.add_argument(
+        "--review-audit",
+        type=Path,
+        required=True,
+        help="Passed corpus audit with accepted human-review evidence for this frozen source manifest.",
+    )
     parser.add_argument(
         "--reference-layer",
         default=os.environ.get("OPERATOR_YOLO_REFERENCE_LAYER", DEFAULT_REFERENCE_LAYER),
@@ -626,13 +653,17 @@ def main() -> int:
     ):
         if not value or any(character not in "abcdefghijklmnopqrstuvwxyz0123456789_-" for character in value):
             raise SystemExit(f"YOLO {label} must be a non-empty canonical slug")
+    manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
+    try:
+        assert_frozen_manifest_training_eligible(args.manifest_path, verify_live=True)
+    except TrainingEligibilityError as exc:
+        raise SystemExit(str(exc)) from exc
     ensure_dependencies()
     if args.force and args.output_dir.exists():
         shutil.rmtree(args.output_dir)
     args.output_dir.mkdir(parents=True, exist_ok=True)
     ensure_yolo_directories(args.output_dir)
 
-    manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
     manifest_samples = manifest.get("samples") or []
     if not manifest_samples:
         raise SystemExit("Operator sample manifest contains no samples")
@@ -672,6 +703,14 @@ def main() -> int:
     if not args.allow_empty_validation and not any(tile["split"] == "val" for tile in kept_tiles):
         raise SystemExit("YOLO tile dataset export produced no validation tiles")
     dataset_yaml = write_dataset_yaml(args.output_dir, class_name)
+    try:
+        release_paths = create_training_release_manifest(
+            train_yaml=dataset_yaml,
+            corpus_manifest=args.manifest_path,
+            review_audit_path=args.review_audit,
+        )
+    except TrainingReleaseError as exc:
+        raise SystemExit(str(exc)) from exc
     positive_tiles = [tile for tile in kept_tiles if not tile["is_negative"]]
     negative_tiles = [tile for tile in kept_tiles if tile["is_negative"]]
     skipped_negative_tiles = [tile for tile in exported_tiles if not tile["kept"] and tile["is_negative"]]
@@ -684,6 +723,10 @@ def main() -> int:
     summary = {
         "status": "ok",
         "dataset_yaml": str(dataset_yaml),
+        "training_release_manifest": str(release_paths["release_manifest"]),
+        "training_release_manifest_sha256": file_sha256(release_paths["release_manifest"]),
+        "training_release_freeze": str(release_paths["release_freeze"]),
+        "training_asset_manifest": str(release_paths["asset_manifest"]),
         "output_dir": str(args.output_dir),
         "class_names": [class_name],
         "reference_source": reference_source,
@@ -697,6 +740,8 @@ def main() -> int:
         "drop_low_variance_negatives": args.drop_low_variance_negatives,
         "blank_range_threshold": args.blank_range_threshold,
         "source_manifest_sample_count": len(manifest_samples),
+        "source_manifest": str(args.manifest_path),
+        "source_manifest_sha256": file_sha256(args.manifest_path),
         "source_sample_count": len(samples),
         "selected_sample_slugs": sorted(
             str(sample.get("sample_slug") or "").strip().lower() for sample in samples