feat(provenance): govern source snapshots and data inputs

This commit is contained in:
Jens
2026-08-01 23:46:17 +02:00
parent cebeb5f3b4
commit 5b3c17b494
96 changed files with 20156 additions and 351 deletions
+48
View File
@@ -7,11 +7,24 @@ import argparse
import hashlib
import json
import shutil
import sys
from pathlib import Path
from PIL import Image
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_training_release,
file_sha256,
training_release_paths,
)
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
@@ -23,9 +36,25 @@ def sha256(path: Path) -> str:
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--summary", type=Path, required=True)
parser.add_argument("--train-yaml", type=Path, required=True)
parser.add_argument("--corpus-manifest", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Accept only an explicitly fixture-only release; never creates a production-ready derived dataset.",
)
parser.add_argument("--force", action="store_true")
args = parser.parse_args()
try:
release = assert_yolo_summary_bound_to_training_release(
summary_path=args.summary,
train_yaml=args.train_yaml,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
if args.output_dir.exists():
if not args.force:
raise SystemExit(f"Output exists: {args.output_dir}")
@@ -54,6 +83,16 @@ def main() -> int:
f"path: {args.output_dir}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
encoding="utf-8",
)
# The source release binds the original image bytes. These transformed
# bytes cannot inherit it, so remove its operational release pointers and
# retain them only as explicitly non-trainable parent evidence below.
for field_name in (
"training_release_manifest",
"training_release_manifest_sha256",
"training_release_freeze",
"training_asset_manifest",
):
summary.pop(field_name, None)
summary.update(
{
"output_dir": str(args.output_dir),
@@ -71,10 +110,19 @@ def main() -> int:
"preprocessing": "luminance_rgb_replicated",
"source_summary": str(args.summary),
"source_summary_sha256": sha256(args.summary),
"source_training_release": str(training_release_paths(args.train_yaml)["release_manifest"]),
"source_training_release_sha256": file_sha256(
training_release_paths(args.train_yaml)["release_manifest"]
),
"source_corpus_manifest_sha256": release["corpus"]["manifest_sha256"],
"converted_tile_count": converted,
"output_summary": str(output_summary),
"output_summary_sha256": sha256(output_summary),
"dataset_yaml": str(dataset_yaml),
"training_eligible": False,
"training_eligibility_reason": (
"Derived image bytes require a new governed corpus, validation report and immutable training release."
),
}
(args.output_dir / "grayscale-dataset-evidence.json").write_text(
json.dumps(evidence, indent=2), encoding="utf-8"