feat(provenance): govern source snapshots and data inputs
This commit is contained in:
@@ -7,11 +7,24 @@ import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from training_release_manifest import ( # noqa: E402
|
||||
TrainingReleaseError,
|
||||
assert_yolo_summary_bound_to_training_release,
|
||||
file_sha256,
|
||||
training_release_paths,
|
||||
)
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
digest = hashlib.sha256()
|
||||
with path.open("rb") as stream:
|
||||
@@ -23,9 +36,25 @@ def sha256(path: Path) -> str:
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--summary", type=Path, required=True)
|
||||
parser.add_argument("--train-yaml", type=Path, required=True)
|
||||
parser.add_argument("--corpus-manifest", type=Path, required=True)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--fixture-mode",
|
||||
action="store_true",
|
||||
help="Accept only an explicitly fixture-only release; never creates a production-ready derived dataset.",
|
||||
)
|
||||
parser.add_argument("--force", action="store_true")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
release = assert_yolo_summary_bound_to_training_release(
|
||||
summary_path=args.summary,
|
||||
train_yaml=args.train_yaml,
|
||||
corpus_manifest=args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
except TrainingReleaseError as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
if args.output_dir.exists():
|
||||
if not args.force:
|
||||
raise SystemExit(f"Output exists: {args.output_dir}")
|
||||
@@ -54,6 +83,16 @@ def main() -> int:
|
||||
f"path: {args.output_dir}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
# The source release binds the original image bytes. These transformed
|
||||
# bytes cannot inherit it, so remove its operational release pointers and
|
||||
# retain them only as explicitly non-trainable parent evidence below.
|
||||
for field_name in (
|
||||
"training_release_manifest",
|
||||
"training_release_manifest_sha256",
|
||||
"training_release_freeze",
|
||||
"training_asset_manifest",
|
||||
):
|
||||
summary.pop(field_name, None)
|
||||
summary.update(
|
||||
{
|
||||
"output_dir": str(args.output_dir),
|
||||
@@ -71,10 +110,19 @@ def main() -> int:
|
||||
"preprocessing": "luminance_rgb_replicated",
|
||||
"source_summary": str(args.summary),
|
||||
"source_summary_sha256": sha256(args.summary),
|
||||
"source_training_release": str(training_release_paths(args.train_yaml)["release_manifest"]),
|
||||
"source_training_release_sha256": file_sha256(
|
||||
training_release_paths(args.train_yaml)["release_manifest"]
|
||||
),
|
||||
"source_corpus_manifest_sha256": release["corpus"]["manifest_sha256"],
|
||||
"converted_tile_count": converted,
|
||||
"output_summary": str(output_summary),
|
||||
"output_summary_sha256": sha256(output_summary),
|
||||
"dataset_yaml": str(dataset_yaml),
|
||||
"training_eligible": False,
|
||||
"training_eligibility_reason": (
|
||||
"Derived image bytes require a new governed corpus, validation report and immutable training release."
|
||||
),
|
||||
}
|
||||
(args.output_dir / "grayscale-dataset-evidence.json").write_text(
|
||||
json.dumps(evidence, indent=2), encoding="utf-8"
|
||||
|
||||
Reference in New Issue
Block a user