feat(provenance): govern source snapshots and data inputs

This commit is contained in:
Jens
2026-08-01 23:46:17 +02:00
parent cebeb5f3b4
commit 5b3c17b494
96 changed files with 20156 additions and 351 deletions
+50 -1
View File
@@ -10,8 +10,20 @@ import json
import math
import os
import shutil
import sys
from pathlib import Path
from typing import Any
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_training_release,
file_sha256,
training_release_paths,
)
def sha256(path: Path) -> str:
@@ -97,6 +109,8 @@ def link_or_copy(source: Path, target: Path) -> None:
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--summary", type=Path, required=True)
parser.add_argument("--train-yaml", type=Path, required=True)
parser.add_argument("--corpus-manifest", type=Path, required=True)
parser.add_argument("--model", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--device", default="cuda:0")
@@ -110,7 +124,21 @@ def main() -> int:
parser.add_argument("--max-prompts-per-pass", type=int, default=96)
parser.add_argument("--fallback-policy", choices=("retain", "drop"), default="retain")
parser.add_argument("--force", action="store_true")
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Accept only an explicitly fixture-only source release; refined labels remain non-trainable.",
)
args = parser.parse_args()
try:
source_release = assert_yolo_summary_bound_to_training_release(
summary_path=args.summary,
train_yaml=args.train_yaml,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
if args.output_dir.exists():
if not args.force:
raise SystemExit(f"Output exists: {args.output_dir}")
@@ -194,6 +222,16 @@ def main() -> int:
print(f"{index}/{len(summary['tiles'])} {tile['sample_slug']}: {len(source_boxes)}", flush=True)
output_summary = dict(summary)
# SAM changes label bytes and semantics. It therefore cannot inherit the
# source release; keep the parent evidence separately and require a new
# governed corpus/review/release before any training consumer can use it.
for field_name in (
"training_release_manifest",
"training_release_manifest_sha256",
"training_release_freeze",
"training_asset_manifest",
):
output_summary.pop(field_name, None)
output_summary.update(
{
"output_dir": str(args.output_dir),
@@ -207,6 +245,10 @@ def main() -> int:
"label_count": refined_count if args.fallback_policy == "drop" else summary["label_count"],
"positive_tile_count": sum(not tile["is_negative"] for tile in output_tiles),
"negative_tile_count": sum(tile["is_negative"] for tile in output_tiles),
"training_eligible": False,
"training_eligibility_reason": (
"SAM-refined labels require a new governed corpus, validation report, human review and immutable training release."
),
}
)
summary_path = args.output_dir / "yolo_tile_dataset_summary.json"
@@ -220,6 +262,11 @@ def main() -> int:
"status": "ok",
"source_summary": str(args.summary),
"source_summary_sha256": sha256(args.summary),
"source_training_release": str(training_release_paths(args.train_yaml)["release_manifest"]),
"source_training_release_sha256": file_sha256(
training_release_paths(args.train_yaml)["release_manifest"]
),
"source_corpus_manifest_sha256": source_release["corpus"]["manifest_sha256"],
"sam_model": str(args.model),
"sam_model_sha256": sha256(args.model),
"device": args.device,
@@ -236,6 +283,8 @@ def main() -> int:
"fallback_label_count": fallback_count,
"dropped_fallback_label_count": fallback_count if args.fallback_policy == "drop" else 0,
"fallback_reason_counts": reason_counts,
"training_eligible": False,
"fixture_mode": bool(args.fixture_mode),
}
(args.output_dir / "sam-refinement.json").write_text(json.dumps(evidence, indent=2), encoding="utf-8")
print(json.dumps(evidence, indent=2))