feat(provenance): govern source snapshots and data inputs
This commit is contained in:
@@ -10,8 +10,20 @@ import json
|
||||
import math
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from training_release_manifest import ( # noqa: E402
|
||||
TrainingReleaseError,
|
||||
assert_yolo_summary_bound_to_training_release,
|
||||
file_sha256,
|
||||
training_release_paths,
|
||||
)
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
@@ -97,6 +109,8 @@ def link_or_copy(source: Path, target: Path) -> None:
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--summary", type=Path, required=True)
|
||||
parser.add_argument("--train-yaml", type=Path, required=True)
|
||||
parser.add_argument("--corpus-manifest", type=Path, required=True)
|
||||
parser.add_argument("--model", type=Path, required=True)
|
||||
parser.add_argument("--output-dir", type=Path, required=True)
|
||||
parser.add_argument("--device", default="cuda:0")
|
||||
@@ -110,7 +124,21 @@ def main() -> int:
|
||||
parser.add_argument("--max-prompts-per-pass", type=int, default=96)
|
||||
parser.add_argument("--fallback-policy", choices=("retain", "drop"), default="retain")
|
||||
parser.add_argument("--force", action="store_true")
|
||||
parser.add_argument(
|
||||
"--fixture-mode",
|
||||
action="store_true",
|
||||
help="Accept only an explicitly fixture-only source release; refined labels remain non-trainable.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
source_release = assert_yolo_summary_bound_to_training_release(
|
||||
summary_path=args.summary,
|
||||
train_yaml=args.train_yaml,
|
||||
corpus_manifest=args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
except TrainingReleaseError as exc:
|
||||
raise SystemExit(str(exc)) from exc
|
||||
if args.output_dir.exists():
|
||||
if not args.force:
|
||||
raise SystemExit(f"Output exists: {args.output_dir}")
|
||||
@@ -194,6 +222,16 @@ def main() -> int:
|
||||
print(f"{index}/{len(summary['tiles'])} {tile['sample_slug']}: {len(source_boxes)}", flush=True)
|
||||
|
||||
output_summary = dict(summary)
|
||||
# SAM changes label bytes and semantics. It therefore cannot inherit the
|
||||
# source release; keep the parent evidence separately and require a new
|
||||
# governed corpus/review/release before any training consumer can use it.
|
||||
for field_name in (
|
||||
"training_release_manifest",
|
||||
"training_release_manifest_sha256",
|
||||
"training_release_freeze",
|
||||
"training_asset_manifest",
|
||||
):
|
||||
output_summary.pop(field_name, None)
|
||||
output_summary.update(
|
||||
{
|
||||
"output_dir": str(args.output_dir),
|
||||
@@ -207,6 +245,10 @@ def main() -> int:
|
||||
"label_count": refined_count if args.fallback_policy == "drop" else summary["label_count"],
|
||||
"positive_tile_count": sum(not tile["is_negative"] for tile in output_tiles),
|
||||
"negative_tile_count": sum(tile["is_negative"] for tile in output_tiles),
|
||||
"training_eligible": False,
|
||||
"training_eligibility_reason": (
|
||||
"SAM-refined labels require a new governed corpus, validation report, human review and immutable training release."
|
||||
),
|
||||
}
|
||||
)
|
||||
summary_path = args.output_dir / "yolo_tile_dataset_summary.json"
|
||||
@@ -220,6 +262,11 @@ def main() -> int:
|
||||
"status": "ok",
|
||||
"source_summary": str(args.summary),
|
||||
"source_summary_sha256": sha256(args.summary),
|
||||
"source_training_release": str(training_release_paths(args.train_yaml)["release_manifest"]),
|
||||
"source_training_release_sha256": file_sha256(
|
||||
training_release_paths(args.train_yaml)["release_manifest"]
|
||||
),
|
||||
"source_corpus_manifest_sha256": source_release["corpus"]["manifest_sha256"],
|
||||
"sam_model": str(args.model),
|
||||
"sam_model_sha256": sha256(args.model),
|
||||
"device": args.device,
|
||||
@@ -236,6 +283,8 @@ def main() -> int:
|
||||
"fallback_label_count": fallback_count,
|
||||
"dropped_fallback_label_count": fallback_count if args.fallback_policy == "drop" else 0,
|
||||
"fallback_reason_counts": reason_counts,
|
||||
"training_eligible": False,
|
||||
"fixture_mode": bool(args.fixture_mode),
|
||||
}
|
||||
(args.output_dir / "sam-refinement.json").write_text(json.dumps(evidence, indent=2), encoding="utf-8")
|
||||
print(json.dumps(evidence, indent=2))
|
||||
|
||||
Reference in New Issue
Block a user