feat(provenance): govern source snapshots and data inputs
This commit is contained in:
@@ -6,10 +6,20 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
|
||||
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
from training_release_manifest import ( # noqa: E402
|
||||
TrainingReleaseError,
|
||||
assert_training_release_eligible,
|
||||
)
|
||||
|
||||
|
||||
def container_running(container: str) -> bool:
|
||||
result = subprocess.run(
|
||||
@@ -44,11 +54,33 @@ def load_completion_command(path: Path) -> list[str]:
|
||||
return command
|
||||
|
||||
|
||||
def verify_training_release(
|
||||
*,
|
||||
train_yaml: Path,
|
||||
corpus_manifest: Path,
|
||||
fixture_mode: bool,
|
||||
) -> None:
|
||||
"""Re-check the exact release before each detached resume command."""
|
||||
|
||||
assert_training_release_eligible(
|
||||
train_yaml=train_yaml,
|
||||
corpus_manifest=corpus_manifest,
|
||||
fixture_mode=fixture_mode,
|
||||
)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--container", required=True)
|
||||
parser.add_argument("--host-run-dir", type=Path, required=True)
|
||||
parser.add_argument("--container-checkpoint", required=True)
|
||||
parser.add_argument("--train-yaml", type=Path, required=True)
|
||||
parser.add_argument("--corpus-manifest", type=Path, required=True)
|
||||
parser.add_argument(
|
||||
"--fixture-mode",
|
||||
action="store_true",
|
||||
help="Only valid for an explicitly fixture-only frozen corpus release.",
|
||||
)
|
||||
parser.add_argument("--run-marker", required=True)
|
||||
parser.add_argument("--yolo", default="/opt/geointel/venv/bin/yolo")
|
||||
parser.add_argument("--poll-seconds", type=int, default=30)
|
||||
@@ -101,6 +133,17 @@ def main() -> int:
|
||||
state["status"] = "resume_budget_exhausted"
|
||||
write_state(state_path, state)
|
||||
return 3
|
||||
try:
|
||||
verify_training_release(
|
||||
train_yaml=args.train_yaml,
|
||||
corpus_manifest=args.corpus_manifest,
|
||||
fixture_mode=args.fixture_mode,
|
||||
)
|
||||
except TrainingReleaseError as exc:
|
||||
state["status"] = "training_release_verification_failed"
|
||||
state["training_release_error"] = str(exc)
|
||||
write_state(state_path, state)
|
||||
return 6
|
||||
result = subprocess.run(
|
||||
["docker", "exec", "-d", args.container, args.yolo, "train",
|
||||
f"resume={args.container_checkpoint}", "device=0"],
|
||||
|
||||
Reference in New Issue
Block a user