Recover governed runtime provenance for legacy YOLO models
This commit is contained in:
@@ -95,6 +95,7 @@ def test_all_in_one_dockerfile_copies_operator_scripts_for_runtime_use() -> None
|
||||
"build_detection_model_promotion_report.py",
|
||||
"run_split_background_promotion_workflow.sh",
|
||||
"activate_promoted_yolo_candidate.py",
|
||||
"migrate_runtime_model_provenance.py",
|
||||
"manage_grb_refresh.py",
|
||||
"orthophoto_release_preflight.py",
|
||||
"provision_walous_sources.py",
|
||||
|
||||
@@ -0,0 +1,137 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from hashlib import sha256
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "migrate_runtime_model_provenance.py"
|
||||
SPEC = importlib.util.spec_from_file_location("migrate_runtime_model_provenance", SCRIPT)
|
||||
assert SPEC and SPEC.loader
|
||||
module = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(module)
|
||||
|
||||
|
||||
def _sha(value: bytes) -> str:
|
||||
return sha256(value).hexdigest()
|
||||
|
||||
|
||||
def _args(tmp_path: Path):
|
||||
model = tmp_path / "active.pt"
|
||||
checkpoint = tmp_path / "best.pt"
|
||||
base_model = tmp_path / "base.pt"
|
||||
training_args = tmp_path / "args.yaml"
|
||||
training_results = tmp_path / "results.csv"
|
||||
dataset_summary = tmp_path / "dataset-summary.json"
|
||||
dataset_yaml = tmp_path / "dataset.yaml"
|
||||
training_summary = tmp_path / "training-summary.json"
|
||||
|
||||
model.write_bytes(b"exact promoted model bytes")
|
||||
checkpoint.write_bytes(model.read_bytes())
|
||||
base_model.write_bytes(b"exact base model bytes")
|
||||
training_args.write_text("epochs: 30\nseed: 0\n", encoding="utf-8")
|
||||
training_results.write_text("epoch,metric\n1,0.1\n", encoding="utf-8")
|
||||
dataset_yaml.write_text("names:\n 0: building\n", encoding="utf-8")
|
||||
dataset_summary.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "ok",
|
||||
"class_names": ["building"],
|
||||
"tile_count": 198,
|
||||
"train_tile_count": 180,
|
||||
"val_tile_count": 18,
|
||||
"label_count": 58_820,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
training_summary.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "ok",
|
||||
"trained_model_sha256": _sha(model.read_bytes()),
|
||||
"base_model_sha256": _sha(base_model.read_bytes()),
|
||||
"dataset_summary_sha256": _sha(dataset_summary.read_bytes()),
|
||||
"dataset_yaml_sha256": _sha(dataset_yaml.read_bytes()),
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return module.parse_args(
|
||||
[
|
||||
"--model-path",
|
||||
str(model),
|
||||
"--checkpoint-path",
|
||||
str(checkpoint),
|
||||
"--base-model-path",
|
||||
str(base_model),
|
||||
"--training-summary-path",
|
||||
str(training_summary),
|
||||
"--training-args-path",
|
||||
str(training_args),
|
||||
"--training-results-path",
|
||||
str(training_results),
|
||||
"--dataset-summary-path",
|
||||
str(dataset_summary),
|
||||
"--dataset-yaml-path",
|
||||
str(dataset_yaml),
|
||||
"--source-version",
|
||||
"sprint174-smallbld-minpx3-img640-ft30",
|
||||
"--framework-version",
|
||||
"8.4.93",
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_recovered_evidence_requires_byte_identical_checkpoint_and_recorded_hashes(tmp_path: Path) -> None:
|
||||
args = _args(tmp_path)
|
||||
|
||||
evidence = module.inspect_evidence(args)
|
||||
|
||||
assert evidence["checksums"]["model"] == evidence["checksums"]["checkpoint"]
|
||||
assert evidence["checksums"]["training_summary"] == _sha(
|
||||
Path(args.training_summary_path).read_bytes()
|
||||
)
|
||||
assert evidence["class_mapping"] == {"0": "building"}
|
||||
|
||||
|
||||
def test_recovered_evidence_rejects_changed_checkpoint(tmp_path: Path) -> None:
|
||||
args = _args(tmp_path)
|
||||
Path(args.checkpoint_path).write_bytes(b"other checkpoint")
|
||||
|
||||
exit_code, payload = module.migrate(args)
|
||||
|
||||
assert exit_code == 2
|
||||
assert payload["status"] == "evidence_invalid"
|
||||
assert "checkpoint/model SHA-256 mismatch" in payload["message"]
|
||||
|
||||
|
||||
def test_generated_sidecar_passes_exact_runtime_contract(tmp_path: Path) -> None:
|
||||
args = _args(tmp_path)
|
||||
evidence = module.inspect_evidence(args)
|
||||
payload = module._manifest_payload(
|
||||
args=args,
|
||||
evidence=evidence,
|
||||
source_registry_id=str(uuid4()),
|
||||
source_snapshot_id=str(uuid4()),
|
||||
imported_at="2026-08-23T21:00:00+00:00",
|
||||
)
|
||||
manifest_path = RuntimeModelProvenanceService.manifest_path_for_model(args.model_path)
|
||||
assert module._write_manifest_atomically(manifest_path, payload) is True
|
||||
|
||||
validated = RuntimeModelProvenanceService.validate_for_runtime(
|
||||
model_path=args.model_path,
|
||||
model_id="yolo-configured",
|
||||
task_type="object_detection",
|
||||
expected_model_version="sprint174-smallbld-minpx3-img640-ft30",
|
||||
allowed_frameworks=("ultralytics/pytorch",),
|
||||
)
|
||||
|
||||
assert validated.model_sha256 == evidence["checksums"]["model"]
|
||||
assert validated.runtime_manifest_sha256 == payload["metadata"]["runtime_manifest_sha256"]
|
||||
assert module._write_manifest_atomically(manifest_path, payload) is False
|
||||
@@ -156,6 +156,7 @@ COPY scripts/build_detection_model_promotion_report.py /app/scripts/build_detect
|
||||
COPY scripts/build_mol_operational_benchmark_report.py /app/scripts/build_mol_operational_benchmark_report.py
|
||||
COPY scripts/run_split_background_promotion_workflow.sh /app/scripts/run_split_background_promotion_workflow.sh
|
||||
COPY scripts/activate_promoted_yolo_candidate.py /app/scripts/activate_promoted_yolo_candidate.py
|
||||
COPY scripts/migrate_runtime_model_provenance.py /app/scripts/migrate_runtime_model_provenance.py
|
||||
COPY scripts/archive_technical_projects.py /app/scripts/archive_technical_projects.py
|
||||
COPY scripts/runtime_state_report.py /app/scripts/runtime_state_report.py
|
||||
COPY scripts/release_backup_guard.py /app/scripts/release_backup_guard.py
|
||||
|
||||
@@ -12935,3 +12935,15 @@ Open:
|
||||
- Physical touch-device and screen-reader certification remain human QA. No
|
||||
commit or deployment was performed because the active execution brief
|
||||
explicitly forbids committing unless requested.
|
||||
# 2026-08-23 — Active YOLO runtime provenance remediation
|
||||
|
||||
- Recovered the original Sprint 174 model run from persistent Tower storage.
|
||||
- Confirmed that the active model and retained `best.pt` are byte-identical at
|
||||
SHA-256 `a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1`.
|
||||
- Verified the surviving base-model, training-argument, results, training-summary,
|
||||
dataset-YAML and dataset-summary checksums before allowing registration.
|
||||
- Added a dry-run-first, atomic and idempotent runtime provenance migration that
|
||||
binds exact model bytes to a governed Postgres source snapshot.
|
||||
- Kept the historical evidence boundary explicit: this runtime receipt does not
|
||||
invent human review, a historical training commit/container, protected-test
|
||||
independence, national validity or a new promotion decision.
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
# Runtime model provenance remediation — 2026-08-23
|
||||
|
||||
## Outcome and claim boundary
|
||||
|
||||
The active detection checkpoint can receive a truthful narrow runtime sidecar
|
||||
because its surviving training artifacts now establish an exact byte chain.
|
||||
This remediation binds model bytes, retained checkpoint, base model, dataset
|
||||
contract inputs and the surviving Ultralytics training receipts. It does not
|
||||
retroactively assert a missing historical code commit/container, signed human
|
||||
review, protected-test independence, national validity or a new promotion.
|
||||
|
||||
## Recovered immutable evidence
|
||||
|
||||
| Artifact | SHA-256 |
|
||||
|---|---|
|
||||
| active model and retained `best.pt` | `a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1` |
|
||||
| base model | `a8a79cf5b0bdc19a0245acc322cf77232c335e222bd5f3c00a17d5f29402c196` |
|
||||
| training `args.yaml` | `2b482e6bbef26f433d4406e1acb5cbbf4ce63a63644b180a2d51b93f8c8f0dcb` |
|
||||
| training `results.csv` | `6f83fdea2c59cfc5f3e4fe9673494e073c4e0054980b3020bad0289d0118b777` |
|
||||
| training summary | `6d438308c923f50d885dc777d381f469fa215a0557f0f3e9d3facc2f75ce0b8e` |
|
||||
| dataset YAML | `3a2ea97c35a18072a1ab6738cd673c0ecec5344b19461c91d72a15e138d46e8d` |
|
||||
| dataset summary | `49b2a07d2105d08356431757b83eafc1498eaf1fb76965b1efe05b776824942a` |
|
||||
|
||||
The checkpoint embeds an Ultralytics detection task, class mapping
|
||||
`0: building`, framework version `8.4.93`, 30 epochs, image size 640, seed 0
|
||||
and deterministic mode. The dataset summary retains 198 tiles, 180 training
|
||||
tiles, 18 validation tiles and 58,820 labels.
|
||||
|
||||
## Guarded migration
|
||||
|
||||
`scripts/migrate_runtime_model_provenance.py` performs the migration. It fails
|
||||
closed on any mismatched file or recorded checksum, is dry-run by default,
|
||||
writes the sidecar atomically and reuses only an identical immutable database
|
||||
snapshot. The production check then validates the sidecar against the
|
||||
server-owned `model` registry and snapshot before model loading.
|
||||
|
||||
The operational source version is
|
||||
`sprint174-20260713-smallbld-minpx3-img640-ft30`. This is a recovered runtime
|
||||
artifact identity, not an accuracy or release-level claim.
|
||||
@@ -771,6 +771,22 @@ selected sample slugs and excluded sample slugs. Split validation still applies
|
||||
after filtering, so a manifest-backed holdout cannot be selected as training by
|
||||
omitting it from `--val-samples`.
|
||||
|
||||
### Runtime provenance migration for surviving checkpoints
|
||||
|
||||
`migrate_runtime_model_provenance.py` recovers the narrow runtime provenance
|
||||
contract for a legacy local YOLO checkpoint only when the active model, retained
|
||||
`best.pt`, base model, training summary, arguments, results, dataset summary and
|
||||
dataset YAML all match their recorded SHA-256 values. The default run is a
|
||||
read-only dry-run; `--apply` writes the neighbouring
|
||||
`.geointel-model.json` atomically and registers the exact model bytes as an
|
||||
immutable `model` source snapshot in Postgres.
|
||||
|
||||
The generated receipt explicitly does not claim a missing historical commit,
|
||||
container digest, human review, protected-test independence, national validity
|
||||
or promotion. It downloads nothing and runs no inference. After applying, use
|
||||
`yolo_preflight.py --check-model-load --json` and a separate real inference
|
||||
smoke to validate the actual CUDA runtime.
|
||||
|
||||
After rebuilding the all-in-one image, the operator scripts are available inside
|
||||
the container at `/app/scripts/...`. Before rebuilding, use the host checkout or
|
||||
temporarily copy scripts into the running container for one-off data prep.
|
||||
|
||||
@@ -0,0 +1,381 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from datetime import datetime, timezone
|
||||
from hashlib import sha256
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
from typing import Any
|
||||
|
||||
from app.db.session import SessionLocal
|
||||
from app.services.data_contract_validation import (
|
||||
PYTORCH_MODEL_CONTRACT_KEY,
|
||||
PYTORCH_MODEL_CONTRACT_VERSION,
|
||||
)
|
||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||
from app.services.source_registry_service import SourceRegistryService
|
||||
|
||||
|
||||
CLAIM_BOUNDARY = (
|
||||
"Recovered legacy runtime artifact binding only. This receipt does not assert complete human review, "
|
||||
"the historical training commit/container, protected-test independence, national validity or model promotion."
|
||||
)
|
||||
|
||||
|
||||
def _sha256_file(path: Path) -> str:
|
||||
digest = sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _required_file(value: str, field: str) -> Path:
|
||||
path = Path(value).expanduser().resolve()
|
||||
if not path.is_file():
|
||||
raise ValueError(f"{field} does not point to a readable file: {path}")
|
||||
return path
|
||||
|
||||
|
||||
def _json_object(path: Path, field: str) -> dict[str, Any]:
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
raise ValueError(f"{field} is not a readable UTF-8 JSON object: {path}") from exc
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"{field} must contain a JSON object: {path}")
|
||||
return payload
|
||||
|
||||
|
||||
def _require_equal(observed: Any, expected: Any, field: str) -> None:
|
||||
if observed != expected:
|
||||
raise ValueError(f"{field} mismatch: expected {expected!r}, observed {observed!r}")
|
||||
|
||||
|
||||
def inspect_evidence(args: argparse.Namespace) -> dict[str, Any]:
|
||||
paths = {
|
||||
"model": _required_file(args.model_path, "--model-path"),
|
||||
"checkpoint": _required_file(args.checkpoint_path, "--checkpoint-path"),
|
||||
"base_model": _required_file(args.base_model_path, "--base-model-path"),
|
||||
"training_summary": _required_file(args.training_summary_path, "--training-summary-path"),
|
||||
"training_args": _required_file(args.training_args_path, "--training-args-path"),
|
||||
"training_results": _required_file(args.training_results_path, "--training-results-path"),
|
||||
"dataset_summary": _required_file(args.dataset_summary_path, "--dataset-summary-path"),
|
||||
"dataset_yaml": _required_file(args.dataset_yaml_path, "--dataset-yaml-path"),
|
||||
}
|
||||
checksums = {name: _sha256_file(path) for name, path in paths.items()}
|
||||
_require_equal(checksums["checkpoint"], checksums["model"], "checkpoint/model SHA-256")
|
||||
|
||||
training_summary = _json_object(paths["training_summary"], "training summary")
|
||||
dataset_summary = _json_object(paths["dataset_summary"], "dataset summary")
|
||||
_require_equal(training_summary.get("status"), "ok", "training_summary.status")
|
||||
_require_equal(training_summary.get("trained_model_sha256"), checksums["model"], "trained_model_sha256")
|
||||
_require_equal(training_summary.get("base_model_sha256"), checksums["base_model"], "base_model_sha256")
|
||||
_require_equal(
|
||||
training_summary.get("dataset_summary_sha256"),
|
||||
checksums["dataset_summary"],
|
||||
"dataset_summary_sha256",
|
||||
)
|
||||
_require_equal(training_summary.get("dataset_yaml_sha256"), checksums["dataset_yaml"], "dataset_yaml_sha256")
|
||||
_require_equal(dataset_summary.get("status"), "ok", "dataset_summary.status")
|
||||
class_names = dataset_summary.get("class_names")
|
||||
if not isinstance(class_names, list) or not class_names or not all(
|
||||
isinstance(value, str) and value.strip() for value in class_names
|
||||
):
|
||||
raise ValueError("dataset_summary.class_names must be a non-empty string list")
|
||||
|
||||
class_mapping = {str(index): value.strip() for index, value in enumerate(class_names)}
|
||||
evidence = {
|
||||
"paths": {name: str(path) for name, path in paths.items()},
|
||||
"checksums": checksums,
|
||||
"class_mapping": class_mapping,
|
||||
"training_summary": training_summary,
|
||||
"dataset_summary": {
|
||||
"status": dataset_summary.get("status"),
|
||||
"tile_count": dataset_summary.get("tile_count"),
|
||||
"train_tile_count": dataset_summary.get("train_tile_count"),
|
||||
"val_tile_count": dataset_summary.get("val_tile_count"),
|
||||
"label_count": dataset_summary.get("label_count"),
|
||||
"class_names": class_names,
|
||||
},
|
||||
}
|
||||
return evidence
|
||||
|
||||
|
||||
def _snapshot_key(*, model_id: str, model_sha256: str) -> str:
|
||||
return f"runtime-model-{model_id}-{model_sha256[:24]}"
|
||||
|
||||
|
||||
def _manifest_payload(
|
||||
*,
|
||||
args: argparse.Namespace,
|
||||
evidence: dict[str, Any],
|
||||
source_registry_id: str,
|
||||
source_snapshot_id: str,
|
||||
imported_at: str,
|
||||
) -> dict[str, Any]:
|
||||
checksums = evidence["checksums"]
|
||||
paths = evidence["paths"]
|
||||
payload: dict[str, Any] = {
|
||||
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
|
||||
"data_contract": {
|
||||
"key": PYTORCH_MODEL_CONTRACT_KEY,
|
||||
"version": PYTORCH_MODEL_CONTRACT_VERSION,
|
||||
},
|
||||
"model": {
|
||||
"model_id": args.model_id,
|
||||
"task_type": args.task_type,
|
||||
"sha256": checksums["model"],
|
||||
"model_format": "pytorch",
|
||||
"framework": "ultralytics/pytorch",
|
||||
"class_mapping": evidence["class_mapping"],
|
||||
"source_version": args.source_version,
|
||||
},
|
||||
"source": {
|
||||
"source_registry_id": source_registry_id,
|
||||
"source_snapshot_id": source_snapshot_id,
|
||||
"source_registry_key": RuntimeModelProvenanceService.SOURCE_REGISTRY_KEY,
|
||||
"source_snapshot_checksum_sha256": checksums["model"],
|
||||
},
|
||||
"lineage": {
|
||||
"upstream_asset_ids": [
|
||||
f"base-model:{Path(paths['base_model']).name}",
|
||||
f"dataset-summary:{Path(paths['dataset_summary']).name}",
|
||||
f"dataset-yaml:{Path(paths['dataset_yaml']).name}",
|
||||
f"training-args:{Path(paths['training_args']).name}",
|
||||
f"training-results:{Path(paths['training_results']).name}",
|
||||
f"training-summary:{Path(paths['training_summary']).name}",
|
||||
],
|
||||
"upstream_checksums_sha256": [
|
||||
checksums["base_model"],
|
||||
checksums["dataset_summary"],
|
||||
checksums["dataset_yaml"],
|
||||
checksums["training_args"],
|
||||
checksums["training_results"],
|
||||
checksums["training_summary"],
|
||||
],
|
||||
"transformations": [
|
||||
{
|
||||
"name": "ultralytics-yolo-finetune",
|
||||
"version": args.framework_version,
|
||||
"checksum_sha256": checksums["training_args"],
|
||||
}
|
||||
],
|
||||
},
|
||||
"metadata": {
|
||||
"training_manifest_sha256": checksums["training_summary"],
|
||||
"claim_boundary": CLAIM_BOUNDARY,
|
||||
"evidence_paths": paths,
|
||||
"checkpoint_sha256": checksums["checkpoint"],
|
||||
},
|
||||
"imported_at": imported_at,
|
||||
}
|
||||
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(
|
||||
payload
|
||||
)
|
||||
return payload
|
||||
|
||||
|
||||
def _write_manifest_atomically(path: Path, payload: dict[str, Any]) -> bool:
|
||||
encoded = (json.dumps(payload, indent=2, sort_keys=True) + "\n").encode("utf-8")
|
||||
if path.exists():
|
||||
if path.read_bytes() == encoded:
|
||||
return False
|
||||
raise ValueError(f"Refusing to overwrite a different runtime provenance sidecar: {path}")
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
temporary: Path | None = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="wb",
|
||||
prefix=f".{path.name}.",
|
||||
suffix=".tmp",
|
||||
dir=path.parent,
|
||||
delete=False,
|
||||
) as handle:
|
||||
handle.write(encoded)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
temporary = Path(handle.name)
|
||||
os.replace(temporary, path)
|
||||
return True
|
||||
finally:
|
||||
if temporary is not None and temporary.exists():
|
||||
temporary.unlink()
|
||||
|
||||
|
||||
def migrate(args: argparse.Namespace) -> tuple[int, dict[str, Any]]:
|
||||
try:
|
||||
evidence = inspect_evidence(args)
|
||||
except (OSError, ValueError) as exc:
|
||||
return 2, {"status": "evidence_invalid", "message": str(exc), "applied": False}
|
||||
|
||||
model_sha256 = evidence["checksums"]["model"]
|
||||
snapshot_key = _snapshot_key(model_id=args.model_id, model_sha256=model_sha256)
|
||||
result: dict[str, Any] = {
|
||||
"status": "ready_to_apply",
|
||||
"message": "Recovered evidence is internally checksum-consistent; re-run with --apply to register it.",
|
||||
"applied": False,
|
||||
"model_id": args.model_id,
|
||||
"task_type": args.task_type,
|
||||
"source_version": args.source_version,
|
||||
"model_sha256": model_sha256,
|
||||
"snapshot_key": snapshot_key,
|
||||
"claim_boundary": CLAIM_BOUNDARY,
|
||||
"evidence": evidence,
|
||||
"will_download_models": False,
|
||||
"will_run_inference": False,
|
||||
}
|
||||
if not args.apply:
|
||||
return 0, result
|
||||
|
||||
db = SessionLocal()
|
||||
manifest_path = RuntimeModelProvenanceService.manifest_path_for_model(evidence["paths"]["model"])
|
||||
created_manifest = False
|
||||
try:
|
||||
source = SourceRegistryService.ensure_server_owned_source(db, RuntimeModelProvenanceService.SOURCE_REGISTRY_KEY)
|
||||
source.ingest_status = "configured"
|
||||
source.freshness_status = "current"
|
||||
registry_metadata = dict(source.registry_metadata_json or {})
|
||||
registry_metadata["runtime_model_contract"] = {
|
||||
"key": PYTORCH_MODEL_CONTRACT_KEY,
|
||||
"version": PYTORCH_MODEL_CONTRACT_VERSION,
|
||||
"claim_boundary": CLAIM_BOUNDARY,
|
||||
}
|
||||
source.registry_metadata_json = registry_metadata
|
||||
|
||||
snapshot = SourceRegistryService.record_snapshot(
|
||||
db,
|
||||
source_key=RuntimeModelProvenanceService.SOURCE_REGISTRY_KEY,
|
||||
snapshot_key=snapshot_key,
|
||||
checksum_sha256=model_sha256,
|
||||
source_version=args.source_version,
|
||||
source_url=f"file://{evidence['paths']['checkpoint']}",
|
||||
crs="not_applicable",
|
||||
units="model_weights",
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
known_limitations=[CLAIM_BOUNDARY],
|
||||
observed_schema={
|
||||
"model_format": "pytorch",
|
||||
"framework": "ultralytics/pytorch",
|
||||
"class_mapping": evidence["class_mapping"],
|
||||
},
|
||||
snapshot_metadata={
|
||||
"evidence_paths": evidence["paths"],
|
||||
"evidence_checksums_sha256": evidence["checksums"],
|
||||
"claim_boundary": CLAIM_BOUNDARY,
|
||||
},
|
||||
reuse_existing_snapshot=True,
|
||||
)
|
||||
if manifest_path.exists():
|
||||
payload = _json_object(manifest_path, "existing runtime provenance sidecar")
|
||||
_require_equal(payload.get("model", {}).get("sha256"), model_sha256, "existing model.sha256")
|
||||
_require_equal(
|
||||
payload.get("model", {}).get("source_version"),
|
||||
args.source_version,
|
||||
"existing model.source_version",
|
||||
)
|
||||
_require_equal(
|
||||
payload.get("source", {}).get("source_registry_id"),
|
||||
str(source.id),
|
||||
"existing source.source_registry_id",
|
||||
)
|
||||
_require_equal(
|
||||
payload.get("source", {}).get("source_snapshot_id"),
|
||||
str(snapshot.id),
|
||||
"existing source.source_snapshot_id",
|
||||
)
|
||||
else:
|
||||
imported_at = datetime.now(timezone.utc).isoformat()
|
||||
payload = _manifest_payload(
|
||||
args=args,
|
||||
evidence=evidence,
|
||||
source_registry_id=str(source.id),
|
||||
source_snapshot_id=str(snapshot.id),
|
||||
imported_at=imported_at,
|
||||
)
|
||||
created_manifest = _write_manifest_atomically(manifest_path, payload)
|
||||
RuntimeModelProvenanceService.validate_for_runtime(
|
||||
model_path=evidence["paths"]["model"],
|
||||
model_id=args.model_id,
|
||||
task_type=args.task_type,
|
||||
expected_model_version=args.source_version,
|
||||
allowed_frameworks=("ultralytics/pytorch",),
|
||||
)
|
||||
db.commit()
|
||||
production_evidence = RuntimeModelProvenanceService.validate_for_production_runtime(
|
||||
db=db,
|
||||
model_path=evidence["paths"]["model"],
|
||||
model_id=args.model_id,
|
||||
task_type=args.task_type,
|
||||
expected_model_version=args.source_version,
|
||||
allowed_frameworks=("ultralytics/pytorch",),
|
||||
)
|
||||
result.update(
|
||||
{
|
||||
"status": "applied",
|
||||
"message": "Runtime model provenance was registered and validated against Postgres.",
|
||||
"applied": True,
|
||||
"manifest_path": str(manifest_path),
|
||||
"manifest_created": created_manifest,
|
||||
"source_registry_id": str(source.id),
|
||||
"source_snapshot_id": str(snapshot.id),
|
||||
"runtime_provenance": production_evidence.as_dict(),
|
||||
}
|
||||
)
|
||||
return 0, result
|
||||
except Exception as exc:
|
||||
db.rollback()
|
||||
if created_manifest and manifest_path.exists():
|
||||
manifest_path.unlink()
|
||||
return 3, {
|
||||
**result,
|
||||
"status": "apply_failed",
|
||||
"message": str(exc),
|
||||
"applied": False,
|
||||
}
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def parse_args(argv: list[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Migrate a surviving legacy YOLO checkpoint into GeoIntel's runtime provenance contract without "
|
||||
"claiming missing training or review evidence."
|
||||
)
|
||||
)
|
||||
parser.add_argument("--model-path", required=True)
|
||||
parser.add_argument("--checkpoint-path", required=True)
|
||||
parser.add_argument("--base-model-path", required=True)
|
||||
parser.add_argument("--training-summary-path", required=True)
|
||||
parser.add_argument("--training-args-path", required=True)
|
||||
parser.add_argument("--training-results-path", required=True)
|
||||
parser.add_argument("--dataset-summary-path", required=True)
|
||||
parser.add_argument("--dataset-yaml-path", required=True)
|
||||
parser.add_argument("--model-id", default="yolo-configured")
|
||||
parser.add_argument("--task-type", default="object_detection")
|
||||
parser.add_argument("--source-version", required=True)
|
||||
parser.add_argument("--framework-version", required=True)
|
||||
parser.add_argument("--apply", action="store_true")
|
||||
parser.add_argument("--json", action="store_true")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = parse_args(argv or sys.argv[1:])
|
||||
exit_code, payload = migrate(args)
|
||||
if args.json:
|
||||
print(json.dumps(payload, indent=2, sort_keys=True))
|
||||
else:
|
||||
print(f"status: {payload['status']}")
|
||||
print(f"message: {payload['message']}")
|
||||
print(f"claim boundary: {payload.get('claim_boundary', CLAIM_BOUNDARY)}")
|
||||
return exit_code
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -125,6 +125,7 @@ ${PYTHON_BIN} -m py_compile scripts/render_detection_false_negative_review_conta
|
||||
${PYTHON_BIN} -m py_compile scripts/validate_detection_false_positive_review_decisions.py
|
||||
${PYTHON_BIN} -m py_compile scripts/validate_detection_false_negative_review_decisions.py
|
||||
${PYTHON_BIN} -m py_compile scripts/activate_promoted_yolo_candidate.py
|
||||
${PYTHON_BIN} -m py_compile scripts/migrate_runtime_model_provenance.py
|
||||
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
|
||||
${PYTHON_BIN} -m py_compile scripts/archive_technical_projects.py
|
||||
${PYTHON_BIN} -m py_compile scripts/release_backup_guard.py
|
||||
|
||||
Reference in New Issue
Block a user