Recover governed runtime provenance for legacy YOLO models
GeoIntel release gates / Compile, test, contracts and builds (push) Failing after 20s
GeoIntel release gates / Python and npm vulnerability policy (push) Failing after 22s
GeoIntel release gates / GIS image, SBOM and container scan (push) Failing after 2m31s

This commit is contained in:
Jens
2026-08-23 23:22:02 +02:00
parent 300fbba5c9
commit be2e092b33
8 changed files with 588 additions and 0 deletions
@@ -95,6 +95,7 @@ def test_all_in_one_dockerfile_copies_operator_scripts_for_runtime_use() -> None
"build_detection_model_promotion_report.py",
"run_split_background_promotion_workflow.sh",
"activate_promoted_yolo_candidate.py",
"migrate_runtime_model_provenance.py",
"manage_grb_refresh.py",
"orthophoto_release_preflight.py",
"provision_walous_sources.py",
@@ -0,0 +1,137 @@
from __future__ import annotations
from hashlib import sha256
import importlib.util
import json
from pathlib import Path
from uuid import uuid4
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "migrate_runtime_model_provenance.py"
SPEC = importlib.util.spec_from_file_location("migrate_runtime_model_provenance", SCRIPT)
assert SPEC and SPEC.loader
module = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(module)
def _sha(value: bytes) -> str:
return sha256(value).hexdigest()
def _args(tmp_path: Path):
model = tmp_path / "active.pt"
checkpoint = tmp_path / "best.pt"
base_model = tmp_path / "base.pt"
training_args = tmp_path / "args.yaml"
training_results = tmp_path / "results.csv"
dataset_summary = tmp_path / "dataset-summary.json"
dataset_yaml = tmp_path / "dataset.yaml"
training_summary = tmp_path / "training-summary.json"
model.write_bytes(b"exact promoted model bytes")
checkpoint.write_bytes(model.read_bytes())
base_model.write_bytes(b"exact base model bytes")
training_args.write_text("epochs: 30\nseed: 0\n", encoding="utf-8")
training_results.write_text("epoch,metric\n1,0.1\n", encoding="utf-8")
dataset_yaml.write_text("names:\n 0: building\n", encoding="utf-8")
dataset_summary.write_text(
json.dumps(
{
"status": "ok",
"class_names": ["building"],
"tile_count": 198,
"train_tile_count": 180,
"val_tile_count": 18,
"label_count": 58_820,
}
),
encoding="utf-8",
)
training_summary.write_text(
json.dumps(
{
"status": "ok",
"trained_model_sha256": _sha(model.read_bytes()),
"base_model_sha256": _sha(base_model.read_bytes()),
"dataset_summary_sha256": _sha(dataset_summary.read_bytes()),
"dataset_yaml_sha256": _sha(dataset_yaml.read_bytes()),
}
),
encoding="utf-8",
)
return module.parse_args(
[
"--model-path",
str(model),
"--checkpoint-path",
str(checkpoint),
"--base-model-path",
str(base_model),
"--training-summary-path",
str(training_summary),
"--training-args-path",
str(training_args),
"--training-results-path",
str(training_results),
"--dataset-summary-path",
str(dataset_summary),
"--dataset-yaml-path",
str(dataset_yaml),
"--source-version",
"sprint174-smallbld-minpx3-img640-ft30",
"--framework-version",
"8.4.93",
]
)
def test_recovered_evidence_requires_byte_identical_checkpoint_and_recorded_hashes(tmp_path: Path) -> None:
args = _args(tmp_path)
evidence = module.inspect_evidence(args)
assert evidence["checksums"]["model"] == evidence["checksums"]["checkpoint"]
assert evidence["checksums"]["training_summary"] == _sha(
Path(args.training_summary_path).read_bytes()
)
assert evidence["class_mapping"] == {"0": "building"}
def test_recovered_evidence_rejects_changed_checkpoint(tmp_path: Path) -> None:
args = _args(tmp_path)
Path(args.checkpoint_path).write_bytes(b"other checkpoint")
exit_code, payload = module.migrate(args)
assert exit_code == 2
assert payload["status"] == "evidence_invalid"
assert "checkpoint/model SHA-256 mismatch" in payload["message"]
def test_generated_sidecar_passes_exact_runtime_contract(tmp_path: Path) -> None:
args = _args(tmp_path)
evidence = module.inspect_evidence(args)
payload = module._manifest_payload(
args=args,
evidence=evidence,
source_registry_id=str(uuid4()),
source_snapshot_id=str(uuid4()),
imported_at="2026-08-23T21:00:00+00:00",
)
manifest_path = RuntimeModelProvenanceService.manifest_path_for_model(args.model_path)
assert module._write_manifest_atomically(manifest_path, payload) is True
validated = RuntimeModelProvenanceService.validate_for_runtime(
model_path=args.model_path,
model_id="yolo-configured",
task_type="object_detection",
expected_model_version="sprint174-smallbld-minpx3-img640-ft30",
allowed_frameworks=("ultralytics/pytorch",),
)
assert validated.model_sha256 == evidence["checksums"]["model"]
assert validated.runtime_manifest_sha256 == payload["metadata"]["runtime_manifest_sha256"]
assert module._write_manifest_atomically(manifest_path, payload) is False
+1
View File
@@ -156,6 +156,7 @@ COPY scripts/build_detection_model_promotion_report.py /app/scripts/build_detect
COPY scripts/build_mol_operational_benchmark_report.py /app/scripts/build_mol_operational_benchmark_report.py
COPY scripts/run_split_background_promotion_workflow.sh /app/scripts/run_split_background_promotion_workflow.sh
COPY scripts/activate_promoted_yolo_candidate.py /app/scripts/activate_promoted_yolo_candidate.py
COPY scripts/migrate_runtime_model_provenance.py /app/scripts/migrate_runtime_model_provenance.py
COPY scripts/archive_technical_projects.py /app/scripts/archive_technical_projects.py
COPY scripts/runtime_state_report.py /app/scripts/runtime_state_report.py
COPY scripts/release_backup_guard.py /app/scripts/release_backup_guard.py
+12
View File
@@ -12935,3 +12935,15 @@ Open:
- Physical touch-device and screen-reader certification remain human QA. No
commit or deployment was performed because the active execution brief
explicitly forbids committing unless requested.
# 2026-08-23 — Active YOLO runtime provenance remediation
- Recovered the original Sprint 174 model run from persistent Tower storage.
- Confirmed that the active model and retained `best.pt` are byte-identical at
SHA-256 `a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1`.
- Verified the surviving base-model, training-argument, results, training-summary,
dataset-YAML and dataset-summary checksums before allowing registration.
- Added a dry-run-first, atomic and idempotent runtime provenance migration that
binds exact model bytes to a governed Postgres source snapshot.
- Kept the historical evidence boundary explicit: this runtime receipt does not
invent human review, a historical training commit/container, protected-test
independence, national validity or a new promotion decision.
@@ -0,0 +1,39 @@
# Runtime model provenance remediation — 2026-08-23
## Outcome and claim boundary
The active detection checkpoint can receive a truthful narrow runtime sidecar
because its surviving training artifacts now establish an exact byte chain.
This remediation binds model bytes, retained checkpoint, base model, dataset
contract inputs and the surviving Ultralytics training receipts. It does not
retroactively assert a missing historical code commit/container, signed human
review, protected-test independence, national validity or a new promotion.
## Recovered immutable evidence
| Artifact | SHA-256 |
|---|---|
| active model and retained `best.pt` | `a9088b8491dfae36694b53e9e9406cb4e3511d334a5712fa34f75078a47759c1` |
| base model | `a8a79cf5b0bdc19a0245acc322cf77232c335e222bd5f3c00a17d5f29402c196` |
| training `args.yaml` | `2b482e6bbef26f433d4406e1acb5cbbf4ce63a63644b180a2d51b93f8c8f0dcb` |
| training `results.csv` | `6f83fdea2c59cfc5f3e4fe9673494e073c4e0054980b3020bad0289d0118b777` |
| training summary | `6d438308c923f50d885dc777d381f469fa215a0557f0f3e9d3facc2f75ce0b8e` |
| dataset YAML | `3a2ea97c35a18072a1ab6738cd673c0ecec5344b19461c91d72a15e138d46e8d` |
| dataset summary | `49b2a07d2105d08356431757b83eafc1498eaf1fb76965b1efe05b776824942a` |
The checkpoint embeds an Ultralytics detection task, class mapping
`0: building`, framework version `8.4.93`, 30 epochs, image size 640, seed 0
and deterministic mode. The dataset summary retains 198 tiles, 180 training
tiles, 18 validation tiles and 58,820 labels.
## Guarded migration
`scripts/migrate_runtime_model_provenance.py` performs the migration. It fails
closed on any mismatched file or recorded checksum, is dry-run by default,
writes the sidecar atomically and reuses only an identical immutable database
snapshot. The production check then validates the sidecar against the
server-owned `model` registry and snapshot before model loading.
The operational source version is
`sprint174-20260713-smallbld-minpx3-img640-ft30`. This is a recovered runtime
artifact identity, not an accuracy or release-level claim.
+16
View File
@@ -771,6 +771,22 @@ selected sample slugs and excluded sample slugs. Split validation still applies
after filtering, so a manifest-backed holdout cannot be selected as training by
omitting it from `--val-samples`.
### Runtime provenance migration for surviving checkpoints
`migrate_runtime_model_provenance.py` recovers the narrow runtime provenance
contract for a legacy local YOLO checkpoint only when the active model, retained
`best.pt`, base model, training summary, arguments, results, dataset summary and
dataset YAML all match their recorded SHA-256 values. The default run is a
read-only dry-run; `--apply` writes the neighbouring
`.geointel-model.json` atomically and registers the exact model bytes as an
immutable `model` source snapshot in Postgres.
The generated receipt explicitly does not claim a missing historical commit,
container digest, human review, protected-test independence, national validity
or promotion. It downloads nothing and runs no inference. After applying, use
`yolo_preflight.py --check-model-load --json` and a separate real inference
smoke to validate the actual CUDA runtime.
After rebuilding the all-in-one image, the operator scripts are available inside
the container at `/app/scripts/...`. Before rebuilding, use the host checkout or
temporarily copy scripts into the running container for one-off data prep.
+381
View File
@@ -0,0 +1,381 @@
from __future__ import annotations
import argparse
from datetime import datetime, timezone
from hashlib import sha256
import json
import os
from pathlib import Path
import sys
import tempfile
from typing import Any
from app.db.session import SessionLocal
from app.services.data_contract_validation import (
PYTORCH_MODEL_CONTRACT_KEY,
PYTORCH_MODEL_CONTRACT_VERSION,
)
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
from app.services.source_registry_service import SourceRegistryService
CLAIM_BOUNDARY = (
"Recovered legacy runtime artifact binding only. This receipt does not assert complete human review, "
"the historical training commit/container, protected-test independence, national validity or model promotion."
)
def _sha256_file(path: Path) -> str:
digest = sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _required_file(value: str, field: str) -> Path:
path = Path(value).expanduser().resolve()
if not path.is_file():
raise ValueError(f"{field} does not point to a readable file: {path}")
return path
def _json_object(path: Path, field: str) -> dict[str, Any]:
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
raise ValueError(f"{field} is not a readable UTF-8 JSON object: {path}") from exc
if not isinstance(payload, dict):
raise ValueError(f"{field} must contain a JSON object: {path}")
return payload
def _require_equal(observed: Any, expected: Any, field: str) -> None:
if observed != expected:
raise ValueError(f"{field} mismatch: expected {expected!r}, observed {observed!r}")
def inspect_evidence(args: argparse.Namespace) -> dict[str, Any]:
paths = {
"model": _required_file(args.model_path, "--model-path"),
"checkpoint": _required_file(args.checkpoint_path, "--checkpoint-path"),
"base_model": _required_file(args.base_model_path, "--base-model-path"),
"training_summary": _required_file(args.training_summary_path, "--training-summary-path"),
"training_args": _required_file(args.training_args_path, "--training-args-path"),
"training_results": _required_file(args.training_results_path, "--training-results-path"),
"dataset_summary": _required_file(args.dataset_summary_path, "--dataset-summary-path"),
"dataset_yaml": _required_file(args.dataset_yaml_path, "--dataset-yaml-path"),
}
checksums = {name: _sha256_file(path) for name, path in paths.items()}
_require_equal(checksums["checkpoint"], checksums["model"], "checkpoint/model SHA-256")
training_summary = _json_object(paths["training_summary"], "training summary")
dataset_summary = _json_object(paths["dataset_summary"], "dataset summary")
_require_equal(training_summary.get("status"), "ok", "training_summary.status")
_require_equal(training_summary.get("trained_model_sha256"), checksums["model"], "trained_model_sha256")
_require_equal(training_summary.get("base_model_sha256"), checksums["base_model"], "base_model_sha256")
_require_equal(
training_summary.get("dataset_summary_sha256"),
checksums["dataset_summary"],
"dataset_summary_sha256",
)
_require_equal(training_summary.get("dataset_yaml_sha256"), checksums["dataset_yaml"], "dataset_yaml_sha256")
_require_equal(dataset_summary.get("status"), "ok", "dataset_summary.status")
class_names = dataset_summary.get("class_names")
if not isinstance(class_names, list) or not class_names or not all(
isinstance(value, str) and value.strip() for value in class_names
):
raise ValueError("dataset_summary.class_names must be a non-empty string list")
class_mapping = {str(index): value.strip() for index, value in enumerate(class_names)}
evidence = {
"paths": {name: str(path) for name, path in paths.items()},
"checksums": checksums,
"class_mapping": class_mapping,
"training_summary": training_summary,
"dataset_summary": {
"status": dataset_summary.get("status"),
"tile_count": dataset_summary.get("tile_count"),
"train_tile_count": dataset_summary.get("train_tile_count"),
"val_tile_count": dataset_summary.get("val_tile_count"),
"label_count": dataset_summary.get("label_count"),
"class_names": class_names,
},
}
return evidence
def _snapshot_key(*, model_id: str, model_sha256: str) -> str:
return f"runtime-model-{model_id}-{model_sha256[:24]}"
def _manifest_payload(
*,
args: argparse.Namespace,
evidence: dict[str, Any],
source_registry_id: str,
source_snapshot_id: str,
imported_at: str,
) -> dict[str, Any]:
checksums = evidence["checksums"]
paths = evidence["paths"]
payload: dict[str, Any] = {
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
"data_contract": {
"key": PYTORCH_MODEL_CONTRACT_KEY,
"version": PYTORCH_MODEL_CONTRACT_VERSION,
},
"model": {
"model_id": args.model_id,
"task_type": args.task_type,
"sha256": checksums["model"],
"model_format": "pytorch",
"framework": "ultralytics/pytorch",
"class_mapping": evidence["class_mapping"],
"source_version": args.source_version,
},
"source": {
"source_registry_id": source_registry_id,
"source_snapshot_id": source_snapshot_id,
"source_registry_key": RuntimeModelProvenanceService.SOURCE_REGISTRY_KEY,
"source_snapshot_checksum_sha256": checksums["model"],
},
"lineage": {
"upstream_asset_ids": [
f"base-model:{Path(paths['base_model']).name}",
f"dataset-summary:{Path(paths['dataset_summary']).name}",
f"dataset-yaml:{Path(paths['dataset_yaml']).name}",
f"training-args:{Path(paths['training_args']).name}",
f"training-results:{Path(paths['training_results']).name}",
f"training-summary:{Path(paths['training_summary']).name}",
],
"upstream_checksums_sha256": [
checksums["base_model"],
checksums["dataset_summary"],
checksums["dataset_yaml"],
checksums["training_args"],
checksums["training_results"],
checksums["training_summary"],
],
"transformations": [
{
"name": "ultralytics-yolo-finetune",
"version": args.framework_version,
"checksum_sha256": checksums["training_args"],
}
],
},
"metadata": {
"training_manifest_sha256": checksums["training_summary"],
"claim_boundary": CLAIM_BOUNDARY,
"evidence_paths": paths,
"checkpoint_sha256": checksums["checkpoint"],
},
"imported_at": imported_at,
}
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(
payload
)
return payload
def _write_manifest_atomically(path: Path, payload: dict[str, Any]) -> bool:
encoded = (json.dumps(payload, indent=2, sort_keys=True) + "\n").encode("utf-8")
if path.exists():
if path.read_bytes() == encoded:
return False
raise ValueError(f"Refusing to overwrite a different runtime provenance sidecar: {path}")
path.parent.mkdir(parents=True, exist_ok=True)
temporary: Path | None = None
try:
with tempfile.NamedTemporaryFile(
mode="wb",
prefix=f".{path.name}.",
suffix=".tmp",
dir=path.parent,
delete=False,
) as handle:
handle.write(encoded)
handle.flush()
os.fsync(handle.fileno())
temporary = Path(handle.name)
os.replace(temporary, path)
return True
finally:
if temporary is not None and temporary.exists():
temporary.unlink()
def migrate(args: argparse.Namespace) -> tuple[int, dict[str, Any]]:
try:
evidence = inspect_evidence(args)
except (OSError, ValueError) as exc:
return 2, {"status": "evidence_invalid", "message": str(exc), "applied": False}
model_sha256 = evidence["checksums"]["model"]
snapshot_key = _snapshot_key(model_id=args.model_id, model_sha256=model_sha256)
result: dict[str, Any] = {
"status": "ready_to_apply",
"message": "Recovered evidence is internally checksum-consistent; re-run with --apply to register it.",
"applied": False,
"model_id": args.model_id,
"task_type": args.task_type,
"source_version": args.source_version,
"model_sha256": model_sha256,
"snapshot_key": snapshot_key,
"claim_boundary": CLAIM_BOUNDARY,
"evidence": evidence,
"will_download_models": False,
"will_run_inference": False,
}
if not args.apply:
return 0, result
db = SessionLocal()
manifest_path = RuntimeModelProvenanceService.manifest_path_for_model(evidence["paths"]["model"])
created_manifest = False
try:
source = SourceRegistryService.ensure_server_owned_source(db, RuntimeModelProvenanceService.SOURCE_REGISTRY_KEY)
source.ingest_status = "configured"
source.freshness_status = "current"
registry_metadata = dict(source.registry_metadata_json or {})
registry_metadata["runtime_model_contract"] = {
"key": PYTORCH_MODEL_CONTRACT_KEY,
"version": PYTORCH_MODEL_CONTRACT_VERSION,
"claim_boundary": CLAIM_BOUNDARY,
}
source.registry_metadata_json = registry_metadata
snapshot = SourceRegistryService.record_snapshot(
db,
source_key=RuntimeModelProvenanceService.SOURCE_REGISTRY_KEY,
snapshot_key=snapshot_key,
checksum_sha256=model_sha256,
source_version=args.source_version,
source_url=f"file://{evidence['paths']['checkpoint']}",
crs="not_applicable",
units="model_weights",
freshness_status="current",
ingest_status="ingested",
known_limitations=[CLAIM_BOUNDARY],
observed_schema={
"model_format": "pytorch",
"framework": "ultralytics/pytorch",
"class_mapping": evidence["class_mapping"],
},
snapshot_metadata={
"evidence_paths": evidence["paths"],
"evidence_checksums_sha256": evidence["checksums"],
"claim_boundary": CLAIM_BOUNDARY,
},
reuse_existing_snapshot=True,
)
if manifest_path.exists():
payload = _json_object(manifest_path, "existing runtime provenance sidecar")
_require_equal(payload.get("model", {}).get("sha256"), model_sha256, "existing model.sha256")
_require_equal(
payload.get("model", {}).get("source_version"),
args.source_version,
"existing model.source_version",
)
_require_equal(
payload.get("source", {}).get("source_registry_id"),
str(source.id),
"existing source.source_registry_id",
)
_require_equal(
payload.get("source", {}).get("source_snapshot_id"),
str(snapshot.id),
"existing source.source_snapshot_id",
)
else:
imported_at = datetime.now(timezone.utc).isoformat()
payload = _manifest_payload(
args=args,
evidence=evidence,
source_registry_id=str(source.id),
source_snapshot_id=str(snapshot.id),
imported_at=imported_at,
)
created_manifest = _write_manifest_atomically(manifest_path, payload)
RuntimeModelProvenanceService.validate_for_runtime(
model_path=evidence["paths"]["model"],
model_id=args.model_id,
task_type=args.task_type,
expected_model_version=args.source_version,
allowed_frameworks=("ultralytics/pytorch",),
)
db.commit()
production_evidence = RuntimeModelProvenanceService.validate_for_production_runtime(
db=db,
model_path=evidence["paths"]["model"],
model_id=args.model_id,
task_type=args.task_type,
expected_model_version=args.source_version,
allowed_frameworks=("ultralytics/pytorch",),
)
result.update(
{
"status": "applied",
"message": "Runtime model provenance was registered and validated against Postgres.",
"applied": True,
"manifest_path": str(manifest_path),
"manifest_created": created_manifest,
"source_registry_id": str(source.id),
"source_snapshot_id": str(snapshot.id),
"runtime_provenance": production_evidence.as_dict(),
}
)
return 0, result
except Exception as exc:
db.rollback()
if created_manifest and manifest_path.exists():
manifest_path.unlink()
return 3, {
**result,
"status": "apply_failed",
"message": str(exc),
"applied": False,
}
finally:
db.close()
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(
description=(
"Migrate a surviving legacy YOLO checkpoint into GeoIntel's runtime provenance contract without "
"claiming missing training or review evidence."
)
)
parser.add_argument("--model-path", required=True)
parser.add_argument("--checkpoint-path", required=True)
parser.add_argument("--base-model-path", required=True)
parser.add_argument("--training-summary-path", required=True)
parser.add_argument("--training-args-path", required=True)
parser.add_argument("--training-results-path", required=True)
parser.add_argument("--dataset-summary-path", required=True)
parser.add_argument("--dataset-yaml-path", required=True)
parser.add_argument("--model-id", default="yolo-configured")
parser.add_argument("--task-type", default="object_detection")
parser.add_argument("--source-version", required=True)
parser.add_argument("--framework-version", required=True)
parser.add_argument("--apply", action="store_true")
parser.add_argument("--json", action="store_true")
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> int:
args = parse_args(argv or sys.argv[1:])
exit_code, payload = migrate(args)
if args.json:
print(json.dumps(payload, indent=2, sort_keys=True))
else:
print(f"status: {payload['status']}")
print(f"message: {payload['message']}")
print(f"claim boundary: {payload.get('claim_boundary', CLAIM_BOUNDARY)}")
return exit_code
if __name__ == "__main__":
raise SystemExit(main())
+1
View File
@@ -125,6 +125,7 @@ ${PYTHON_BIN} -m py_compile scripts/render_detection_false_negative_review_conta
${PYTHON_BIN} -m py_compile scripts/validate_detection_false_positive_review_decisions.py
${PYTHON_BIN} -m py_compile scripts/validate_detection_false_negative_review_decisions.py
${PYTHON_BIN} -m py_compile scripts/activate_promoted_yolo_candidate.py
${PYTHON_BIN} -m py_compile scripts/migrate_runtime_model_provenance.py
${PYTHON_BIN} -m py_compile scripts/cleanup_demo_artifacts.py
${PYTHON_BIN} -m py_compile scripts/archive_technical_projects.py
${PYTHON_BIN} -m py_compile scripts/release_backup_guard.py