"""Fail-closed provenance validation for configured local model files. Configured YOLO and SAM weights are intentionally not trusted merely because a file exists on the server. Before an adapter is allowed to load local weights, this service verifies a neighbouring immutable sidecar manifest, validates the model artifact against ``geointel.model.pytorch@1.0.0`` and binds that sidecar to the server-owned source registry and source snapshot recorded in Postgres. ``validate_for_runtime`` remains a structural sidecar check for catalog and preflight inspection. Production inference must call ``validate_for_production_runtime`` with a database session; it rejects an unregistered, mismatched, stale, unsafe or quarantined source snapshot before an adapter can load model bytes. This keeps focused unit tests able to inspect sidecars without inventing database rows while keeping the production boundary strict. """ from __future__ import annotations from dataclasses import dataclass from datetime import datetime, timezone from hashlib import sha256 import json from pathlib import Path import re from typing import Any, Mapping from uuid import UUID from app.core.errors import AppError from app.models import SourceRegistry, SourceSnapshot from app.services.data_contract_validation import ( PYTORCH_MODEL_CONTRACT_KEY, PYTORCH_MODEL_CONTRACT_VERSION, LineageEvidence, TransformationEvidence, build_model_validation_input, validate_registered_asset, ) _SHA256 = re.compile(r"^[0-9a-f]{64}$") _CONSUMABLE_FRESHNESS = {"current", "not_applicable"} _SAFE_SOURCE_REGISTRY_INGEST_STATUSES = {"configured", "ingested"} _SAFE_SOURCE_SNAPSHOT_INGEST_STATUS = "ingested" @dataclass(frozen=True) class RuntimeModelProvenance: """Validated, immutable evidence attached to one local inference run.""" model_id: str task_type: str model_path: str manifest_path: str model_sha256: str manifest_sha256: str runtime_manifest_sha256: str data_contract_key: str data_contract_version: str validation_report_sha256: str source_registry_id: str source_snapshot_id: str source_snapshot_checksum_sha256: str source_version: str def as_dict(self) -> dict[str, str]: return { "model_id": self.model_id, "task_type": self.task_type, "model_path": self.model_path, "manifest_path": self.manifest_path, "model_sha256": self.model_sha256, "manifest_sha256": self.manifest_sha256, "runtime_manifest_sha256": self.runtime_manifest_sha256, "data_contract_key": self.data_contract_key, "data_contract_version": self.data_contract_version, "validation_report_sha256": self.validation_report_sha256, "source_registry_id": self.source_registry_id, "source_snapshot_id": self.source_snapshot_id, "source_snapshot_checksum_sha256": self.source_snapshot_checksum_sha256, "source_version": self.source_version, } class RuntimeModelProvenanceService: """Validate model sidecars and their production database binding. A sidecar lives next to its model as ``.geointel-model.json``. Its ``metadata.runtime_manifest_sha256`` is the SHA-256 of canonical JSON after omitting that one self-referential field. Any other mutation of the manifest therefore invalidates it. Structural validation is deliberately separate from :meth:`validate_for_production_runtime`: discovery and preflight have no database session, whereas every production inference path must prove an active source registry/snapshot binding. """ MANIFEST_SUFFIX = ".geointel-model.json" MANIFEST_SCHEMA_VERSION = "geointel.runtime-model-manifest/v1" SOURCE_REGISTRY_KEY = "model" @classmethod def manifest_path_for_model(cls, model_path: str | Path) -> Path: path = Path(model_path).expanduser() return Path(f"{path}{cls.MANIFEST_SUFFIX}") @staticmethod def manifest_self_checksum(payload: Mapping[str, Any]) -> str: """Hash sidecar semantics without its self-referential checksum field.""" canonical_payload = json.loads(json.dumps(payload, sort_keys=True, ensure_ascii=True)) metadata = canonical_payload.get("metadata") if isinstance(metadata, dict): metadata.pop("runtime_manifest_sha256", None) encoded = json.dumps( canonical_payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True, ).encode("utf-8") return sha256(encoded).hexdigest() @classmethod def validate_for_runtime( cls, *, model_path: str | Path, model_id: str, task_type: str, expected_model_version: str | None = None, allowed_frameworks: tuple[str, ...] = (), ) -> RuntimeModelProvenance: """Return structural sidecar evidence without asserting database state. This method is appropriate for read-only catalog/preflight checks and focused sidecar unit tests. It is insufficient for production inference; adapters must use :meth:`validate_for_production_runtime`. """ path = Path(model_path).expanduser() if not path.exists() or not path.is_file(): cls._raise( "MODEL_PROVENANCE_MODEL_FILE_MISSING", "Configured model file is missing; runtime provenance cannot be verified.", model_path=str(path), ) manifest_path = cls.manifest_path_for_model(path) if not manifest_path.exists() or not manifest_path.is_file(): cls._raise( "MODEL_PROVENANCE_MANIFEST_MISSING", "Configured model requires an immutable .geointel-model.json sidecar before inference.", model_path=str(path), manifest_path=str(manifest_path), ) try: raw_manifest = manifest_path.read_bytes() payload = json.loads(raw_manifest.decode("utf-8")) except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: cls._raise( "MODEL_PROVENANCE_MANIFEST_INVALID", "Configured model sidecar must be a readable UTF-8 JSON object.", manifest_path=str(manifest_path), error_type=type(exc).__name__, ) if not isinstance(payload, dict): cls._raise( "MODEL_PROVENANCE_MANIFEST_INVALID", "Configured model sidecar must contain a JSON object.", manifest_path=str(manifest_path), ) cls._require_exact_text( payload.get("schema_version"), cls.MANIFEST_SCHEMA_VERSION, field="schema_version", manifest_path=manifest_path, ) contract = cls._require_mapping(payload, "data_contract", manifest_path) contract_key = cls._require_text(contract, "key", manifest_path) contract_version = cls._require_text(contract, "version", manifest_path) # A valid manifest for a different artifact family must never make a # local PyTorch/SAM weight executable. The structural validator below # has a registry lookup too, but pinning the identity here keeps this # runtime gate fail-closed if more model contracts are introduced. cls._require_exact_text( contract_key, PYTORCH_MODEL_CONTRACT_KEY, field="data_contract.key", manifest_path=manifest_path, ) cls._require_exact_text( contract_version, PYTORCH_MODEL_CONTRACT_VERSION, field="data_contract.version", manifest_path=manifest_path, ) model = cls._require_mapping(payload, "model", manifest_path) declared_model_id = cls._require_text(model, "model_id", manifest_path) declared_task_type = cls._require_text(model, "task_type", manifest_path) cls._require_exact_text(declared_model_id, model_id, field="model.model_id", manifest_path=manifest_path) cls._require_exact_text(declared_task_type, task_type, field="model.task_type", manifest_path=manifest_path) declared_checksum = cls._require_checksum(model.get("sha256"), "model.sha256", manifest_path) model_checksum = cls._file_sha256(path) if declared_checksum != model_checksum: cls._raise( "MODEL_PROVENANCE_MODEL_CHECKSUM_MISMATCH", "Model bytes do not match the checksum bound by the runtime sidecar.", model_path=str(path), expected=declared_checksum, observed=model_checksum, ) model_format = cls._require_text(model, "model_format", manifest_path) framework = cls._require_text(model, "framework", manifest_path) class_mapping = model.get("class_mapping") if not isinstance(class_mapping, (dict, list, tuple)) or not class_mapping: cls._raise( "MODEL_PROVENANCE_MANIFEST_INVALID", "model.class_mapping must be a non-empty mapping or sequence.", manifest_path=str(manifest_path), ) normalized_framework = framework.strip().lower() if allowed_frameworks and normalized_framework not in {value.strip().lower() for value in allowed_frameworks}: cls._raise( "MODEL_PROVENANCE_FRAMEWORK_MISMATCH", "Model framework does not match the configured runtime adapter.", manifest_path=str(manifest_path), expected=sorted({value.strip().lower() for value in allowed_frameworks}), observed=framework, ) source_version = cls._require_text(model, "source_version", manifest_path) if expected_model_version and source_version != expected_model_version: cls._raise( "MODEL_PROVENANCE_VERSION_MISMATCH", "Model sidecar version does not match the configured model version.", manifest_path=str(manifest_path), expected=expected_model_version, observed=source_version, ) source = cls._require_mapping(payload, "source", manifest_path) source_registry_id = cls._require_uuid(source.get("source_registry_id"), "source.source_registry_id", manifest_path) source_snapshot_id = cls._require_uuid(source.get("source_snapshot_id"), "source.source_snapshot_id", manifest_path) cls._require_exact_text( cls._require_text(source, "source_registry_key", manifest_path), cls.SOURCE_REGISTRY_KEY, field="source.source_registry_key", manifest_path=manifest_path, ) snapshot_checksum = cls._require_checksum( source.get("source_snapshot_checksum_sha256"), "source.source_snapshot_checksum_sha256", manifest_path, ) if snapshot_checksum != model_checksum: cls._raise( "MODEL_PROVENANCE_SNAPSHOT_CHECKSUM_MISMATCH", "Model source snapshot checksum must bind the exact model bytes.", manifest_path=str(manifest_path), expected=model_checksum, observed=snapshot_checksum, ) metadata = cls._require_mapping(payload, "metadata", manifest_path) training_manifest_sha256 = cls._require_checksum( metadata.get("training_manifest_sha256"), "metadata.training_manifest_sha256", manifest_path, ) declared_runtime_manifest_sha256 = cls._require_checksum( metadata.get("runtime_manifest_sha256"), "metadata.runtime_manifest_sha256", manifest_path, ) computed_runtime_manifest_sha256 = cls.manifest_self_checksum(payload) if declared_runtime_manifest_sha256 != computed_runtime_manifest_sha256: cls._raise( "MODEL_PROVENANCE_MANIFEST_CHECKSUM_MISMATCH", "Runtime model sidecar integrity checksum does not match its canonical contents.", manifest_path=str(manifest_path), expected=declared_runtime_manifest_sha256, observed=computed_runtime_manifest_sha256, ) lineage = cls._lineage_evidence(payload, manifest_path) imported_at = cls._parse_imported_at(payload.get("imported_at"), manifest_path) report = validate_registered_asset( build_model_validation_input( asset_id=f"{model_id}:{model_checksum}", model_metadata={ "model_format": model_format, "framework": framework, "class_mapping": class_mapping, }, checksum_sha256=declared_checksum, computed_checksum_sha256=model_checksum, source_registry_id=source_registry_id, source_snapshot_id=source_snapshot_id, imported_at=imported_at, metadata={ "training_manifest_sha256": training_manifest_sha256, "runtime_manifest_sha256": declared_runtime_manifest_sha256, }, source_version=source_version, lineage=lineage, data_contract_key=contract_key, data_contract_version=contract_version, ) ) if report.failed: cls._raise( "MODEL_PROVENANCE_CONTRACT_FAILED", "Configured model sidecar failed the exact versioned model data contract.", manifest_path=str(manifest_path), data_contract=f"{contract_key}@{contract_version}", issue_codes=[issue.code for issue in report.issues], validation_report_sha256=report.report_sha256, ) return RuntimeModelProvenance( model_id=model_id, task_type=task_type, model_path=str(path.resolve()), manifest_path=str(manifest_path.resolve()), model_sha256=model_checksum, manifest_sha256=sha256(raw_manifest).hexdigest(), runtime_manifest_sha256=declared_runtime_manifest_sha256, data_contract_key=contract_key, data_contract_version=contract_version, validation_report_sha256=report.report_sha256, source_registry_id=source_registry_id, source_snapshot_id=source_snapshot_id, source_snapshot_checksum_sha256=snapshot_checksum, source_version=source_version, ) @classmethod def validate_for_production_runtime( cls, *, db: Any, model_path: str | Path, model_id: str, task_type: str, expected_model_version: str | None = None, allowed_frameworks: tuple[str, ...] = (), ) -> RuntimeModelProvenance: """Validate byte-bound model evidence against governed database state. The sidecar is not itself a source of authority. The model bytes may enter production inference only when the declared source registry and immutable source snapshot both exist, belong together, are safe to consume and bind the same SHA-256 and source version as the sidecar. This check is intentionally invoked immediately before adapter loading. """ if db is None or not callable(getattr(db, "get", None)): cls._raise( "MODEL_PROVENANCE_DATABASE_REQUIRED", "Production model inference requires a database session for source snapshot provenance.", ) evidence = cls.validate_for_runtime( model_path=model_path, model_id=model_id, task_type=task_type, expected_model_version=expected_model_version, allowed_frameworks=allowed_frameworks, ) cls._assert_database_binding(db, evidence) return evidence @classmethod def _assert_database_binding(cls, db: Any, evidence: RuntimeModelProvenance) -> None: registry_id = UUID(evidence.source_registry_id) snapshot_id = UUID(evidence.source_snapshot_id) source_registry = db.get(SourceRegistry, registry_id) if not isinstance(source_registry, SourceRegistry): cls._raise( "MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND", "Configured model sidecar refers to a source registry record that does not exist.", source_registry_id=evidence.source_registry_id, model_id=evidence.model_id, ) source_snapshot = db.get(SourceSnapshot, snapshot_id) if not isinstance(source_snapshot, SourceSnapshot): cls._raise( "MODEL_PROVENANCE_SOURCE_SNAPSHOT_NOT_FOUND", "Configured model sidecar refers to a source snapshot record that does not exist.", source_snapshot_id=evidence.source_snapshot_id, model_id=evidence.model_id, ) if str(source_registry.id) != evidence.source_registry_id or source_registry.source_key != cls.SOURCE_REGISTRY_KEY: cls._raise( "MODEL_PROVENANCE_SOURCE_REGISTRY_IDENTITY_MISMATCH", "Database source registry does not match the immutable model sidecar identity.", expected_source_registry_id=evidence.source_registry_id, observed_source_registry_id=str(source_registry.id), expected_source_key=cls.SOURCE_REGISTRY_KEY, observed_source_key=source_registry.source_key, ) if str(source_snapshot.id) != evidence.source_snapshot_id or str(source_snapshot.source_registry_id) != evidence.source_registry_id: cls._raise( "MODEL_PROVENANCE_SOURCE_SNAPSHOT_REGISTRY_MISMATCH", "Model source snapshot does not belong to the declared source registry.", source_registry_id=evidence.source_registry_id, source_snapshot_id=evidence.source_snapshot_id, observed_snapshot_registry_id=str(source_snapshot.source_registry_id), ) registry_ingest_status = cls._normalise_status(source_registry.ingest_status) registry_freshness_status = cls._normalise_status(source_registry.freshness_status) if ( registry_ingest_status not in _SAFE_SOURCE_REGISTRY_INGEST_STATUSES or registry_freshness_status not in _CONSUMABLE_FRESHNESS ): cls._raise( "MODEL_PROVENANCE_SOURCE_REGISTRY_UNSAFE", "Configured model source registry is not in a safe configured/current state.", source_registry_id=evidence.source_registry_id, ingest_status=registry_ingest_status, freshness_status=registry_freshness_status, ) snapshot_ingest_status = cls._normalise_status(source_snapshot.ingest_status) snapshot_freshness_status = cls._normalise_status(source_snapshot.freshness_status) if ( snapshot_ingest_status != _SAFE_SOURCE_SNAPSHOT_INGEST_STATUS or snapshot_freshness_status not in _CONSUMABLE_FRESHNESS ): cls._raise( "MODEL_PROVENANCE_SOURCE_SNAPSHOT_UNSAFE", "Configured model source snapshot is not an ingested current immutable artifact.", source_snapshot_id=evidence.source_snapshot_id, ingest_status=snapshot_ingest_status, freshness_status=snapshot_freshness_status, ) snapshot_checksum = str(source_snapshot.checksum_sha256 or "").strip().lower() if snapshot_checksum != evidence.source_snapshot_checksum_sha256 or snapshot_checksum != evidence.model_sha256: cls._raise( "MODEL_PROVENANCE_DATABASE_SNAPSHOT_CHECKSUM_MISMATCH", "Database model source snapshot checksum does not bind the exact sidecar and model bytes.", source_snapshot_id=evidence.source_snapshot_id, expected_model_sha256=evidence.model_sha256, expected_sidecar_snapshot_sha256=evidence.source_snapshot_checksum_sha256, observed_snapshot_sha256=snapshot_checksum, ) snapshot_version = str(source_snapshot.source_version or "").strip() if snapshot_version != evidence.source_version: cls._raise( "MODEL_PROVENANCE_SOURCE_SNAPSHOT_VERSION_MISMATCH", "Database model source snapshot version does not match the immutable sidecar model version.", source_snapshot_id=evidence.source_snapshot_id, expected_source_version=evidence.source_version, observed_source_version=snapshot_version or None, ) # The persisted snapshot state is the primary quarantine signal. The # relationship check is a defensive second line for an active # quarantine record that predates or bypassed a status transition. active_quarantines = getattr(source_snapshot, "quarantines", ()) if any(cls._normalise_status(getattr(item, "status", None)) == "quarantined" for item in active_quarantines): cls._raise( "MODEL_PROVENANCE_SOURCE_SNAPSHOT_QUARANTINED", "Configured model source snapshot has an active quarantine record.", source_snapshot_id=evidence.source_snapshot_id, ) @staticmethod def _normalise_status(value: Any) -> str: return str(value or "").strip().lower() @classmethod def _lineage_evidence(cls, payload: Mapping[str, Any], manifest_path: Path) -> LineageEvidence: lineage = cls._require_mapping(payload, "lineage", manifest_path) raw_asset_ids = lineage.get("upstream_asset_ids") raw_checksums = lineage.get("upstream_checksums_sha256") raw_transformations = lineage.get("transformations") if not isinstance(raw_asset_ids, list) or not raw_asset_ids or not all( isinstance(value, str) and value.strip() for value in raw_asset_ids ): cls._raise( "MODEL_PROVENANCE_MANIFEST_INVALID", "lineage.upstream_asset_ids must be a non-empty string list.", manifest_path=str(manifest_path), ) if not isinstance(raw_checksums, list) or len(raw_checksums) != len(raw_asset_ids): cls._raise( "MODEL_PROVENANCE_MANIFEST_INVALID", "lineage.upstream_checksums_sha256 must match upstream_asset_ids one-for-one.", manifest_path=str(manifest_path), ) upstream_checksums = tuple( cls._require_checksum(value, f"lineage.upstream_checksums_sha256[{index}]", manifest_path) for index, value in enumerate(raw_checksums) ) if not isinstance(raw_transformations, list) or not raw_transformations: cls._raise( "MODEL_PROVENANCE_MANIFEST_INVALID", "lineage.transformations must contain at least one immutable transformation record.", manifest_path=str(manifest_path), ) transformations: list[TransformationEvidence] = [] for index, raw in enumerate(raw_transformations): if not isinstance(raw, Mapping): cls._raise( "MODEL_PROVENANCE_MANIFEST_INVALID", "Each lineage transformation must be an object.", manifest_path=str(manifest_path), index=index, ) transformations.append( TransformationEvidence( name=cls._require_text(raw, "name", manifest_path, prefix=f"lineage.transformations[{index}]."), version=cls._require_text(raw, "version", manifest_path, prefix=f"lineage.transformations[{index}]."), checksum_sha256=cls._require_checksum( raw.get("checksum_sha256"), f"lineage.transformations[{index}].checksum_sha256", manifest_path, ), ) ) return LineageEvidence( upstream_asset_ids=tuple(raw_asset_ids), upstream_checksums_sha256=upstream_checksums, transformations=tuple(transformations), ) @staticmethod def _file_sha256(path: Path) -> str: digest = sha256() try: with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) except OSError as exc: RuntimeModelProvenanceService._raise( "MODEL_PROVENANCE_MODEL_FILE_UNREADABLE", "Configured model file could not be read for checksum validation.", model_path=str(path), error_type=type(exc).__name__, ) return digest.hexdigest() @classmethod def _parse_imported_at(cls, value: Any, manifest_path: Path) -> datetime: if not isinstance(value, str) or not value.strip(): cls._raise( "MODEL_PROVENANCE_MANIFEST_INVALID", "imported_at must be a timezone-aware ISO-8601 timestamp.", manifest_path=str(manifest_path), ) try: parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) except ValueError: cls._raise( "MODEL_PROVENANCE_MANIFEST_INVALID", "imported_at must be a timezone-aware ISO-8601 timestamp.", manifest_path=str(manifest_path), observed=value, ) if parsed.tzinfo is None: cls._raise( "MODEL_PROVENANCE_MANIFEST_INVALID", "imported_at must include a timezone offset.", manifest_path=str(manifest_path), observed=value, ) return parsed.astimezone(timezone.utc) @classmethod def _require_mapping(cls, payload: Mapping[str, Any], key: str, manifest_path: Path) -> Mapping[str, Any]: value = payload.get(key) if not isinstance(value, Mapping): cls._raise( "MODEL_PROVENANCE_MANIFEST_INVALID", f"{key} must be a JSON object.", manifest_path=str(manifest_path), ) return value @classmethod def _require_text( cls, payload: Mapping[str, Any], key: str, manifest_path: Path, *, prefix: str = "", ) -> str: value = payload.get(key) if not isinstance(value, str) or not value.strip(): cls._raise( "MODEL_PROVENANCE_MANIFEST_INVALID", f"{prefix}{key} must be a non-empty string.", manifest_path=str(manifest_path), ) return value.strip() @classmethod def _require_checksum(cls, value: Any, field: str, manifest_path: Path) -> str: normalized = value.strip().lower() if isinstance(value, str) else "" if not _SHA256.fullmatch(normalized) or value != normalized: cls._raise( "MODEL_PROVENANCE_MANIFEST_INVALID", f"{field} must be a lowercase SHA-256 digest.", manifest_path=str(manifest_path), ) return normalized @classmethod def _require_uuid(cls, value: Any, field: str, manifest_path: Path) -> str: if not isinstance(value, str) or not value.strip(): cls._raise( "MODEL_PROVENANCE_MANIFEST_INVALID", f"{field} must be a UUID string.", manifest_path=str(manifest_path), ) try: return str(UUID(value)) except (AttributeError, ValueError): cls._raise( "MODEL_PROVENANCE_MANIFEST_INVALID", f"{field} must be a UUID string.", manifest_path=str(manifest_path), observed=value, ) @classmethod def _require_exact_text( cls, observed: Any, expected: str, *, field: str, manifest_path: Path, ) -> None: if not isinstance(observed, str) or observed.strip() != expected: cls._raise( "MODEL_PROVENANCE_MANIFEST_INVALID", f"{field} does not match the configured runtime identity.", manifest_path=str(manifest_path), expected=expected, observed=observed, ) @staticmethod def _raise(code: str, message: str, **details: Any) -> None: raise AppError(code=code, message=message, details=details, status_code=422)