653 lines
29 KiB
Python
653 lines
29 KiB
Python
"""Fail-closed provenance validation for configured local model files.
|
|
|
|
Configured YOLO and SAM weights are intentionally not trusted merely because a
|
|
file exists on the server. Before an adapter is allowed to load local weights,
|
|
this service verifies a neighbouring immutable sidecar manifest, validates the
|
|
model artifact against ``geointel.model.pytorch@1.0.0`` and binds that sidecar
|
|
to the server-owned source registry and source snapshot recorded in Postgres.
|
|
|
|
``validate_for_runtime`` remains a structural sidecar check for catalog and
|
|
preflight inspection. Production inference must call
|
|
``validate_for_production_runtime`` with a database session; it rejects an
|
|
unregistered, mismatched, stale, unsafe or quarantined source snapshot before
|
|
an adapter can load model bytes. This keeps focused unit tests able to inspect
|
|
sidecars without inventing database rows while keeping the production boundary
|
|
strict.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from datetime import datetime, timezone
|
|
from hashlib import sha256
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
from typing import Any, Mapping
|
|
from uuid import UUID
|
|
|
|
from app.core.errors import AppError
|
|
from app.models import SourceRegistry, SourceSnapshot
|
|
from app.services.data_contract_validation import (
|
|
PYTORCH_MODEL_CONTRACT_KEY,
|
|
PYTORCH_MODEL_CONTRACT_VERSION,
|
|
LineageEvidence,
|
|
TransformationEvidence,
|
|
build_model_validation_input,
|
|
validate_registered_asset,
|
|
)
|
|
|
|
|
|
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
|
_CONSUMABLE_FRESHNESS = {"current", "not_applicable"}
|
|
_SAFE_SOURCE_REGISTRY_INGEST_STATUSES = {"configured", "ingested"}
|
|
_SAFE_SOURCE_SNAPSHOT_INGEST_STATUS = "ingested"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RuntimeModelProvenance:
|
|
"""Validated, immutable evidence attached to one local inference run."""
|
|
|
|
model_id: str
|
|
task_type: str
|
|
model_path: str
|
|
manifest_path: str
|
|
model_sha256: str
|
|
manifest_sha256: str
|
|
runtime_manifest_sha256: str
|
|
data_contract_key: str
|
|
data_contract_version: str
|
|
validation_report_sha256: str
|
|
source_registry_id: str
|
|
source_snapshot_id: str
|
|
source_snapshot_checksum_sha256: str
|
|
source_version: str
|
|
|
|
def as_dict(self) -> dict[str, str]:
|
|
return {
|
|
"model_id": self.model_id,
|
|
"task_type": self.task_type,
|
|
"model_path": self.model_path,
|
|
"manifest_path": self.manifest_path,
|
|
"model_sha256": self.model_sha256,
|
|
"manifest_sha256": self.manifest_sha256,
|
|
"runtime_manifest_sha256": self.runtime_manifest_sha256,
|
|
"data_contract_key": self.data_contract_key,
|
|
"data_contract_version": self.data_contract_version,
|
|
"validation_report_sha256": self.validation_report_sha256,
|
|
"source_registry_id": self.source_registry_id,
|
|
"source_snapshot_id": self.source_snapshot_id,
|
|
"source_snapshot_checksum_sha256": self.source_snapshot_checksum_sha256,
|
|
"source_version": self.source_version,
|
|
}
|
|
|
|
|
|
class RuntimeModelProvenanceService:
|
|
"""Validate model sidecars and their production database binding.
|
|
|
|
A sidecar lives next to its model as ``<model-file>.geointel-model.json``.
|
|
Its ``metadata.runtime_manifest_sha256`` is the SHA-256 of canonical JSON
|
|
after omitting that one self-referential field. Any other mutation of the
|
|
manifest therefore invalidates it. Structural validation is deliberately
|
|
separate from :meth:`validate_for_production_runtime`: discovery and
|
|
preflight have no database session, whereas every production inference
|
|
path must prove an active source registry/snapshot binding.
|
|
"""
|
|
|
|
MANIFEST_SUFFIX = ".geointel-model.json"
|
|
MANIFEST_SCHEMA_VERSION = "geointel.runtime-model-manifest/v1"
|
|
SOURCE_REGISTRY_KEY = "model"
|
|
|
|
@classmethod
|
|
def manifest_path_for_model(cls, model_path: str | Path) -> Path:
|
|
path = Path(model_path).expanduser()
|
|
return Path(f"{path}{cls.MANIFEST_SUFFIX}")
|
|
|
|
@staticmethod
|
|
def manifest_self_checksum(payload: Mapping[str, Any]) -> str:
|
|
"""Hash sidecar semantics without its self-referential checksum field."""
|
|
|
|
canonical_payload = json.loads(json.dumps(payload, sort_keys=True, ensure_ascii=True))
|
|
metadata = canonical_payload.get("metadata")
|
|
if isinstance(metadata, dict):
|
|
metadata.pop("runtime_manifest_sha256", None)
|
|
encoded = json.dumps(
|
|
canonical_payload,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
ensure_ascii=True,
|
|
).encode("utf-8")
|
|
return sha256(encoded).hexdigest()
|
|
|
|
@classmethod
|
|
def validate_for_runtime(
|
|
cls,
|
|
*,
|
|
model_path: str | Path,
|
|
model_id: str,
|
|
task_type: str,
|
|
expected_model_version: str | None = None,
|
|
allowed_frameworks: tuple[str, ...] = (),
|
|
) -> RuntimeModelProvenance:
|
|
"""Return structural sidecar evidence without asserting database state.
|
|
|
|
This method is appropriate for read-only catalog/preflight checks and
|
|
focused sidecar unit tests. It is insufficient for production
|
|
inference; adapters must use :meth:`validate_for_production_runtime`.
|
|
"""
|
|
|
|
path = Path(model_path).expanduser()
|
|
if not path.exists() or not path.is_file():
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_MODEL_FILE_MISSING",
|
|
"Configured model file is missing; runtime provenance cannot be verified.",
|
|
model_path=str(path),
|
|
)
|
|
manifest_path = cls.manifest_path_for_model(path)
|
|
if not manifest_path.exists() or not manifest_path.is_file():
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_MANIFEST_MISSING",
|
|
"Configured model requires an immutable .geointel-model.json sidecar before inference.",
|
|
model_path=str(path),
|
|
manifest_path=str(manifest_path),
|
|
)
|
|
|
|
try:
|
|
raw_manifest = manifest_path.read_bytes()
|
|
payload = json.loads(raw_manifest.decode("utf-8"))
|
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
|
"Configured model sidecar must be a readable UTF-8 JSON object.",
|
|
manifest_path=str(manifest_path),
|
|
error_type=type(exc).__name__,
|
|
)
|
|
if not isinstance(payload, dict):
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
|
"Configured model sidecar must contain a JSON object.",
|
|
manifest_path=str(manifest_path),
|
|
)
|
|
|
|
cls._require_exact_text(
|
|
payload.get("schema_version"),
|
|
cls.MANIFEST_SCHEMA_VERSION,
|
|
field="schema_version",
|
|
manifest_path=manifest_path,
|
|
)
|
|
contract = cls._require_mapping(payload, "data_contract", manifest_path)
|
|
contract_key = cls._require_text(contract, "key", manifest_path)
|
|
contract_version = cls._require_text(contract, "version", manifest_path)
|
|
# A valid manifest for a different artifact family must never make a
|
|
# local PyTorch/SAM weight executable. The structural validator below
|
|
# has a registry lookup too, but pinning the identity here keeps this
|
|
# runtime gate fail-closed if more model contracts are introduced.
|
|
cls._require_exact_text(
|
|
contract_key,
|
|
PYTORCH_MODEL_CONTRACT_KEY,
|
|
field="data_contract.key",
|
|
manifest_path=manifest_path,
|
|
)
|
|
cls._require_exact_text(
|
|
contract_version,
|
|
PYTORCH_MODEL_CONTRACT_VERSION,
|
|
field="data_contract.version",
|
|
manifest_path=manifest_path,
|
|
)
|
|
model = cls._require_mapping(payload, "model", manifest_path)
|
|
declared_model_id = cls._require_text(model, "model_id", manifest_path)
|
|
declared_task_type = cls._require_text(model, "task_type", manifest_path)
|
|
cls._require_exact_text(declared_model_id, model_id, field="model.model_id", manifest_path=manifest_path)
|
|
cls._require_exact_text(declared_task_type, task_type, field="model.task_type", manifest_path=manifest_path)
|
|
|
|
declared_checksum = cls._require_checksum(model.get("sha256"), "model.sha256", manifest_path)
|
|
model_checksum = cls._file_sha256(path)
|
|
if declared_checksum != model_checksum:
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_MODEL_CHECKSUM_MISMATCH",
|
|
"Model bytes do not match the checksum bound by the runtime sidecar.",
|
|
model_path=str(path),
|
|
expected=declared_checksum,
|
|
observed=model_checksum,
|
|
)
|
|
|
|
model_format = cls._require_text(model, "model_format", manifest_path)
|
|
framework = cls._require_text(model, "framework", manifest_path)
|
|
class_mapping = model.get("class_mapping")
|
|
if not isinstance(class_mapping, (dict, list, tuple)) or not class_mapping:
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
|
"model.class_mapping must be a non-empty mapping or sequence.",
|
|
manifest_path=str(manifest_path),
|
|
)
|
|
normalized_framework = framework.strip().lower()
|
|
if allowed_frameworks and normalized_framework not in {value.strip().lower() for value in allowed_frameworks}:
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_FRAMEWORK_MISMATCH",
|
|
"Model framework does not match the configured runtime adapter.",
|
|
manifest_path=str(manifest_path),
|
|
expected=sorted({value.strip().lower() for value in allowed_frameworks}),
|
|
observed=framework,
|
|
)
|
|
source_version = cls._require_text(model, "source_version", manifest_path)
|
|
if expected_model_version and source_version != expected_model_version:
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_VERSION_MISMATCH",
|
|
"Model sidecar version does not match the configured model version.",
|
|
manifest_path=str(manifest_path),
|
|
expected=expected_model_version,
|
|
observed=source_version,
|
|
)
|
|
|
|
source = cls._require_mapping(payload, "source", manifest_path)
|
|
source_registry_id = cls._require_uuid(source.get("source_registry_id"), "source.source_registry_id", manifest_path)
|
|
source_snapshot_id = cls._require_uuid(source.get("source_snapshot_id"), "source.source_snapshot_id", manifest_path)
|
|
cls._require_exact_text(
|
|
cls._require_text(source, "source_registry_key", manifest_path),
|
|
cls.SOURCE_REGISTRY_KEY,
|
|
field="source.source_registry_key",
|
|
manifest_path=manifest_path,
|
|
)
|
|
snapshot_checksum = cls._require_checksum(
|
|
source.get("source_snapshot_checksum_sha256"),
|
|
"source.source_snapshot_checksum_sha256",
|
|
manifest_path,
|
|
)
|
|
if snapshot_checksum != model_checksum:
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_SNAPSHOT_CHECKSUM_MISMATCH",
|
|
"Model source snapshot checksum must bind the exact model bytes.",
|
|
manifest_path=str(manifest_path),
|
|
expected=model_checksum,
|
|
observed=snapshot_checksum,
|
|
)
|
|
|
|
metadata = cls._require_mapping(payload, "metadata", manifest_path)
|
|
training_manifest_sha256 = cls._require_checksum(
|
|
metadata.get("training_manifest_sha256"),
|
|
"metadata.training_manifest_sha256",
|
|
manifest_path,
|
|
)
|
|
declared_runtime_manifest_sha256 = cls._require_checksum(
|
|
metadata.get("runtime_manifest_sha256"),
|
|
"metadata.runtime_manifest_sha256",
|
|
manifest_path,
|
|
)
|
|
computed_runtime_manifest_sha256 = cls.manifest_self_checksum(payload)
|
|
if declared_runtime_manifest_sha256 != computed_runtime_manifest_sha256:
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_MANIFEST_CHECKSUM_MISMATCH",
|
|
"Runtime model sidecar integrity checksum does not match its canonical contents.",
|
|
manifest_path=str(manifest_path),
|
|
expected=declared_runtime_manifest_sha256,
|
|
observed=computed_runtime_manifest_sha256,
|
|
)
|
|
|
|
lineage = cls._lineage_evidence(payload, manifest_path)
|
|
imported_at = cls._parse_imported_at(payload.get("imported_at"), manifest_path)
|
|
report = validate_registered_asset(
|
|
build_model_validation_input(
|
|
asset_id=f"{model_id}:{model_checksum}",
|
|
model_metadata={
|
|
"model_format": model_format,
|
|
"framework": framework,
|
|
"class_mapping": class_mapping,
|
|
},
|
|
checksum_sha256=declared_checksum,
|
|
computed_checksum_sha256=model_checksum,
|
|
source_registry_id=source_registry_id,
|
|
source_snapshot_id=source_snapshot_id,
|
|
imported_at=imported_at,
|
|
metadata={
|
|
"training_manifest_sha256": training_manifest_sha256,
|
|
"runtime_manifest_sha256": declared_runtime_manifest_sha256,
|
|
},
|
|
source_version=source_version,
|
|
lineage=lineage,
|
|
data_contract_key=contract_key,
|
|
data_contract_version=contract_version,
|
|
)
|
|
)
|
|
if report.failed:
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_CONTRACT_FAILED",
|
|
"Configured model sidecar failed the exact versioned model data contract.",
|
|
manifest_path=str(manifest_path),
|
|
data_contract=f"{contract_key}@{contract_version}",
|
|
issue_codes=[issue.code for issue in report.issues],
|
|
validation_report_sha256=report.report_sha256,
|
|
)
|
|
return RuntimeModelProvenance(
|
|
model_id=model_id,
|
|
task_type=task_type,
|
|
model_path=str(path.resolve()),
|
|
manifest_path=str(manifest_path.resolve()),
|
|
model_sha256=model_checksum,
|
|
manifest_sha256=sha256(raw_manifest).hexdigest(),
|
|
runtime_manifest_sha256=declared_runtime_manifest_sha256,
|
|
data_contract_key=contract_key,
|
|
data_contract_version=contract_version,
|
|
validation_report_sha256=report.report_sha256,
|
|
source_registry_id=source_registry_id,
|
|
source_snapshot_id=source_snapshot_id,
|
|
source_snapshot_checksum_sha256=snapshot_checksum,
|
|
source_version=source_version,
|
|
)
|
|
|
|
@classmethod
|
|
def validate_for_production_runtime(
|
|
cls,
|
|
*,
|
|
db: Any,
|
|
model_path: str | Path,
|
|
model_id: str,
|
|
task_type: str,
|
|
expected_model_version: str | None = None,
|
|
allowed_frameworks: tuple[str, ...] = (),
|
|
) -> RuntimeModelProvenance:
|
|
"""Validate byte-bound model evidence against governed database state.
|
|
|
|
The sidecar is not itself a source of authority. The model bytes may
|
|
enter production inference only when the declared source registry and
|
|
immutable source snapshot both exist, belong together, are safe to
|
|
consume and bind the same SHA-256 and source version as the sidecar.
|
|
This check is intentionally invoked immediately before adapter loading.
|
|
"""
|
|
|
|
if db is None or not callable(getattr(db, "get", None)):
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_DATABASE_REQUIRED",
|
|
"Production model inference requires a database session for source snapshot provenance.",
|
|
)
|
|
|
|
evidence = cls.validate_for_runtime(
|
|
model_path=model_path,
|
|
model_id=model_id,
|
|
task_type=task_type,
|
|
expected_model_version=expected_model_version,
|
|
allowed_frameworks=allowed_frameworks,
|
|
)
|
|
cls._assert_database_binding(db, evidence)
|
|
return evidence
|
|
|
|
@classmethod
|
|
def _assert_database_binding(cls, db: Any, evidence: RuntimeModelProvenance) -> None:
|
|
registry_id = UUID(evidence.source_registry_id)
|
|
snapshot_id = UUID(evidence.source_snapshot_id)
|
|
source_registry = db.get(SourceRegistry, registry_id)
|
|
if not isinstance(source_registry, SourceRegistry):
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND",
|
|
"Configured model sidecar refers to a source registry record that does not exist.",
|
|
source_registry_id=evidence.source_registry_id,
|
|
model_id=evidence.model_id,
|
|
)
|
|
source_snapshot = db.get(SourceSnapshot, snapshot_id)
|
|
if not isinstance(source_snapshot, SourceSnapshot):
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_NOT_FOUND",
|
|
"Configured model sidecar refers to a source snapshot record that does not exist.",
|
|
source_snapshot_id=evidence.source_snapshot_id,
|
|
model_id=evidence.model_id,
|
|
)
|
|
|
|
if str(source_registry.id) != evidence.source_registry_id or source_registry.source_key != cls.SOURCE_REGISTRY_KEY:
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_SOURCE_REGISTRY_IDENTITY_MISMATCH",
|
|
"Database source registry does not match the immutable model sidecar identity.",
|
|
expected_source_registry_id=evidence.source_registry_id,
|
|
observed_source_registry_id=str(source_registry.id),
|
|
expected_source_key=cls.SOURCE_REGISTRY_KEY,
|
|
observed_source_key=source_registry.source_key,
|
|
)
|
|
if str(source_snapshot.id) != evidence.source_snapshot_id or str(source_snapshot.source_registry_id) != evidence.source_registry_id:
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_REGISTRY_MISMATCH",
|
|
"Model source snapshot does not belong to the declared source registry.",
|
|
source_registry_id=evidence.source_registry_id,
|
|
source_snapshot_id=evidence.source_snapshot_id,
|
|
observed_snapshot_registry_id=str(source_snapshot.source_registry_id),
|
|
)
|
|
|
|
registry_ingest_status = cls._normalise_status(source_registry.ingest_status)
|
|
registry_freshness_status = cls._normalise_status(source_registry.freshness_status)
|
|
if (
|
|
registry_ingest_status not in _SAFE_SOURCE_REGISTRY_INGEST_STATUSES
|
|
or registry_freshness_status not in _CONSUMABLE_FRESHNESS
|
|
):
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_SOURCE_REGISTRY_UNSAFE",
|
|
"Configured model source registry is not in a safe configured/current state.",
|
|
source_registry_id=evidence.source_registry_id,
|
|
ingest_status=registry_ingest_status,
|
|
freshness_status=registry_freshness_status,
|
|
)
|
|
|
|
snapshot_ingest_status = cls._normalise_status(source_snapshot.ingest_status)
|
|
snapshot_freshness_status = cls._normalise_status(source_snapshot.freshness_status)
|
|
if (
|
|
snapshot_ingest_status != _SAFE_SOURCE_SNAPSHOT_INGEST_STATUS
|
|
or snapshot_freshness_status not in _CONSUMABLE_FRESHNESS
|
|
):
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_UNSAFE",
|
|
"Configured model source snapshot is not an ingested current immutable artifact.",
|
|
source_snapshot_id=evidence.source_snapshot_id,
|
|
ingest_status=snapshot_ingest_status,
|
|
freshness_status=snapshot_freshness_status,
|
|
)
|
|
|
|
snapshot_checksum = str(source_snapshot.checksum_sha256 or "").strip().lower()
|
|
if snapshot_checksum != evidence.source_snapshot_checksum_sha256 or snapshot_checksum != evidence.model_sha256:
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_DATABASE_SNAPSHOT_CHECKSUM_MISMATCH",
|
|
"Database model source snapshot checksum does not bind the exact sidecar and model bytes.",
|
|
source_snapshot_id=evidence.source_snapshot_id,
|
|
expected_model_sha256=evidence.model_sha256,
|
|
expected_sidecar_snapshot_sha256=evidence.source_snapshot_checksum_sha256,
|
|
observed_snapshot_sha256=snapshot_checksum,
|
|
)
|
|
|
|
snapshot_version = str(source_snapshot.source_version or "").strip()
|
|
if snapshot_version != evidence.source_version:
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_VERSION_MISMATCH",
|
|
"Database model source snapshot version does not match the immutable sidecar model version.",
|
|
source_snapshot_id=evidence.source_snapshot_id,
|
|
expected_source_version=evidence.source_version,
|
|
observed_source_version=snapshot_version or None,
|
|
)
|
|
|
|
# The persisted snapshot state is the primary quarantine signal. The
|
|
# relationship check is a defensive second line for an active
|
|
# quarantine record that predates or bypassed a status transition.
|
|
active_quarantines = getattr(source_snapshot, "quarantines", ())
|
|
if any(cls._normalise_status(getattr(item, "status", None)) == "quarantined" for item in active_quarantines):
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_QUARANTINED",
|
|
"Configured model source snapshot has an active quarantine record.",
|
|
source_snapshot_id=evidence.source_snapshot_id,
|
|
)
|
|
|
|
@staticmethod
|
|
def _normalise_status(value: Any) -> str:
|
|
return str(value or "").strip().lower()
|
|
|
|
@classmethod
|
|
def _lineage_evidence(cls, payload: Mapping[str, Any], manifest_path: Path) -> LineageEvidence:
|
|
lineage = cls._require_mapping(payload, "lineage", manifest_path)
|
|
raw_asset_ids = lineage.get("upstream_asset_ids")
|
|
raw_checksums = lineage.get("upstream_checksums_sha256")
|
|
raw_transformations = lineage.get("transformations")
|
|
if not isinstance(raw_asset_ids, list) or not raw_asset_ids or not all(
|
|
isinstance(value, str) and value.strip() for value in raw_asset_ids
|
|
):
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
|
"lineage.upstream_asset_ids must be a non-empty string list.",
|
|
manifest_path=str(manifest_path),
|
|
)
|
|
if not isinstance(raw_checksums, list) or len(raw_checksums) != len(raw_asset_ids):
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
|
"lineage.upstream_checksums_sha256 must match upstream_asset_ids one-for-one.",
|
|
manifest_path=str(manifest_path),
|
|
)
|
|
upstream_checksums = tuple(
|
|
cls._require_checksum(value, f"lineage.upstream_checksums_sha256[{index}]", manifest_path)
|
|
for index, value in enumerate(raw_checksums)
|
|
)
|
|
if not isinstance(raw_transformations, list) or not raw_transformations:
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
|
"lineage.transformations must contain at least one immutable transformation record.",
|
|
manifest_path=str(manifest_path),
|
|
)
|
|
transformations: list[TransformationEvidence] = []
|
|
for index, raw in enumerate(raw_transformations):
|
|
if not isinstance(raw, Mapping):
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
|
"Each lineage transformation must be an object.",
|
|
manifest_path=str(manifest_path),
|
|
index=index,
|
|
)
|
|
transformations.append(
|
|
TransformationEvidence(
|
|
name=cls._require_text(raw, "name", manifest_path, prefix=f"lineage.transformations[{index}]."),
|
|
version=cls._require_text(raw, "version", manifest_path, prefix=f"lineage.transformations[{index}]."),
|
|
checksum_sha256=cls._require_checksum(
|
|
raw.get("checksum_sha256"),
|
|
f"lineage.transformations[{index}].checksum_sha256",
|
|
manifest_path,
|
|
),
|
|
)
|
|
)
|
|
return LineageEvidence(
|
|
upstream_asset_ids=tuple(raw_asset_ids),
|
|
upstream_checksums_sha256=upstream_checksums,
|
|
transformations=tuple(transformations),
|
|
)
|
|
|
|
@staticmethod
|
|
def _file_sha256(path: Path) -> str:
|
|
digest = sha256()
|
|
try:
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
except OSError as exc:
|
|
RuntimeModelProvenanceService._raise(
|
|
"MODEL_PROVENANCE_MODEL_FILE_UNREADABLE",
|
|
"Configured model file could not be read for checksum validation.",
|
|
model_path=str(path),
|
|
error_type=type(exc).__name__,
|
|
)
|
|
return digest.hexdigest()
|
|
|
|
@classmethod
|
|
def _parse_imported_at(cls, value: Any, manifest_path: Path) -> datetime:
|
|
if not isinstance(value, str) or not value.strip():
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
|
"imported_at must be a timezone-aware ISO-8601 timestamp.",
|
|
manifest_path=str(manifest_path),
|
|
)
|
|
try:
|
|
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
except ValueError:
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
|
"imported_at must be a timezone-aware ISO-8601 timestamp.",
|
|
manifest_path=str(manifest_path),
|
|
observed=value,
|
|
)
|
|
if parsed.tzinfo is None:
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
|
"imported_at must include a timezone offset.",
|
|
manifest_path=str(manifest_path),
|
|
observed=value,
|
|
)
|
|
return parsed.astimezone(timezone.utc)
|
|
|
|
@classmethod
|
|
def _require_mapping(cls, payload: Mapping[str, Any], key: str, manifest_path: Path) -> Mapping[str, Any]:
|
|
value = payload.get(key)
|
|
if not isinstance(value, Mapping):
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
|
f"{key} must be a JSON object.",
|
|
manifest_path=str(manifest_path),
|
|
)
|
|
return value
|
|
|
|
@classmethod
|
|
def _require_text(
|
|
cls,
|
|
payload: Mapping[str, Any],
|
|
key: str,
|
|
manifest_path: Path,
|
|
*,
|
|
prefix: str = "",
|
|
) -> str:
|
|
value = payload.get(key)
|
|
if not isinstance(value, str) or not value.strip():
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
|
f"{prefix}{key} must be a non-empty string.",
|
|
manifest_path=str(manifest_path),
|
|
)
|
|
return value.strip()
|
|
|
|
@classmethod
|
|
def _require_checksum(cls, value: Any, field: str, manifest_path: Path) -> str:
|
|
normalized = value.strip().lower() if isinstance(value, str) else ""
|
|
if not _SHA256.fullmatch(normalized) or value != normalized:
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
|
f"{field} must be a lowercase SHA-256 digest.",
|
|
manifest_path=str(manifest_path),
|
|
)
|
|
return normalized
|
|
|
|
@classmethod
|
|
def _require_uuid(cls, value: Any, field: str, manifest_path: Path) -> str:
|
|
if not isinstance(value, str) or not value.strip():
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
|
f"{field} must be a UUID string.",
|
|
manifest_path=str(manifest_path),
|
|
)
|
|
try:
|
|
return str(UUID(value))
|
|
except (AttributeError, ValueError):
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
|
f"{field} must be a UUID string.",
|
|
manifest_path=str(manifest_path),
|
|
observed=value,
|
|
)
|
|
|
|
@classmethod
|
|
def _require_exact_text(
|
|
cls,
|
|
observed: Any,
|
|
expected: str,
|
|
*,
|
|
field: str,
|
|
manifest_path: Path,
|
|
) -> None:
|
|
if not isinstance(observed, str) or observed.strip() != expected:
|
|
cls._raise(
|
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
|
f"{field} does not match the configured runtime identity.",
|
|
manifest_path=str(manifest_path),
|
|
expected=expected,
|
|
observed=observed,
|
|
)
|
|
|
|
@staticmethod
|
|
def _raise(code: str, message: str, **details: Any) -> None:
|
|
raise AppError(code=code, message=message, details=details, status_code=422)
|