Initial public release
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
This commit is contained in:
@@ -0,0 +1,652 @@
|
||||
"""Fail-closed provenance validation for configured local model files.
|
||||
|
||||
Configured YOLO and SAM weights are intentionally not trusted merely because a
|
||||
file exists on the server. Before an adapter is allowed to load local weights,
|
||||
this service verifies a neighbouring immutable sidecar manifest, validates the
|
||||
model artifact against ``geointel.model.pytorch@1.0.0`` and binds that sidecar
|
||||
to the server-owned source registry and source snapshot recorded in Postgres.
|
||||
|
||||
``validate_for_runtime`` remains a structural sidecar check for catalog and
|
||||
preflight inspection. Production inference must call
|
||||
``validate_for_production_runtime`` with a database session; it rejects an
|
||||
unregistered, mismatched, stale, unsafe or quarantined source snapshot before
|
||||
an adapter can load model bytes. This keeps focused unit tests able to inspect
|
||||
sidecars without inventing database rows while keeping the production boundary
|
||||
strict.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
from typing import Any, Mapping
|
||||
from uuid import UUID
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import SourceRegistry, SourceSnapshot
|
||||
from app.services.data_contract_validation import (
|
||||
PYTORCH_MODEL_CONTRACT_KEY,
|
||||
PYTORCH_MODEL_CONTRACT_VERSION,
|
||||
LineageEvidence,
|
||||
TransformationEvidence,
|
||||
build_model_validation_input,
|
||||
validate_registered_asset,
|
||||
)
|
||||
|
||||
|
||||
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||
_CONSUMABLE_FRESHNESS = {"current", "not_applicable"}
|
||||
_SAFE_SOURCE_REGISTRY_INGEST_STATUSES = {"configured", "ingested"}
|
||||
_SAFE_SOURCE_SNAPSHOT_INGEST_STATUS = "ingested"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RuntimeModelProvenance:
|
||||
"""Validated, immutable evidence attached to one local inference run."""
|
||||
|
||||
model_id: str
|
||||
task_type: str
|
||||
model_path: str
|
||||
manifest_path: str
|
||||
model_sha256: str
|
||||
manifest_sha256: str
|
||||
runtime_manifest_sha256: str
|
||||
data_contract_key: str
|
||||
data_contract_version: str
|
||||
validation_report_sha256: str
|
||||
source_registry_id: str
|
||||
source_snapshot_id: str
|
||||
source_snapshot_checksum_sha256: str
|
||||
source_version: str
|
||||
|
||||
def as_dict(self) -> dict[str, str]:
|
||||
return {
|
||||
"model_id": self.model_id,
|
||||
"task_type": self.task_type,
|
||||
"model_path": self.model_path,
|
||||
"manifest_path": self.manifest_path,
|
||||
"model_sha256": self.model_sha256,
|
||||
"manifest_sha256": self.manifest_sha256,
|
||||
"runtime_manifest_sha256": self.runtime_manifest_sha256,
|
||||
"data_contract_key": self.data_contract_key,
|
||||
"data_contract_version": self.data_contract_version,
|
||||
"validation_report_sha256": self.validation_report_sha256,
|
||||
"source_registry_id": self.source_registry_id,
|
||||
"source_snapshot_id": self.source_snapshot_id,
|
||||
"source_snapshot_checksum_sha256": self.source_snapshot_checksum_sha256,
|
||||
"source_version": self.source_version,
|
||||
}
|
||||
|
||||
|
||||
class RuntimeModelProvenanceService:
|
||||
"""Validate model sidecars and their production database binding.
|
||||
|
||||
A sidecar lives next to its model as ``<model-file>.geointel-model.json``.
|
||||
Its ``metadata.runtime_manifest_sha256`` is the SHA-256 of canonical JSON
|
||||
after omitting that one self-referential field. Any other mutation of the
|
||||
manifest therefore invalidates it. Structural validation is deliberately
|
||||
separate from :meth:`validate_for_production_runtime`: discovery and
|
||||
preflight have no database session, whereas every production inference
|
||||
path must prove an active source registry/snapshot binding.
|
||||
"""
|
||||
|
||||
MANIFEST_SUFFIX = ".geointel-model.json"
|
||||
MANIFEST_SCHEMA_VERSION = "geointel.runtime-model-manifest/v1"
|
||||
SOURCE_REGISTRY_KEY = "model"
|
||||
|
||||
@classmethod
|
||||
def manifest_path_for_model(cls, model_path: str | Path) -> Path:
|
||||
path = Path(model_path).expanduser()
|
||||
return Path(f"{path}{cls.MANIFEST_SUFFIX}")
|
||||
|
||||
@staticmethod
|
||||
def manifest_self_checksum(payload: Mapping[str, Any]) -> str:
|
||||
"""Hash sidecar semantics without its self-referential checksum field."""
|
||||
|
||||
canonical_payload = json.loads(json.dumps(payload, sort_keys=True, ensure_ascii=True))
|
||||
metadata = canonical_payload.get("metadata")
|
||||
if isinstance(metadata, dict):
|
||||
metadata.pop("runtime_manifest_sha256", None)
|
||||
encoded = json.dumps(
|
||||
canonical_payload,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
).encode("utf-8")
|
||||
return sha256(encoded).hexdigest()
|
||||
|
||||
@classmethod
|
||||
def validate_for_runtime(
|
||||
cls,
|
||||
*,
|
||||
model_path: str | Path,
|
||||
model_id: str,
|
||||
task_type: str,
|
||||
expected_model_version: str | None = None,
|
||||
allowed_frameworks: tuple[str, ...] = (),
|
||||
) -> RuntimeModelProvenance:
|
||||
"""Return structural sidecar evidence without asserting database state.
|
||||
|
||||
This method is appropriate for read-only catalog/preflight checks and
|
||||
focused sidecar unit tests. It is insufficient for production
|
||||
inference; adapters must use :meth:`validate_for_production_runtime`.
|
||||
"""
|
||||
|
||||
path = Path(model_path).expanduser()
|
||||
if not path.exists() or not path.is_file():
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MODEL_FILE_MISSING",
|
||||
"Configured model file is missing; runtime provenance cannot be verified.",
|
||||
model_path=str(path),
|
||||
)
|
||||
manifest_path = cls.manifest_path_for_model(path)
|
||||
if not manifest_path.exists() or not manifest_path.is_file():
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_MISSING",
|
||||
"Configured model requires an immutable .geointel-model.json sidecar before inference.",
|
||||
model_path=str(path),
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
|
||||
try:
|
||||
raw_manifest = manifest_path.read_bytes()
|
||||
payload = json.loads(raw_manifest.decode("utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
"Configured model sidecar must be a readable UTF-8 JSON object.",
|
||||
manifest_path=str(manifest_path),
|
||||
error_type=type(exc).__name__,
|
||||
)
|
||||
if not isinstance(payload, dict):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
"Configured model sidecar must contain a JSON object.",
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
|
||||
cls._require_exact_text(
|
||||
payload.get("schema_version"),
|
||||
cls.MANIFEST_SCHEMA_VERSION,
|
||||
field="schema_version",
|
||||
manifest_path=manifest_path,
|
||||
)
|
||||
contract = cls._require_mapping(payload, "data_contract", manifest_path)
|
||||
contract_key = cls._require_text(contract, "key", manifest_path)
|
||||
contract_version = cls._require_text(contract, "version", manifest_path)
|
||||
# A valid manifest for a different artifact family must never make a
|
||||
# local PyTorch/SAM weight executable. The structural validator below
|
||||
# has a registry lookup too, but pinning the identity here keeps this
|
||||
# runtime gate fail-closed if more model contracts are introduced.
|
||||
cls._require_exact_text(
|
||||
contract_key,
|
||||
PYTORCH_MODEL_CONTRACT_KEY,
|
||||
field="data_contract.key",
|
||||
manifest_path=manifest_path,
|
||||
)
|
||||
cls._require_exact_text(
|
||||
contract_version,
|
||||
PYTORCH_MODEL_CONTRACT_VERSION,
|
||||
field="data_contract.version",
|
||||
manifest_path=manifest_path,
|
||||
)
|
||||
model = cls._require_mapping(payload, "model", manifest_path)
|
||||
declared_model_id = cls._require_text(model, "model_id", manifest_path)
|
||||
declared_task_type = cls._require_text(model, "task_type", manifest_path)
|
||||
cls._require_exact_text(declared_model_id, model_id, field="model.model_id", manifest_path=manifest_path)
|
||||
cls._require_exact_text(declared_task_type, task_type, field="model.task_type", manifest_path=manifest_path)
|
||||
|
||||
declared_checksum = cls._require_checksum(model.get("sha256"), "model.sha256", manifest_path)
|
||||
model_checksum = cls._file_sha256(path)
|
||||
if declared_checksum != model_checksum:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MODEL_CHECKSUM_MISMATCH",
|
||||
"Model bytes do not match the checksum bound by the runtime sidecar.",
|
||||
model_path=str(path),
|
||||
expected=declared_checksum,
|
||||
observed=model_checksum,
|
||||
)
|
||||
|
||||
model_format = cls._require_text(model, "model_format", manifest_path)
|
||||
framework = cls._require_text(model, "framework", manifest_path)
|
||||
class_mapping = model.get("class_mapping")
|
||||
if not isinstance(class_mapping, (dict, list, tuple)) or not class_mapping:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
"model.class_mapping must be a non-empty mapping or sequence.",
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
normalized_framework = framework.strip().lower()
|
||||
if allowed_frameworks and normalized_framework not in {value.strip().lower() for value in allowed_frameworks}:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_FRAMEWORK_MISMATCH",
|
||||
"Model framework does not match the configured runtime adapter.",
|
||||
manifest_path=str(manifest_path),
|
||||
expected=sorted({value.strip().lower() for value in allowed_frameworks}),
|
||||
observed=framework,
|
||||
)
|
||||
source_version = cls._require_text(model, "source_version", manifest_path)
|
||||
if expected_model_version and source_version != expected_model_version:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_VERSION_MISMATCH",
|
||||
"Model sidecar version does not match the configured model version.",
|
||||
manifest_path=str(manifest_path),
|
||||
expected=expected_model_version,
|
||||
observed=source_version,
|
||||
)
|
||||
|
||||
source = cls._require_mapping(payload, "source", manifest_path)
|
||||
source_registry_id = cls._require_uuid(source.get("source_registry_id"), "source.source_registry_id", manifest_path)
|
||||
source_snapshot_id = cls._require_uuid(source.get("source_snapshot_id"), "source.source_snapshot_id", manifest_path)
|
||||
cls._require_exact_text(
|
||||
cls._require_text(source, "source_registry_key", manifest_path),
|
||||
cls.SOURCE_REGISTRY_KEY,
|
||||
field="source.source_registry_key",
|
||||
manifest_path=manifest_path,
|
||||
)
|
||||
snapshot_checksum = cls._require_checksum(
|
||||
source.get("source_snapshot_checksum_sha256"),
|
||||
"source.source_snapshot_checksum_sha256",
|
||||
manifest_path,
|
||||
)
|
||||
if snapshot_checksum != model_checksum:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_SNAPSHOT_CHECKSUM_MISMATCH",
|
||||
"Model source snapshot checksum must bind the exact model bytes.",
|
||||
manifest_path=str(manifest_path),
|
||||
expected=model_checksum,
|
||||
observed=snapshot_checksum,
|
||||
)
|
||||
|
||||
metadata = cls._require_mapping(payload, "metadata", manifest_path)
|
||||
training_manifest_sha256 = cls._require_checksum(
|
||||
metadata.get("training_manifest_sha256"),
|
||||
"metadata.training_manifest_sha256",
|
||||
manifest_path,
|
||||
)
|
||||
declared_runtime_manifest_sha256 = cls._require_checksum(
|
||||
metadata.get("runtime_manifest_sha256"),
|
||||
"metadata.runtime_manifest_sha256",
|
||||
manifest_path,
|
||||
)
|
||||
computed_runtime_manifest_sha256 = cls.manifest_self_checksum(payload)
|
||||
if declared_runtime_manifest_sha256 != computed_runtime_manifest_sha256:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_CHECKSUM_MISMATCH",
|
||||
"Runtime model sidecar integrity checksum does not match its canonical contents.",
|
||||
manifest_path=str(manifest_path),
|
||||
expected=declared_runtime_manifest_sha256,
|
||||
observed=computed_runtime_manifest_sha256,
|
||||
)
|
||||
|
||||
lineage = cls._lineage_evidence(payload, manifest_path)
|
||||
imported_at = cls._parse_imported_at(payload.get("imported_at"), manifest_path)
|
||||
report = validate_registered_asset(
|
||||
build_model_validation_input(
|
||||
asset_id=f"{model_id}:{model_checksum}",
|
||||
model_metadata={
|
||||
"model_format": model_format,
|
||||
"framework": framework,
|
||||
"class_mapping": class_mapping,
|
||||
},
|
||||
checksum_sha256=declared_checksum,
|
||||
computed_checksum_sha256=model_checksum,
|
||||
source_registry_id=source_registry_id,
|
||||
source_snapshot_id=source_snapshot_id,
|
||||
imported_at=imported_at,
|
||||
metadata={
|
||||
"training_manifest_sha256": training_manifest_sha256,
|
||||
"runtime_manifest_sha256": declared_runtime_manifest_sha256,
|
||||
},
|
||||
source_version=source_version,
|
||||
lineage=lineage,
|
||||
data_contract_key=contract_key,
|
||||
data_contract_version=contract_version,
|
||||
)
|
||||
)
|
||||
if report.failed:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_CONTRACT_FAILED",
|
||||
"Configured model sidecar failed the exact versioned model data contract.",
|
||||
manifest_path=str(manifest_path),
|
||||
data_contract=f"{contract_key}@{contract_version}",
|
||||
issue_codes=[issue.code for issue in report.issues],
|
||||
validation_report_sha256=report.report_sha256,
|
||||
)
|
||||
return RuntimeModelProvenance(
|
||||
model_id=model_id,
|
||||
task_type=task_type,
|
||||
model_path=str(path.resolve()),
|
||||
manifest_path=str(manifest_path.resolve()),
|
||||
model_sha256=model_checksum,
|
||||
manifest_sha256=sha256(raw_manifest).hexdigest(),
|
||||
runtime_manifest_sha256=declared_runtime_manifest_sha256,
|
||||
data_contract_key=contract_key,
|
||||
data_contract_version=contract_version,
|
||||
validation_report_sha256=report.report_sha256,
|
||||
source_registry_id=source_registry_id,
|
||||
source_snapshot_id=source_snapshot_id,
|
||||
source_snapshot_checksum_sha256=snapshot_checksum,
|
||||
source_version=source_version,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def validate_for_production_runtime(
|
||||
cls,
|
||||
*,
|
||||
db: Any,
|
||||
model_path: str | Path,
|
||||
model_id: str,
|
||||
task_type: str,
|
||||
expected_model_version: str | None = None,
|
||||
allowed_frameworks: tuple[str, ...] = (),
|
||||
) -> RuntimeModelProvenance:
|
||||
"""Validate byte-bound model evidence against governed database state.
|
||||
|
||||
The sidecar is not itself a source of authority. The model bytes may
|
||||
enter production inference only when the declared source registry and
|
||||
immutable source snapshot both exist, belong together, are safe to
|
||||
consume and bind the same SHA-256 and source version as the sidecar.
|
||||
This check is intentionally invoked immediately before adapter loading.
|
||||
"""
|
||||
|
||||
if db is None or not callable(getattr(db, "get", None)):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_DATABASE_REQUIRED",
|
||||
"Production model inference requires a database session for source snapshot provenance.",
|
||||
)
|
||||
|
||||
evidence = cls.validate_for_runtime(
|
||||
model_path=model_path,
|
||||
model_id=model_id,
|
||||
task_type=task_type,
|
||||
expected_model_version=expected_model_version,
|
||||
allowed_frameworks=allowed_frameworks,
|
||||
)
|
||||
cls._assert_database_binding(db, evidence)
|
||||
return evidence
|
||||
|
||||
@classmethod
|
||||
def _assert_database_binding(cls, db: Any, evidence: RuntimeModelProvenance) -> None:
|
||||
registry_id = UUID(evidence.source_registry_id)
|
||||
snapshot_id = UUID(evidence.source_snapshot_id)
|
||||
source_registry = db.get(SourceRegistry, registry_id)
|
||||
if not isinstance(source_registry, SourceRegistry):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND",
|
||||
"Configured model sidecar refers to a source registry record that does not exist.",
|
||||
source_registry_id=evidence.source_registry_id,
|
||||
model_id=evidence.model_id,
|
||||
)
|
||||
source_snapshot = db.get(SourceSnapshot, snapshot_id)
|
||||
if not isinstance(source_snapshot, SourceSnapshot):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_NOT_FOUND",
|
||||
"Configured model sidecar refers to a source snapshot record that does not exist.",
|
||||
source_snapshot_id=evidence.source_snapshot_id,
|
||||
model_id=evidence.model_id,
|
||||
)
|
||||
|
||||
if str(source_registry.id) != evidence.source_registry_id or source_registry.source_key != cls.SOURCE_REGISTRY_KEY:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_SOURCE_REGISTRY_IDENTITY_MISMATCH",
|
||||
"Database source registry does not match the immutable model sidecar identity.",
|
||||
expected_source_registry_id=evidence.source_registry_id,
|
||||
observed_source_registry_id=str(source_registry.id),
|
||||
expected_source_key=cls.SOURCE_REGISTRY_KEY,
|
||||
observed_source_key=source_registry.source_key,
|
||||
)
|
||||
if str(source_snapshot.id) != evidence.source_snapshot_id or str(source_snapshot.source_registry_id) != evidence.source_registry_id:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_REGISTRY_MISMATCH",
|
||||
"Model source snapshot does not belong to the declared source registry.",
|
||||
source_registry_id=evidence.source_registry_id,
|
||||
source_snapshot_id=evidence.source_snapshot_id,
|
||||
observed_snapshot_registry_id=str(source_snapshot.source_registry_id),
|
||||
)
|
||||
|
||||
registry_ingest_status = cls._normalise_status(source_registry.ingest_status)
|
||||
registry_freshness_status = cls._normalise_status(source_registry.freshness_status)
|
||||
if (
|
||||
registry_ingest_status not in _SAFE_SOURCE_REGISTRY_INGEST_STATUSES
|
||||
or registry_freshness_status not in _CONSUMABLE_FRESHNESS
|
||||
):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_SOURCE_REGISTRY_UNSAFE",
|
||||
"Configured model source registry is not in a safe configured/current state.",
|
||||
source_registry_id=evidence.source_registry_id,
|
||||
ingest_status=registry_ingest_status,
|
||||
freshness_status=registry_freshness_status,
|
||||
)
|
||||
|
||||
snapshot_ingest_status = cls._normalise_status(source_snapshot.ingest_status)
|
||||
snapshot_freshness_status = cls._normalise_status(source_snapshot.freshness_status)
|
||||
if (
|
||||
snapshot_ingest_status != _SAFE_SOURCE_SNAPSHOT_INGEST_STATUS
|
||||
or snapshot_freshness_status not in _CONSUMABLE_FRESHNESS
|
||||
):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_UNSAFE",
|
||||
"Configured model source snapshot is not an ingested current immutable artifact.",
|
||||
source_snapshot_id=evidence.source_snapshot_id,
|
||||
ingest_status=snapshot_ingest_status,
|
||||
freshness_status=snapshot_freshness_status,
|
||||
)
|
||||
|
||||
snapshot_checksum = str(source_snapshot.checksum_sha256 or "").strip().lower()
|
||||
if snapshot_checksum != evidence.source_snapshot_checksum_sha256 or snapshot_checksum != evidence.model_sha256:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_DATABASE_SNAPSHOT_CHECKSUM_MISMATCH",
|
||||
"Database model source snapshot checksum does not bind the exact sidecar and model bytes.",
|
||||
source_snapshot_id=evidence.source_snapshot_id,
|
||||
expected_model_sha256=evidence.model_sha256,
|
||||
expected_sidecar_snapshot_sha256=evidence.source_snapshot_checksum_sha256,
|
||||
observed_snapshot_sha256=snapshot_checksum,
|
||||
)
|
||||
|
||||
snapshot_version = str(source_snapshot.source_version or "").strip()
|
||||
if snapshot_version != evidence.source_version:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_VERSION_MISMATCH",
|
||||
"Database model source snapshot version does not match the immutable sidecar model version.",
|
||||
source_snapshot_id=evidence.source_snapshot_id,
|
||||
expected_source_version=evidence.source_version,
|
||||
observed_source_version=snapshot_version or None,
|
||||
)
|
||||
|
||||
# The persisted snapshot state is the primary quarantine signal. The
|
||||
# relationship check is a defensive second line for an active
|
||||
# quarantine record that predates or bypassed a status transition.
|
||||
active_quarantines = getattr(source_snapshot, "quarantines", ())
|
||||
if any(cls._normalise_status(getattr(item, "status", None)) == "quarantined" for item in active_quarantines):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_QUARANTINED",
|
||||
"Configured model source snapshot has an active quarantine record.",
|
||||
source_snapshot_id=evidence.source_snapshot_id,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _normalise_status(value: Any) -> str:
|
||||
return str(value or "").strip().lower()
|
||||
|
||||
@classmethod
|
||||
def _lineage_evidence(cls, payload: Mapping[str, Any], manifest_path: Path) -> LineageEvidence:
|
||||
lineage = cls._require_mapping(payload, "lineage", manifest_path)
|
||||
raw_asset_ids = lineage.get("upstream_asset_ids")
|
||||
raw_checksums = lineage.get("upstream_checksums_sha256")
|
||||
raw_transformations = lineage.get("transformations")
|
||||
if not isinstance(raw_asset_ids, list) or not raw_asset_ids or not all(
|
||||
isinstance(value, str) and value.strip() for value in raw_asset_ids
|
||||
):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
"lineage.upstream_asset_ids must be a non-empty string list.",
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
if not isinstance(raw_checksums, list) or len(raw_checksums) != len(raw_asset_ids):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
"lineage.upstream_checksums_sha256 must match upstream_asset_ids one-for-one.",
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
upstream_checksums = tuple(
|
||||
cls._require_checksum(value, f"lineage.upstream_checksums_sha256[{index}]", manifest_path)
|
||||
for index, value in enumerate(raw_checksums)
|
||||
)
|
||||
if not isinstance(raw_transformations, list) or not raw_transformations:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
"lineage.transformations must contain at least one immutable transformation record.",
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
transformations: list[TransformationEvidence] = []
|
||||
for index, raw in enumerate(raw_transformations):
|
||||
if not isinstance(raw, Mapping):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
"Each lineage transformation must be an object.",
|
||||
manifest_path=str(manifest_path),
|
||||
index=index,
|
||||
)
|
||||
transformations.append(
|
||||
TransformationEvidence(
|
||||
name=cls._require_text(raw, "name", manifest_path, prefix=f"lineage.transformations[{index}]."),
|
||||
version=cls._require_text(raw, "version", manifest_path, prefix=f"lineage.transformations[{index}]."),
|
||||
checksum_sha256=cls._require_checksum(
|
||||
raw.get("checksum_sha256"),
|
||||
f"lineage.transformations[{index}].checksum_sha256",
|
||||
manifest_path,
|
||||
),
|
||||
)
|
||||
)
|
||||
return LineageEvidence(
|
||||
upstream_asset_ids=tuple(raw_asset_ids),
|
||||
upstream_checksums_sha256=upstream_checksums,
|
||||
transformations=tuple(transformations),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _file_sha256(path: Path) -> str:
|
||||
digest = sha256()
|
||||
try:
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
except OSError as exc:
|
||||
RuntimeModelProvenanceService._raise(
|
||||
"MODEL_PROVENANCE_MODEL_FILE_UNREADABLE",
|
||||
"Configured model file could not be read for checksum validation.",
|
||||
model_path=str(path),
|
||||
error_type=type(exc).__name__,
|
||||
)
|
||||
return digest.hexdigest()
|
||||
|
||||
@classmethod
|
||||
def _parse_imported_at(cls, value: Any, manifest_path: Path) -> datetime:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
"imported_at must be a timezone-aware ISO-8601 timestamp.",
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
"imported_at must be a timezone-aware ISO-8601 timestamp.",
|
||||
manifest_path=str(manifest_path),
|
||||
observed=value,
|
||||
)
|
||||
if parsed.tzinfo is None:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
"imported_at must include a timezone offset.",
|
||||
manifest_path=str(manifest_path),
|
||||
observed=value,
|
||||
)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
@classmethod
|
||||
def _require_mapping(cls, payload: Mapping[str, Any], key: str, manifest_path: Path) -> Mapping[str, Any]:
|
||||
value = payload.get(key)
|
||||
if not isinstance(value, Mapping):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
f"{key} must be a JSON object.",
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def _require_text(
|
||||
cls,
|
||||
payload: Mapping[str, Any],
|
||||
key: str,
|
||||
manifest_path: Path,
|
||||
*,
|
||||
prefix: str = "",
|
||||
) -> str:
|
||||
value = payload.get(key)
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
f"{prefix}{key} must be a non-empty string.",
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
return value.strip()
|
||||
|
||||
@classmethod
|
||||
def _require_checksum(cls, value: Any, field: str, manifest_path: Path) -> str:
|
||||
normalized = value.strip().lower() if isinstance(value, str) else ""
|
||||
if not _SHA256.fullmatch(normalized) or value != normalized:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
f"{field} must be a lowercase SHA-256 digest.",
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
return normalized
|
||||
|
||||
@classmethod
|
||||
def _require_uuid(cls, value: Any, field: str, manifest_path: Path) -> str:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
f"{field} must be a UUID string.",
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
try:
|
||||
return str(UUID(value))
|
||||
except (AttributeError, ValueError):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
f"{field} must be a UUID string.",
|
||||
manifest_path=str(manifest_path),
|
||||
observed=value,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _require_exact_text(
|
||||
cls,
|
||||
observed: Any,
|
||||
expected: str,
|
||||
*,
|
||||
field: str,
|
||||
manifest_path: Path,
|
||||
) -> None:
|
||||
if not isinstance(observed, str) or observed.strip() != expected:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
f"{field} does not match the configured runtime identity.",
|
||||
manifest_path=str(manifest_path),
|
||||
expected=expected,
|
||||
observed=observed,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _raise(code: str, message: str, **details: Any) -> None:
|
||||
raise AppError(code=code, message=message, details=details, status_code=422)
|
||||
Reference in New Issue
Block a user