"""Versioned, fail-closed validation contracts for GeoIntel data assets. This module intentionally has no ORM, route or storage dependency. Import services build :class:`DataAssetValidationInput` from a staged artifact and persist the report/decision in their own transaction. Keeping validation pure makes it safe to run before an artifact is eligible for training, inference or publication. The contracts are deliberately explicit: an unknown contract version, an unknown CRS, a missing checksum, incomplete lineage or an uncertain temporal claim is a validation failure rather than a best-effort import. """ from __future__ import annotations from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from enum import StrEnum from hashlib import sha256 from math import isfinite from numbers import Integral, Real from typing import Any, Iterable, Mapping, Sequence import json import re from pyproj import CRS from shapely.geometry import shape from shapely.geometry.base import BaseGeometry from shapely.strtree import STRtree _SHA256_RE = re.compile(r"^[0-9a-f]{64}$") class ContractKind(StrEnum): """The supported top-level artifact families.""" RASTER = "raster" VECTOR = "vector" LABEL = "label" MODEL = "model" class ValidationStatus(StrEnum): """Persisted validation status agreed for Phase 2 provenance fields.""" PASSED = "passed" FAILED = "failed" class ProvenanceStatus(StrEnum): COMPLETE = "complete" INCOMPLETE = "incomplete" NOT_APPLICABLE = "not_applicable" class LineageStatus(StrEnum): COMPLETE = "complete" INCOMPLETE = "incomplete" NOT_APPLICABLE = "not_applicable" class QuarantineStatus(StrEnum): NOT_QUARANTINED = "not_quarantined" QUARANTINED = "quarantined" class IssueSeverity(StrEnum): ERROR = "error" WARNING = "warning" class RequirementLevel(StrEnum): REQUIRED = "required" OPTIONAL = "optional" NOT_APPLICABLE = "not_applicable" UNKNOWN_WITH_REASON = "unknown_with_reason" @dataclass(frozen=True) class BoundingBox: """A numeric bounding box in the explicitly declared coordinate system.""" min_x: float min_y: float max_x: float max_y: float @classmethod def from_value(cls, value: BoundingBox | Mapping[str, Any] | Sequence[float]) -> BoundingBox: if isinstance(value, BoundingBox): return value if isinstance(value, Mapping): try: return cls( min_x=float(value.get("min_x", value.get("minx"))), min_y=float(value.get("min_y", value.get("miny"))), max_x=float(value.get("max_x", value.get("maxx"))), max_y=float(value.get("max_y", value.get("maxy"))), ) except (TypeError, ValueError) as exc: raise ValueError("Bounding box mapping requires min/max x/y values") from exc if isinstance(value, Sequence) and not isinstance(value, (str, bytes)) and len(value) == 4: try: return cls(*(float(item) for item in value)) except (TypeError, ValueError) as exc: raise ValueError("Bounding box values must be numeric") from exc raise ValueError("Bounding box must be a four-value sequence or mapping") def is_valid(self) -> bool: values = (self.min_x, self.min_y, self.max_x, self.max_y) return all(isfinite(value) for value in values) and self.min_x <= self.max_x and self.min_y <= self.max_y def contains(self, other: BoundingBox, *, tolerance: float = 0.0) -> bool: return ( self.min_x - tolerance <= other.min_x and self.min_y - tolerance <= other.min_y and self.max_x + tolerance >= other.max_x and self.max_y + tolerance >= other.max_y ) def nearly_equals(self, other: BoundingBox, *, tolerance: float) -> bool: return all( abs(left - right) <= tolerance for left, right in zip( (self.min_x, self.min_y, self.max_x, self.max_y), (other.min_x, other.min_y, other.max_x, other.max_y), strict=True, ) ) def to_dict(self) -> dict[str, float]: return { "min_x": self.min_x, "min_y": self.min_y, "max_x": self.max_x, "max_y": self.max_y, } @dataclass(frozen=True) class Resolution: """Explicit raster ground/sample resolution; no implicit unit conversion.""" x: float y: float unit: str @classmethod def from_value(cls, value: Resolution | Mapping[str, Any] | Sequence[Any]) -> Resolution: if isinstance(value, Resolution): return value if isinstance(value, Mapping): try: return cls(float(value["x"]), float(value["y"]), str(value["unit"]).strip()) except (KeyError, TypeError, ValueError) as exc: raise ValueError("Resolution mapping requires x, y and unit") from exc if isinstance(value, Sequence) and not isinstance(value, (str, bytes)) and len(value) == 3: try: return cls(float(value[0]), float(value[1]), str(value[2]).strip()) except (TypeError, ValueError) as exc: raise ValueError("Resolution values must contain numeric x/y and a unit") from exc raise ValueError("Resolution must be a mapping or three-value sequence") def is_valid(self) -> bool: return isfinite(self.x) and isfinite(self.y) and self.x > 0.0 and self.y > 0.0 and bool(self.unit) def to_dict(self) -> dict[str, float | str]: return {"x": self.x, "y": self.y, "unit": self.unit} @dataclass(frozen=True) class AttributeRule: """An expected feature attribute and its portable JSON type contract.""" name: str required: bool = True nullable: bool = False accepted_types: tuple[str, ...] = ("string",) allowed_values: frozenset[Any] = frozenset() @dataclass(frozen=True) class GeometryRules: allowed_geometry_types: frozenset[str] = frozenset() attribute_rules: tuple[AttributeRule, ...] = () unique_attribute_fields: tuple[str, ...] = () require_features: bool = True forbid_shared_area: bool = False topology_max_features: int = 10_000 @dataclass(frozen=True) class RasterRules: required_profile_fields: tuple[str, ...] = ("width", "height", "band_count", "dtype") allowed_band_counts: frozenset[int] = frozenset() allowed_dtypes: frozenset[str] = frozenset() @dataclass(frozen=True) class LabelRules: allowed_class_ids: frozenset[int] = frozenset() normalized_coordinates: bool = True required_fields: tuple[str, ...] = ("class_id", "x_center", "y_center", "width", "height") # Empty YOLO text files are not implicit negatives. A later contract # version can permit them only when the caller declares and evidences a # reviewed pure-background sample. allow_empty_pure_background: bool = False pure_background_required_metadata_fields: tuple[str, ...] = () allowed_pure_background_splits: frozenset[str] = frozenset() @dataclass(frozen=True) class ModelRules: required_fields: tuple[str, ...] = ("model_format", "framework", "class_mapping") allowed_formats: frozenset[str] = frozenset() minimum_class_count: int | None = None @dataclass(frozen=True) class ResolutionRules: required: bool = True allowed_units: frozenset[str] = frozenset({"m"}) min_x: float | None = None max_x: float | None = None min_y: float | None = None max_y: float | None = None @dataclass(frozen=True) class FreshnessRules: observed_at: RequirementLevel = RequirementLevel.OPTIONAL source_version: RequirementLevel = RequirementLevel.OPTIONAL imported_at_required: bool = True max_age: timedelta | None = None allow_future_observation: bool = False @dataclass(frozen=True) class LineageRules: """The evidence required before an artifact can be considered traceable.""" require_source_registry: bool = True require_source_snapshot: bool = True require_upstream_assets: bool = False require_transformation_when_crs_changes: bool = True @dataclass(frozen=True) class TransformationEvidence: name: str version: str checksum_sha256: str @dataclass(frozen=True) class LineageEvidence: upstream_asset_ids: tuple[str, ...] = () upstream_checksums_sha256: tuple[str, ...] = () transformations: tuple[TransformationEvidence, ...] = () @dataclass(frozen=True) class GeometryRecord: """A geometry plus the source attributes needed for vector schema checks.""" geometry: BaseGeometry | Mapping[str, Any] properties: Mapping[str, Any] = field(default_factory=dict) identifier: str | None = None @dataclass(frozen=True) class DataContract: """A schema and evidence contract identified by immutable key/version.""" key: str version: str kind: ContractKind accepted_source_crs: frozenset[str] = frozenset() canonical_storage_crs: str | None = None require_storage_crs: bool = True spatial_domain: BoundingBox | None = None bounds_tolerance: float = 0.0 require_bounds: bool = False require_checksum: bool = True required_metadata_fields: tuple[str, ...] = () metadata_checksum_fields: tuple[str, ...] = () expected_units: Mapping[str, frozenset[str]] = field(default_factory=dict) geometry_rules: GeometryRules | None = None raster_rules: RasterRules | None = None label_rules: LabelRules | None = None model_rules: ModelRules | None = None resolution_rules: ResolutionRules | None = None freshness_rules: FreshnessRules = field(default_factory=FreshnessRules) lineage_rules: LineageRules = field(default_factory=LineageRules) quarantine_on_warning: bool = True def fingerprint(self) -> str: """Return a deterministic hash of the schema semantics, not a filename.""" payload = { "key": self.key, "version": self.version, "kind": self.kind.value, "accepted_source_crs": sorted(self.accepted_source_crs), "canonical_storage_crs": self.canonical_storage_crs, "require_storage_crs": self.require_storage_crs, "spatial_domain": self.spatial_domain.to_dict() if self.spatial_domain else None, "bounds_tolerance": self.bounds_tolerance, "require_bounds": self.require_bounds, "require_checksum": self.require_checksum, "required_metadata_fields": list(self.required_metadata_fields), "metadata_checksum_fields": list(self.metadata_checksum_fields), "expected_units": {key: sorted(value) for key, value in sorted(self.expected_units.items())}, "geometry_rules": _geometry_rules_payload(self.geometry_rules), "raster_rules": _raster_rules_payload(self.raster_rules), "label_rules": _label_rules_payload(self.label_rules), "model_rules": _model_rules_payload(self.model_rules), "resolution_rules": _resolution_rules_payload(self.resolution_rules), "freshness_rules": _freshness_rules_payload(self.freshness_rules), "lineage_rules": _lineage_rules_payload(self.lineage_rules), "quarantine_on_warning": self.quarantine_on_warning, } return _stable_sha256(payload) @dataclass(frozen=True) class DataAssetValidationInput: """Validated metadata from an already staged artifact. ``content`` is optional for large files. In that case the caller must supply a trusted ``computed_checksum_sha256`` calculated while streaming the staged bytes; a filename alone never satisfies checksum validation. Geometry coordinates are expected in ``storage_crs``. """ asset_id: str data_contract_key: str data_contract_version: str kind: ContractKind source_crs: str | None = None storage_crs: str | None = None bounds: BoundingBox | Mapping[str, Any] | Sequence[float] | None = None checksum_sha256: str | None = None computed_checksum_sha256: str | None = None content: bytes | None = None metadata: Mapping[str, Any] = field(default_factory=dict) units: Mapping[str, str] = field(default_factory=dict) resolution: Resolution | Mapping[str, Any] | Sequence[Any] | None = None # Vector checks make a bounds pass and a schema pass. Callers with a # large partitioned source must therefore supply a *re-iterable* # collection, not a one-shot generator. This permits bounded-memory # validation without weakening any geometry or attribute checks. geometry_records: Iterable[GeometryRecord] = () raster_profile: Mapping[str, Any] = field(default_factory=dict) label_records: tuple[Mapping[str, Any], ...] = () label_mode: str = "objects" model_metadata: Mapping[str, Any] = field(default_factory=dict) source_registry_id: str | None = None source_snapshot_id: str | None = None lineage: LineageEvidence = field(default_factory=LineageEvidence) imported_at: datetime | None = None observed_at: datetime | None = None valid_from: datetime | None = None valid_to: datetime | None = None temporal_unknown_reason: str | None = None source_version: str | None = None source_version_unknown_reason: str | None = None @dataclass(frozen=True) class ValidationIssue: code: str category: str message: str severity: IssueSeverity = IssueSeverity.ERROR field: str | None = None expected: Any = None observed: Any = None def to_dict(self) -> dict[str, Any]: return { "code": self.code, "category": self.category, "message": self.message, "severity": self.severity.value, "field": self.field, "expected": _json_safe(self.expected), "observed": _json_safe(self.observed), } @dataclass(frozen=True) class ValidationReport: asset_id: str data_contract_key: str data_contract_version: str contract_fingerprint_sha256: str | None validation_status: ValidationStatus provenance_status: ProvenanceStatus lineage_status: LineageStatus quarantine_status: QuarantineStatus validation_scope: tuple[str, ...] checked_at: datetime issues: tuple[ValidationIssue, ...] = () @property def report_sha256(self) -> str: return _stable_sha256(self.to_dict(include_hash=False)) @property def failed(self) -> bool: return self.validation_status == ValidationStatus.FAILED def to_dict(self, *, include_hash: bool = True) -> dict[str, Any]: payload: dict[str, Any] = { "asset_id": self.asset_id, "data_contract_key": self.data_contract_key, "data_contract_version": self.data_contract_version, "contract_fingerprint_sha256": self.contract_fingerprint_sha256, "validation_status": self.validation_status.value, "provenance_status": self.provenance_status.value, "lineage_status": self.lineage_status.value, "quarantine_status": self.quarantine_status.value, "validation_scope": list(self.validation_scope), "checked_at": _datetime_payload(self.checked_at), "issues": [issue.to_dict() for issue in self.issues], } if include_hash: payload["report_sha256"] = self.report_sha256 return payload def persistence_fields(self) -> dict[str, Any]: """Fields that map directly to the additive Phase 2 Dataset columns.""" return { "data_contract_key": self.data_contract_key, "data_contract_version": self.data_contract_version, "validation_status": self.validation_status.value, "validation_report_json": self.to_dict(), "provenance_status": self.provenance_status.value, "lineage_status": self.lineage_status.value, "quarantine_status": self.quarantine_status.value, } class DataContractRegistry: """An in-memory exact-version registry; no implicit latest-version lookup.""" def __init__(self, contracts: Iterable[DataContract] = ()) -> None: self._contracts: dict[tuple[str, str], DataContract] = {} for contract in contracts: self.register(contract) def register(self, contract: DataContract) -> None: key = _contract_identity(contract.key, contract.version) if key in self._contracts: raise ValueError(f"Data contract {contract.key}@{contract.version} is already registered") self._contracts[key] = contract def resolve(self, key: str, version: str) -> DataContract | None: try: identity = _contract_identity(key, version) except (AttributeError, ValueError): return None return self._contracts.get(identity) def registered_contracts(self) -> tuple[DataContract, ...]: """Return every exact contract identity in deterministic order. Audit/evidence tooling may enumerate contracts, but callers still have to resolve a concrete key/version to validate an asset. There is no implicit "latest" policy. """ return tuple( contract for _identity, contract in sorted(self._contracts.items(), key=lambda item: item[0]) ) def validate(self, asset: DataAssetValidationInput, *, now: datetime | None = None) -> ValidationReport: contract = self.resolve(asset.data_contract_key, asset.data_contract_version) if contract is None: issue = ValidationIssue( code="DATA_CONTRACT_UNKNOWN", category="contract", field="data_contract_key", message="No exact data contract version is registered for this asset.", expected="registered key and version", observed=f"{asset.data_contract_key}@{asset.data_contract_version}", ) return _failed_unknown_contract_report(asset, issue, now=now) return DataContractValidator.validate(contract, asset, now=now) class DataContractValidator: """Pure validation engine used by staged imports and derived-artifact jobs.""" @classmethod def validate( cls, contract: DataContract, asset: DataAssetValidationInput, *, now: datetime | None = None, ) -> ValidationReport: checked_at = _as_utc(now) or datetime.now(timezone.utc) issues: list[ValidationIssue] = [] cls._check_identity(contract, asset, issues) cls._check_checksum(contract, asset, issues) cls._check_metadata(contract, asset, issues) cls._check_temporal(contract, asset, checked_at, issues) cls._check_provenance_and_lineage(contract, asset, issues) cls._check_crs_and_bounds(contract, asset, issues) cls._check_units(contract, asset, issues) cls._check_resolution(contract, asset, issues) if contract.kind == ContractKind.VECTOR: cls._check_vector(contract, asset, issues) elif contract.kind == ContractKind.RASTER: cls._check_raster(contract, asset, issues) elif contract.kind == ContractKind.LABEL: cls._check_labels(contract, asset, issues) elif contract.kind == ContractKind.MODEL: cls._check_model(contract, asset, issues) has_error = any(issue.severity == IssueSeverity.ERROR for issue in issues) has_warning = any(issue.severity == IssueSeverity.WARNING for issue in issues) quarantined = has_error or (has_warning and contract.quarantine_on_warning) validation_status = ValidationStatus.FAILED if quarantined else ValidationStatus.PASSED provenance_status = _provenance_status(contract, issues) lineage_status = _lineage_status(contract, issues) return ValidationReport( asset_id=asset.asset_id, data_contract_key=contract.key, data_contract_version=contract.version, contract_fingerprint_sha256=contract.fingerprint(), validation_status=validation_status, provenance_status=provenance_status, lineage_status=lineage_status, quarantine_status=(QuarantineStatus.QUARANTINED if quarantined else QuarantineStatus.NOT_QUARANTINED), validation_scope=_validation_scope(contract), checked_at=checked_at, issues=tuple(issues), ) @staticmethod def _check_identity( contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue], ) -> None: if not asset.asset_id.strip(): _issue(issues, "ASSET_ID_REQUIRED", "identity", "asset_id", "A non-empty asset id is required.") if asset.data_contract_key != contract.key or asset.data_contract_version != contract.version: _issue( issues, "DATA_CONTRACT_IDENTITY_MISMATCH", "contract", "data_contract_key", "Asset contract identity does not match the supplied validator contract.", expected=f"{contract.key}@{contract.version}", observed=f"{asset.data_contract_key}@{asset.data_contract_version}", ) if asset.kind != contract.kind: _issue( issues, "DATA_KIND_MISMATCH", "contract", "kind", "Asset kind does not match the selected data contract.", expected=contract.kind.value, observed=asset.kind.value, ) @staticmethod def _check_checksum( contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue], ) -> None: declared = _normalise_checksum(asset.checksum_sha256) computed = _normalise_checksum(asset.computed_checksum_sha256) if asset.checksum_sha256 and declared is None: _issue(issues, "CHECKSUM_FORMAT_INVALID", "checksum", "checksum_sha256", "Checksum must be a lowercase SHA-256 hex digest.") if asset.computed_checksum_sha256 and computed is None: _issue( issues, "COMPUTED_CHECKSUM_FORMAT_INVALID", "checksum", "computed_checksum_sha256", "Computed checksum must be a lowercase SHA-256 hex digest.", ) if asset.content is not None: actual = sha256(asset.content).hexdigest() if computed is not None and computed != actual: _issue( issues, "COMPUTED_CHECKSUM_MISMATCH", "checksum", "computed_checksum_sha256", "Provided computed checksum does not match staged bytes.", expected=actual, observed=computed, ) computed = actual if contract.require_checksum and (declared is None or computed is None): _issue( issues, "CHECKSUM_EVIDENCE_REQUIRED", "checksum", "checksum_sha256", "Both declared and computed checksum evidence are required before use.", ) if declared is not None and computed is not None and declared != computed: _issue( issues, "CHECKSUM_MISMATCH", "checksum", "checksum_sha256", "Declared checksum does not match the staged artifact checksum.", expected=computed, observed=declared, ) @staticmethod def _check_metadata( contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue], ) -> None: for field_name in contract.required_metadata_fields: value = asset.metadata.get(field_name) if value is None or (isinstance(value, str) and not value.strip()): _issue( issues, "METADATA_FIELD_REQUIRED", "metadata", f"metadata.{field_name}", "Required metadata field is missing or empty.", expected=field_name, observed=value, ) for field_name in contract.metadata_checksum_fields: value = asset.metadata.get(field_name) if _normalise_checksum(value) is None: _issue( issues, "METADATA_CHECKSUM_INVALID", "checksum", f"metadata.{field_name}", "Metadata checksum must be a lowercase SHA-256 digest.", observed=value, ) @classmethod def _check_temporal( cls, contract: DataContract, asset: DataAssetValidationInput, checked_at: datetime, issues: list[ValidationIssue], ) -> None: rules = contract.freshness_rules observed_at = _as_utc(asset.observed_at) imported_at = _as_utc(asset.imported_at) valid_from = _as_utc(asset.valid_from) valid_to = _as_utc(asset.valid_to) if rules.imported_at_required and imported_at is None: _issue(issues, "IMPORT_TIMESTAMP_REQUIRED", "temporal", "imported_at", "Import timestamp is required.") if asset.imported_at is not None and imported_at is None: _issue(issues, "IMPORT_TIMESTAMP_INVALID", "temporal", "imported_at", "Import timestamp must be timezone-aware.") if asset.observed_at is not None and observed_at is None: _issue(issues, "OBSERVATION_TIMESTAMP_INVALID", "temporal", "observed_at", "Observation timestamp must be timezone-aware.") if rules.observed_at == RequirementLevel.REQUIRED and observed_at is None: _issue(issues, "OBSERVATION_TIMESTAMP_REQUIRED", "temporal", "observed_at", "Observation timestamp is required by this contract.") if rules.observed_at == RequirementLevel.UNKNOWN_WITH_REASON and observed_at is None and not _nonempty(asset.temporal_unknown_reason): _issue( issues, "OBSERVATION_UNKNOWN_REASON_REQUIRED", "temporal", "temporal_unknown_reason", "A documented reason is required when observation time is unknown.", ) if rules.observed_at == RequirementLevel.NOT_APPLICABLE and observed_at is not None: _issue( issues, "OBSERVATION_TIMESTAMP_NOT_APPLICABLE", "temporal", "observed_at", "This contract does not permit an observation timestamp claim.", ) if rules.source_version == RequirementLevel.REQUIRED and not _nonempty(asset.source_version): _issue(issues, "SOURCE_VERSION_REQUIRED", "temporal", "source_version", "Source version is required by this contract.") if rules.source_version == RequirementLevel.UNKNOWN_WITH_REASON and not _nonempty(asset.source_version) and not _nonempty(asset.source_version_unknown_reason): _issue( issues, "SOURCE_VERSION_UNKNOWN_REASON_REQUIRED", "temporal", "source_version_unknown_reason", "A documented reason is required when source version is unknown.", ) if rules.source_version == RequirementLevel.NOT_APPLICABLE and _nonempty(asset.source_version): _issue( issues, "SOURCE_VERSION_NOT_APPLICABLE", "temporal", "source_version", "This contract does not permit a source version claim.", ) if valid_from is not None and valid_to is not None and valid_to < valid_from: _issue( issues, "VALIDITY_RANGE_INVALID", "temporal", "valid_to", "valid_to must be on or after valid_from.", expected=_datetime_payload(valid_from), observed=_datetime_payload(valid_to), ) if observed_at is not None and not rules.allow_future_observation and observed_at > checked_at: _issue( issues, "OBSERVATION_IN_FUTURE", "freshness", "observed_at", "Observation time cannot be in the future for this contract.", expected=f"<= {_datetime_payload(checked_at)}", observed=_datetime_payload(observed_at), ) if rules.max_age is not None and observed_at is not None and checked_at - observed_at > rules.max_age: _issue( issues, "FRESHNESS_EXCEEDED", "freshness", "observed_at", "Observation evidence exceeds this contract's maximum age.", expected=f"at most {rules.max_age.total_seconds()} seconds old", observed=f"{(checked_at - observed_at).total_seconds()} seconds old", ) @staticmethod def _check_provenance_and_lineage( contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue], ) -> None: rules = contract.lineage_rules if rules.require_source_registry and not _nonempty(asset.source_registry_id): _issue( issues, "SOURCE_REGISTRY_REQUIRED", "provenance", "source_registry_id", "A server-attested source registry id is required.", ) if rules.require_source_snapshot and not _nonempty(asset.source_snapshot_id): _issue( issues, "SOURCE_SNAPSHOT_REQUIRED", "provenance", "source_snapshot_id", "An immutable source snapshot id is required.", ) if rules.require_upstream_assets: if not asset.lineage.upstream_asset_ids or not asset.lineage.upstream_checksums_sha256: _issue( issues, "UPSTREAM_LINEAGE_REQUIRED", "lineage", "lineage.upstream_asset_ids", "Derived artifacts require upstream asset ids and checksums.", ) elif len(asset.lineage.upstream_asset_ids) != len(asset.lineage.upstream_checksums_sha256): _issue( issues, "UPSTREAM_LINEAGE_CARDINALITY_INVALID", "lineage", "lineage", "Each upstream asset id must have one corresponding checksum.", expected=len(asset.lineage.upstream_asset_ids), observed=len(asset.lineage.upstream_checksums_sha256), ) for checksum in asset.lineage.upstream_checksums_sha256: if _normalise_checksum(checksum) is None: _issue( issues, "UPSTREAM_CHECKSUM_FORMAT_INVALID", "lineage", "lineage.upstream_checksums_sha256", "Each upstream checksum must be a lowercase SHA-256 digest.", observed=checksum, ) for transformation in asset.lineage.transformations: if not _nonempty(transformation.name) or not _nonempty(transformation.version) or _normalise_checksum(transformation.checksum_sha256) is None: _issue( issues, "TRANSFORMATION_EVIDENCE_INVALID", "lineage", "lineage.transformations", "Every transformation requires name, version and checksum.", observed={ "name": transformation.name, "version": transformation.version, "checksum_sha256": transformation.checksum_sha256, }, ) @classmethod def _check_crs_and_bounds( cls, contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue], ) -> None: source_crs = _normalise_crs(asset.source_crs) storage_crs = _normalise_crs(asset.storage_crs) if asset.source_crs and source_crs is None: _issue(issues, "SOURCE_CRS_INVALID", "crs", "source_crs", "Source CRS is not parseable.", observed=asset.source_crs) if asset.storage_crs and storage_crs is None: _issue(issues, "STORAGE_CRS_INVALID", "crs", "storage_crs", "Storage CRS is not parseable.", observed=asset.storage_crs) if contract.require_storage_crs and storage_crs is None: _issue( issues, "STORAGE_CRS_REQUIRED", "crs", "storage_crs", "An explicit storage CRS is required by this contract.", ) if contract.accepted_source_crs: expected = {_normalise_crs(value) for value in contract.accepted_source_crs} if source_crs is None or source_crs not in expected: _issue( issues, "SOURCE_CRS_NOT_ALLOWED", "crs", "source_crs", "Source CRS is not allowed by this contract.", expected=sorted(value for value in expected if value), observed=source_crs or asset.source_crs, ) if contract.canonical_storage_crs: expected_storage_crs = _normalise_crs(contract.canonical_storage_crs) if storage_crs != expected_storage_crs: _issue( issues, "CANONICAL_STORAGE_CRS_REQUIRED", "crs", "storage_crs", "Stored coordinates must use the contract's canonical CRS.", expected=expected_storage_crs, observed=storage_crs or asset.storage_crs, ) if source_crs is not None and storage_crs is not None and source_crs != storage_crs and contract.lineage_rules.require_transformation_when_crs_changes: if not asset.lineage.transformations: _issue( issues, "CRS_TRANSFORMATION_LINEAGE_REQUIRED", "lineage", "lineage.transformations", "A CRS change requires an explicit transformation record.", expected=f"{source_crs} -> {storage_crs}", ) bounds = _coerce_bounds(asset.bounds, issues) observed_bounds = _geometry_bounds(asset.geometry_records) effective_bounds = observed_bounds or bounds if contract.require_bounds and effective_bounds is None: _issue(issues, "BOUNDS_REQUIRED", "bounds", "bounds", "Spatial bounds are required by this contract.") if bounds is not None and not bounds.is_valid(): _issue(issues, "BOUNDS_INVALID", "bounds", "bounds", "Bounds must be finite and ordered.", observed=bounds.to_dict()) if bounds is not None and observed_bounds is not None and not bounds.nearly_equals(observed_bounds, tolerance=contract.bounds_tolerance): _issue( issues, "BOUNDS_GEOMETRY_MISMATCH", "bounds", "bounds", "Declared bounds do not match the geometry envelope.", expected=observed_bounds.to_dict(), observed=bounds.to_dict(), ) if contract.spatial_domain is not None and effective_bounds is not None and effective_bounds.is_valid(): if not contract.spatial_domain.contains(effective_bounds, tolerance=contract.bounds_tolerance): _issue( issues, "CRS_COORDINATE_DOMAIN_VIOLATION", "crs", "bounds", "Coordinates fall outside the contract's declared storage CRS domain.", expected=contract.spatial_domain.to_dict(), observed=effective_bounds.to_dict(), ) @staticmethod def _check_units(contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue]) -> None: for field_name, allowed_units in contract.expected_units.items(): observed = asset.units.get(field_name) normalised_allowed = {unit.strip().lower() for unit in allowed_units} if not _nonempty(observed): _issue( issues, "UNIT_REQUIRED", "units", f"units.{field_name}", "A declared unit is required for this field.", expected=sorted(normalised_allowed), ) elif str(observed).strip().lower() not in normalised_allowed: _issue( issues, "UNIT_NOT_ALLOWED", "units", f"units.{field_name}", "Unit is not allowed by this contract; implicit conversion is forbidden.", expected=sorted(normalised_allowed), observed=observed, ) @staticmethod def _check_resolution(contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue]) -> None: rules = contract.resolution_rules if rules is None: return resolution = _coerce_resolution(asset.resolution, issues) if resolution is None: if rules.required: _issue(issues, "RESOLUTION_REQUIRED", "resolution", "resolution", "Resolution is required by this contract.") return if not resolution.is_valid(): _issue(issues, "RESOLUTION_INVALID", "resolution", "resolution", "Resolution must be finite, positive and unit-labelled.", observed=resolution.to_dict()) return allowed_units = {unit.strip().lower() for unit in rules.allowed_units} if allowed_units and resolution.unit.strip().lower() not in allowed_units: _issue( issues, "RESOLUTION_UNIT_NOT_ALLOWED", "resolution", "resolution.unit", "Resolution unit is not allowed; no implicit conversion is applied.", expected=sorted(allowed_units), observed=resolution.unit, ) for field_name, value, minimum, maximum in ( ("x", resolution.x, rules.min_x, rules.max_x), ("y", resolution.y, rules.min_y, rules.max_y), ): if minimum is not None and value < minimum or maximum is not None and value > maximum: _issue( issues, "RESOLUTION_OUT_OF_RANGE", "resolution", f"resolution.{field_name}", "Resolution is outside the contract's permitted range.", expected={"min": minimum, "max": maximum}, observed=value, ) @classmethod def _check_vector(cls, contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue]) -> None: rules = contract.geometry_rules if rules is None: _issue(issues, "VECTOR_RULES_REQUIRED", "schema", "geometry_rules", "Vector contracts require geometry rules.") return records = asset.geometry_records has_records = False # Topology checks necessarily need the complete geometry set. Default # GeoJSON contracts do not prohibit overlapping source features, so # keep large regional import validation streaming unless a stricter # source-specific contract explicitly asks for that topology rule. parsed: list[BaseGeometry] | None = [] if rules.forbid_shared_area else None unique_values: dict[str, dict[Any, int]] = { field_name: {} for field_name in rules.unique_attribute_fields } allowed_types = {value.lower() for value in rules.allowed_geometry_types} for index, record in enumerate(records): has_records = True geometry = _coerce_geometry(record.geometry, index, issues) if geometry is None: continue if geometry.is_empty: _issue(issues, "GEOMETRY_EMPTY", "geometry", f"geometry_records[{index}]", "Geometry must not be empty.") continue if not geometry.is_valid: _issue( issues, "GEOMETRY_INVALID", "geometry", f"geometry_records[{index}]", "Geometry is invalid; this validator never silently repairs geometry.", ) continue if allowed_types and geometry.geom_type.lower() not in allowed_types: _issue( issues, "GEOMETRY_TYPE_NOT_ALLOWED", "geometry", f"geometry_records[{index}]", "Geometry type is not allowed by this contract.", expected=sorted(rules.allowed_geometry_types), observed=geometry.geom_type, ) cls._check_attributes(rules.attribute_rules, record.properties, index, issues) cls._check_unique_attribute_values( rules.unique_attribute_fields, record.properties, index, unique_values, issues, ) if parsed is not None: parsed.append(geometry) if rules.require_features and not has_records: _issue(issues, "VECTOR_FEATURES_REQUIRED", "geometry", "geometry_records", "At least one vector feature is required.") return if parsed is not None: cls._check_shared_area(parsed, rules, issues) @staticmethod def _check_attributes( rules: tuple[AttributeRule, ...], properties: Mapping[str, Any], index: int, issues: list[ValidationIssue], ) -> None: for rule in rules: present = rule.name in properties value = properties.get(rule.name) location = f"geometry_records[{index}].properties.{rule.name}" if not present and rule.required: _issue(issues, "ATTRIBUTE_REQUIRED", "attributes", location, "Required attribute is missing.", expected=rule.name) continue if not present: continue if value is None: if not rule.nullable: _issue(issues, "ATTRIBUTE_NULL_NOT_ALLOWED", "attributes", location, "Null is not allowed for this attribute.") continue observed_type = _json_value_type(value) if rule.accepted_types and observed_type not in rule.accepted_types: _issue( issues, "ATTRIBUTE_TYPE_INVALID", "attributes", location, "Attribute type does not match the contract.", expected=list(rule.accepted_types), observed=observed_type, ) if rule.allowed_values and value not in rule.allowed_values: _issue( issues, "ATTRIBUTE_VALUE_NOT_ALLOWED", "attributes", location, "Attribute value is not in the contract allowlist.", expected=_sorted_json_values(rule.allowed_values), observed=value, ) @staticmethod def _check_shared_area( geometries: list[BaseGeometry], rules: GeometryRules, issues: list[ValidationIssue]) -> None: if len(geometries) > rules.topology_max_features: _issue( issues, "TOPOLOGY_CHECK_LIMIT_EXCEEDED", "topology", "geometry_records", "Topology check was not run because the batch exceeds its declared safe limit.", expected=f"<= {rules.topology_max_features} features", observed=len(geometries), ) return tree = STRtree(geometries) for index, geometry in enumerate(geometries): for candidate_index in tree.query(geometry): if not isinstance(candidate_index, Integral): continue if candidate_index <= index: continue candidate = geometries[int(candidate_index)] if geometry.intersection(candidate).area > 0.0: _issue( issues, "TOPOLOGY_SHARED_AREA", "topology", "geometry_records", "Features share non-zero polygon area where this contract forbids overlap.", observed={"left_index": index, "right_index": int(candidate_index)}, ) return @staticmethod def _check_unique_attribute_values( field_names: tuple[str, ...], properties: Mapping[str, Any], index: int, values_by_field: dict[str, dict[Any, int]], issues: list[ValidationIssue], ) -> None: for field_name in field_names: value = properties.get(field_name) if value is None: continue values = values_by_field[field_name] try: previous_index = values.get(value) except TypeError: _issue( issues, "ATTRIBUTE_UNIQUENESS_VALUE_UNHASHABLE", "attributes", f"geometry_records[{index}].properties.{field_name}", "A unique attribute must have a scalar, hashable value.", observed=value, ) continue if previous_index is not None: _issue( issues, "ATTRIBUTE_UNIQUENESS_VIOLATION", "attributes", f"geometry_records[{index}].properties.{field_name}", "A field declared unique has a duplicate value.", observed={"value": value, "first_index": previous_index, "duplicate_index": index}, ) continue values[value] = index @staticmethod def _check_raster(contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue]) -> None: rules = contract.raster_rules if rules is None: _issue(issues, "RASTER_RULES_REQUIRED", "schema", "raster_rules", "Raster contracts require raster profile rules.") return profile = asset.raster_profile for field_name in rules.required_profile_fields: if profile.get(field_name) is None: _issue( issues, "RASTER_PROFILE_FIELD_REQUIRED", "raster", f"raster_profile.{field_name}", "Raster profile field is required.", ) for field_name in ("width", "height", "band_count"): value = profile.get(field_name) if value is not None and (not isinstance(value, Integral) or isinstance(value, bool) or value <= 0): _issue( issues, "RASTER_PROFILE_VALUE_INVALID", "raster", f"raster_profile.{field_name}", "Raster dimensions and band count must be positive integers.", observed=value, ) band_count = profile.get("band_count") if rules.allowed_band_counts and isinstance(band_count, Integral) and band_count not in rules.allowed_band_counts: _issue( issues, "RASTER_BAND_COUNT_NOT_ALLOWED", "raster", "raster_profile.band_count", "Raster band count is not allowed by this contract.", expected=sorted(rules.allowed_band_counts), observed=band_count, ) dtype_values = profile.get("dtype") dtypes = dtype_values if isinstance(dtype_values, (list, tuple, set)) else [dtype_values] if rules.allowed_dtypes and any(dtype not in rules.allowed_dtypes for dtype in dtypes if dtype is not None): _issue( issues, "RASTER_DTYPE_NOT_ALLOWED", "raster", "raster_profile.dtype", "Raster dtype is not allowed by this contract.", expected=sorted(rules.allowed_dtypes), observed=list(dtypes), ) @staticmethod def _check_labels(contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue]) -> None: rules = contract.label_rules if rules is None: _issue(issues, "LABEL_RULES_REQUIRED", "schema", "label_rules", "Label contracts require label rules.") return label_mode = str(asset.label_mode or "").strip().lower() declared_mode = str(asset.metadata.get("label_mode") or "").strip().lower() if declared_mode and declared_mode != label_mode: _issue( issues, "LABEL_MODE_MISMATCH", "labels", "metadata.label_mode", "The persisted label mode must match the validation input.", expected=label_mode, observed=declared_mode, ) if not asset.label_records: if not rules.allow_empty_pure_background: _issue(issues, "LABEL_RECORDS_REQUIRED", "labels", "label_records", "At least one label record is required.") return if label_mode != "pure_background" or declared_mode != "pure_background": _issue( issues, "PURE_BACKGROUND_MODE_REQUIRED", "labels", "metadata.label_mode", "An empty YOLO label is valid only as explicitly declared pure_background evidence.", expected="pure_background", observed=declared_mode or label_mode or None, ) return DataContractValidator._check_pure_background_label(rules, asset, issues) return if label_mode != "objects": _issue( issues, "LABEL_MODE_WITH_OBJECTS_INVALID", "labels", "label_mode", "Non-empty YOLO labels must use the objects label mode.", expected="objects", observed=label_mode or None, ) for index, record in enumerate(asset.label_records): for field_name in rules.required_fields: if field_name not in record: _issue( issues, "LABEL_FIELD_REQUIRED", "labels", f"label_records[{index}].{field_name}", "Label record field is required.", ) class_id = record.get("class_id") if not isinstance(class_id, Integral) or isinstance(class_id, bool): _issue( issues, "LABEL_CLASS_ID_INVALID", "labels", f"label_records[{index}].class_id", "Label class_id must be an integer.", observed=class_id, ) elif rules.allowed_class_ids and class_id not in rules.allowed_class_ids: _issue( issues, "LABEL_CLASS_ID_NOT_ALLOWED", "labels", f"label_records[{index}].class_id", "Label class_id is not present in the contract ontology.", expected=sorted(rules.allowed_class_ids), observed=class_id, ) values: dict[str, float] = {} for field_name in ("x_center", "y_center", "width", "height"): value = record.get(field_name) if not isinstance(value, Real) or isinstance(value, bool) or not isfinite(float(value)): _issue( issues, "LABEL_COORDINATE_INVALID", "labels", f"label_records[{index}].{field_name}", "Label coordinates must be finite numeric values.", observed=value, ) else: values[field_name] = float(value) if len(values) == 4 and rules.normalized_coordinates: x_center, y_center, width, height = (values[field] for field in ("x_center", "y_center", "width", "height")) if width <= 0.0 or height <= 0.0 or width > 1.0 or height > 1.0 or not 0.0 <= x_center <= 1.0 or not 0.0 <= y_center <= 1.0: _issue( issues, "LABEL_NORMALIZED_COORDINATE_INVALID", "labels", f"label_records[{index}]", "Normalized labels must have positive dimensions and coordinates within [0, 1].", observed=values, ) elif x_center - width / 2 < 0.0 or x_center + width / 2 > 1.0 or y_center - height / 2 < 0.0 or y_center + height / 2 > 1.0: _issue( issues, "LABEL_BOX_OUTSIDE_IMAGE", "labels", f"label_records[{index}]", "Label bounding box exceeds normalized image bounds.", observed=values, ) @staticmethod def _check_pure_background_label( rules: LabelRules, asset: DataAssetValidationInput, issues: list[ValidationIssue], ) -> None: """Require explicit provenance and human-review evidence for a zero-object label.""" metadata = asset.metadata for field_name in rules.pure_background_required_metadata_fields: value = metadata.get(field_name) if value is None or (isinstance(value, str) and not value.strip()): _issue( issues, "PURE_BACKGROUND_EVIDENCE_REQUIRED", "labels", f"metadata.{field_name}", "Pure-background labels require explicit source, split and review evidence.", expected=field_name, observed=value, ) split = str(metadata.get("split") or "").strip().lower() if rules.allowed_pure_background_splits and split not in rules.allowed_pure_background_splits: _issue( issues, "PURE_BACKGROUND_SPLIT_INVALID", "labels", "metadata.split", "Pure-background label split is not allowed by this contract.", expected=sorted(rules.allowed_pure_background_splits), observed=split or None, ) if metadata.get("review_decision") != "accepted": _issue( issues, "PURE_BACKGROUND_REVIEW_NOT_ACCEPTED", "labels", "metadata.review_decision", "A zero-object label must have an accepted human review decision.", expected="accepted", observed=metadata.get("review_decision"), ) if _normalise_checksum(metadata.get("review_artifact_sha256")) is None: _issue( issues, "PURE_BACKGROUND_REVIEW_ARTIFACT_CHECKSUM_INVALID", "labels", "metadata.review_artifact_sha256", "A zero-object label must bind the reviewed artifact checksum.", observed=metadata.get("review_artifact_sha256"), ) reviewed_at = metadata.get("reviewed_at") if not isinstance(reviewed_at, str) or not reviewed_at.strip(): _issue( issues, "PURE_BACKGROUND_REVIEW_TIMESTAMP_INVALID", "labels", "metadata.reviewed_at", "A zero-object label must record a timezone-aware review timestamp.", observed=reviewed_at, ) else: try: timestamp = datetime.fromisoformat(reviewed_at.strip().replace("Z", "+00:00")) except ValueError: timestamp = None if timestamp is None or timestamp.tzinfo is None: _issue( issues, "PURE_BACKGROUND_REVIEW_TIMESTAMP_INVALID", "labels", "metadata.reviewed_at", "A zero-object label must record a timezone-aware review timestamp.", observed=reviewed_at, ) @staticmethod def _check_model(contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue]) -> None: rules = contract.model_rules if rules is None: _issue(issues, "MODEL_RULES_REQUIRED", "schema", "model_rules", "Model contracts require model metadata rules.") return metadata = asset.model_metadata for field_name in rules.required_fields: value = metadata.get(field_name) if value is None or (isinstance(value, str) and not value.strip()): _issue( issues, "MODEL_METADATA_FIELD_REQUIRED", "model", f"model_metadata.{field_name}", "Model metadata field is required.", ) model_format = metadata.get("model_format") if rules.allowed_formats and model_format not in rules.allowed_formats: _issue( issues, "MODEL_FORMAT_NOT_ALLOWED", "model", "model_metadata.model_format", "Model format is not allowed by this contract.", expected=sorted(rules.allowed_formats), observed=model_format, ) class_mapping = metadata.get("class_mapping") if rules.minimum_class_count is not None: class_count = len(class_mapping) if isinstance(class_mapping, (Mapping, list, tuple)) else 0 if class_count < rules.minimum_class_count: _issue( issues, "MODEL_CLASS_MAPPING_INCOMPLETE", "model", "model_metadata.class_mapping", "Model class mapping does not meet the minimum ontology size.", expected=rules.minimum_class_count, observed=class_count, ) def _failed_unknown_contract_report( asset: DataAssetValidationInput, issue: ValidationIssue, *, now: datetime | None, ) -> ValidationReport: checked_at = _as_utc(now) or datetime.now(timezone.utc) return ValidationReport( asset_id=asset.asset_id, data_contract_key=asset.data_contract_key, data_contract_version=asset.data_contract_version, contract_fingerprint_sha256=None, validation_status=ValidationStatus.FAILED, provenance_status=ProvenanceStatus.INCOMPLETE, lineage_status=LineageStatus.INCOMPLETE, quarantine_status=QuarantineStatus.QUARANTINED, validation_scope=("contract",), checked_at=checked_at, issues=(issue,), ) def _issue( issues: list[ValidationIssue], code: str, category: str, field: str | None, message: str, *, expected: Any = None, observed: Any = None, severity: IssueSeverity = IssueSeverity.ERROR, ) -> None: issues.append( ValidationIssue( code=code, category=category, field=field, message=message, expected=expected, observed=observed, severity=severity, ) ) def _normalise_crs(value: str | None) -> str | None: if not _nonempty(value): return None try: crs = CRS.from_user_input(value) except Exception: return None authority = crs.to_authority() if authority: return f"{authority[0].upper()}:{authority[1]}" return crs.to_string() def _coerce_bounds(value: Any, issues: list[ValidationIssue]) -> BoundingBox | None: if value is None: return None try: return BoundingBox.from_value(value) except ValueError as exc: _issue(issues, "BOUNDS_FORMAT_INVALID", "bounds", "bounds", str(exc), observed=value) return None def _coerce_resolution(value: Any, issues: list[ValidationIssue]) -> Resolution | None: if value is None: return None try: return Resolution.from_value(value) except ValueError as exc: _issue(issues, "RESOLUTION_FORMAT_INVALID", "resolution", "resolution", str(exc), observed=value) return None def _coerce_geometry(value: BaseGeometry | Mapping[str, Any], index: int, issues: list[ValidationIssue]) -> BaseGeometry | None: if isinstance(value, BaseGeometry): return value try: return shape(value) except Exception: _issue( issues, "GEOMETRY_PARSE_FAILED", "geometry", f"geometry_records[{index}]", "Geometry cannot be parsed as GeoJSON/Shapely geometry.", ) return None def _geometry_bounds(records: Iterable[GeometryRecord]) -> BoundingBox | None: min_x = min_y = max_x = max_y = None for record in records: if isinstance(record.geometry, BaseGeometry): geometry = record.geometry else: try: geometry = shape(record.geometry) except Exception: continue if not geometry.is_empty: record_min_x, record_min_y, record_max_x, record_max_y = ( float(value) for value in geometry.bounds ) min_x = record_min_x if min_x is None else min(min_x, record_min_x) min_y = record_min_y if min_y is None else min(min_y, record_min_y) max_x = record_max_x if max_x is None else max(max_x, record_max_x) max_y = record_max_y if max_y is None else max(max_y, record_max_y) if min_x is None or min_y is None or max_x is None or max_y is None: return None return BoundingBox(min_x, min_y, max_x, max_y) def _normalise_checksum(value: str | None) -> str | None: if not _nonempty(value): return None normalised = str(value).strip().lower() return normalised if _SHA256_RE.fullmatch(normalised) else None def _as_utc(value: datetime | None) -> datetime | None: if value is None or value.tzinfo is None: return None return value.astimezone(timezone.utc) def _nonempty(value: Any) -> bool: return value is not None and (not isinstance(value, str) or bool(value.strip())) def _json_value_type(value: Any) -> str: if value is None: return "null" if isinstance(value, bool): return "boolean" if isinstance(value, Integral): return "integer" if isinstance(value, Real): return "number" if isinstance(value, str): return "string" if isinstance(value, Mapping): return "object" if isinstance(value, (list, tuple)): return "array" return type(value).__name__ def _contract_identity(key: str, version: str) -> tuple[str, str]: normalized_key = key.strip() normalized_version = version.strip() if not normalized_key or not normalized_version: raise ValueError("Data contract key and version must be non-empty") return normalized_key, normalized_version def _stable_sha256(payload: Mapping[str, Any]) -> str: encoded = json.dumps(_json_safe(payload), sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") return sha256(encoded).hexdigest() def _json_safe(value: Any) -> Any: if isinstance(value, StrEnum): return value.value if isinstance(value, datetime): return _datetime_payload(value) if isinstance(value, BoundingBox): return value.to_dict() if isinstance(value, Resolution): return value.to_dict() if isinstance(value, Mapping): return {str(key): _json_safe(item) for key, item in value.items()} if isinstance(value, (list, tuple, set, frozenset)): return [_json_safe(item) for item in value] return value def _datetime_payload(value: datetime) -> str: return value.astimezone(timezone.utc).isoformat() def _geometry_rules_payload(value: GeometryRules | None) -> dict[str, Any] | None: if value is None: return None return { "allowed_geometry_types": sorted(value.allowed_geometry_types), "attribute_rules": [ { "name": rule.name, "required": rule.required, "nullable": rule.nullable, "accepted_types": list(rule.accepted_types), "allowed_values": _sorted_json_values(rule.allowed_values), } for rule in value.attribute_rules ], "unique_attribute_fields": list(value.unique_attribute_fields), "require_features": value.require_features, "forbid_shared_area": value.forbid_shared_area, "topology_max_features": value.topology_max_features, } def _raster_rules_payload(value: RasterRules | None) -> dict[str, Any] | None: if value is None: return None return { "required_profile_fields": list(value.required_profile_fields), "allowed_band_counts": sorted(value.allowed_band_counts), "allowed_dtypes": sorted(value.allowed_dtypes), } def _label_rules_payload(value: LabelRules | None) -> dict[str, Any] | None: if value is None: return None payload: dict[str, Any] = { "allowed_class_ids": sorted(value.allowed_class_ids), "normalized_coordinates": value.normalized_coordinates, "required_fields": list(value.required_fields), } # Keep the historical v1.0.0 fingerprint stable. Pure-background support # is introduced by a new exact contract version rather than silently # widening the meaning of an already frozen label contract. if value.allow_empty_pure_background: payload.update( { "allow_empty_pure_background": True, "pure_background_required_metadata_fields": list(value.pure_background_required_metadata_fields), "allowed_pure_background_splits": sorted(value.allowed_pure_background_splits), } ) return payload def _model_rules_payload(value: ModelRules | None) -> dict[str, Any] | None: if value is None: return None return { "required_fields": list(value.required_fields), "allowed_formats": sorted(value.allowed_formats), "minimum_class_count": value.minimum_class_count, } def _resolution_rules_payload(value: ResolutionRules | None) -> dict[str, Any] | None: if value is None: return None return { "required": value.required, "allowed_units": sorted(value.allowed_units), "min_x": value.min_x, "max_x": value.max_x, "min_y": value.min_y, "max_y": value.max_y, } def _freshness_rules_payload(value: FreshnessRules) -> dict[str, Any]: return { "observed_at": value.observed_at.value, "source_version": value.source_version.value, "imported_at_required": value.imported_at_required, "max_age_seconds": value.max_age.total_seconds() if value.max_age else None, "allow_future_observation": value.allow_future_observation, } def _lineage_rules_payload(value: LineageRules) -> dict[str, Any]: return { "require_source_registry": value.require_source_registry, "require_source_snapshot": value.require_source_snapshot, "require_upstream_assets": value.require_upstream_assets, "require_transformation_when_crs_changes": value.require_transformation_when_crs_changes, } def _sorted_json_values(values: Iterable[Any]) -> list[Any]: serialised = [_json_safe(value) for value in values] return sorted(serialised, key=lambda value: json.dumps(value, sort_keys=True, ensure_ascii=True)) def _provenance_status(contract: DataContract, issues: list[ValidationIssue]) -> ProvenanceStatus: categories = {"provenance", "checksum", "temporal", "freshness", "crs", "bounds", "units", "resolution", "metadata"} if any(issue.category in categories for issue in issues): return ProvenanceStatus.INCOMPLETE if not contract.lineage_rules.require_source_registry and not contract.lineage_rules.require_source_snapshot: return ProvenanceStatus.NOT_APPLICABLE return ProvenanceStatus.COMPLETE def _lineage_status(contract: DataContract, issues: list[ValidationIssue]) -> LineageStatus: if any(issue.category == "lineage" for issue in issues): return LineageStatus.INCOMPLETE if not contract.lineage_rules.require_upstream_assets and not contract.lineage_rules.require_transformation_when_crs_changes: return LineageStatus.NOT_APPLICABLE return LineageStatus.COMPLETE def _validation_scope(contract: DataContract) -> tuple[str, ...]: checks = ["contract", "checksum", "metadata", "temporal", "provenance", "lineage", "crs", "bounds", "units"] if contract.resolution_rules is not None: checks.append("resolution") checks.append(contract.kind.value) return tuple(checks) # The generic contracts below are deliberately narrow in evidence requirements # but broad in legitimate Belgian source CRSs. A source-specific registry may # register an additional, stricter version; ingestion must always choose an # explicit key/version and may never silently choose a "latest" contract. VECTOR_GEOJSON_CONTRACT_KEY = "geointel.vector.geojson" VECTOR_GEOJSON_CONTRACT_VERSION = "1.0.0" RASTER_GEOTIFF_CONTRACT_KEY = "geointel.raster.geotiff" RASTER_GEOTIFF_CONTRACT_VERSION = "1.0.0" YOLO_LABEL_CONTRACT_KEY = "geointel.label.yolo" YOLO_LABEL_LEGACY_CONTRACT_VERSION = "1.0.0" YOLO_LABEL_CONTRACT_VERSION = "1.1.0" PYTORCH_MODEL_CONTRACT_KEY = "geointel.model.pytorch" PYTORCH_MODEL_CONTRACT_VERSION = "1.0.0" _BELGIUM_AND_NORTH_SEA_WGS84_DOMAIN = BoundingBox(min_x=1.5, min_y=48.5, max_x=7.5, max_y=52.5) _BELGIAN_SOURCE_CRS = frozenset({"EPSG:4326", "EPSG:31370", "EPSG:3812"}) def build_default_data_contract_registry() -> DataContractRegistry: """Build the concrete exact-version registry used by generic ingestion. The defaults are not a trust registry. They validate a safely staged artifact only after the caller supplies server-attested source registry and snapshot identities. GRB/PICC/UrbIS and source-specific semantic rules are intentionally supplied by stricter source-registry contracts. """ vector_contract = DataContract( key=VECTOR_GEOJSON_CONTRACT_KEY, version=VECTOR_GEOJSON_CONTRACT_VERSION, kind=ContractKind.VECTOR, accepted_source_crs=_BELGIAN_SOURCE_CRS, canonical_storage_crs="EPSG:4326", spatial_domain=_BELGIUM_AND_NORTH_SEA_WGS84_DOMAIN, require_bounds=True, required_metadata_fields=("license",), geometry_rules=GeometryRules(require_features=True), freshness_rules=FreshnessRules( observed_at=RequirementLevel.UNKNOWN_WITH_REASON, source_version=RequirementLevel.UNKNOWN_WITH_REASON, ), ) raster_contract = DataContract( key=RASTER_GEOTIFF_CONTRACT_KEY, version=RASTER_GEOTIFF_CONTRACT_VERSION, kind=ContractKind.RASTER, accepted_source_crs=_BELGIAN_SOURCE_CRS, require_bounds=True, required_metadata_fields=("license",), raster_rules=RasterRules(), resolution_rules=ResolutionRules( allowed_units=frozenset({"m", "degree"}), min_x=0.000001, max_x=10_000.0, min_y=0.000001, max_y=10_000.0, ), freshness_rules=FreshnessRules( observed_at=RequirementLevel.UNKNOWN_WITH_REASON, source_version=RequirementLevel.UNKNOWN_WITH_REASON, ), lineage_rules=LineageRules(require_transformation_when_crs_changes=False), ) legacy_label_contract = DataContract( key=YOLO_LABEL_CONTRACT_KEY, version=YOLO_LABEL_LEGACY_CONTRACT_VERSION, kind=ContractKind.LABEL, require_storage_crs=False, required_metadata_fields=("image_checksum_sha256", "class_ontology_version", "tile_manifest_sha256"), metadata_checksum_fields=("image_checksum_sha256", "tile_manifest_sha256"), label_rules=LabelRules(allowed_class_ids=frozenset({0})), freshness_rules=FreshnessRules( observed_at=RequirementLevel.UNKNOWN_WITH_REASON, source_version=RequirementLevel.UNKNOWN_WITH_REASON, ), lineage_rules=LineageRules( require_source_registry=True, require_source_snapshot=True, require_upstream_assets=True, require_transformation_when_crs_changes=False, ), ) label_contract = DataContract( key=YOLO_LABEL_CONTRACT_KEY, version=YOLO_LABEL_CONTRACT_VERSION, kind=ContractKind.LABEL, require_storage_crs=False, required_metadata_fields=( "image_checksum_sha256", "class_ontology_version", "source_corpus_manifest_sha256", "label_mode", ), metadata_checksum_fields=("image_checksum_sha256", "source_corpus_manifest_sha256"), label_rules=LabelRules( allowed_class_ids=frozenset({0}), allow_empty_pure_background=True, pure_background_required_metadata_fields=( "sample_slug", "split", "raster_dataset_id", "reference_dataset_id", "review_decision", "reviewer_id", "reviewed_at", "review_artifact_sha256", ), allowed_pure_background_splits=frozenset({"train", "val"}), ), freshness_rules=FreshnessRules( observed_at=RequirementLevel.UNKNOWN_WITH_REASON, source_version=RequirementLevel.UNKNOWN_WITH_REASON, ), lineage_rules=LineageRules( require_source_registry=True, require_source_snapshot=True, require_upstream_assets=True, require_transformation_when_crs_changes=False, ), ) model_contract = DataContract( key=PYTORCH_MODEL_CONTRACT_KEY, version=PYTORCH_MODEL_CONTRACT_VERSION, kind=ContractKind.MODEL, require_storage_crs=False, required_metadata_fields=("training_manifest_sha256", "runtime_manifest_sha256"), metadata_checksum_fields=("training_manifest_sha256", "runtime_manifest_sha256"), model_rules=ModelRules(allowed_formats=frozenset({"pytorch", "ultralytics"}), minimum_class_count=1), freshness_rules=FreshnessRules( observed_at=RequirementLevel.NOT_APPLICABLE, source_version=RequirementLevel.REQUIRED, ), lineage_rules=LineageRules( require_source_registry=True, require_source_snapshot=True, require_upstream_assets=True, require_transformation_when_crs_changes=False, ), ) return DataContractRegistry((vector_contract, raster_contract, legacy_label_contract, label_contract, model_contract)) def validate_registered_asset( asset: DataAssetValidationInput, *, registry: DataContractRegistry | None = None, now: datetime | None = None, ) -> ValidationReport: """Validate an explicitly versioned asset against a supplied/default registry.""" active_registry = registry or build_default_data_contract_registry() return active_registry.validate(asset, now=now) def build_vector_ingest_input( *, asset_id: str, source_crs: str | None, storage_crs: str | None, feature_collection: Mapping[str, Any], checksum_sha256: str | None, computed_checksum_sha256: str | None, source_registry_id: str | None, source_snapshot_id: str | None, imported_at: datetime | None, metadata: Mapping[str, Any] | None = None, content: bytes | None = None, units: Mapping[str, str] | None = None, observed_at: datetime | None = None, valid_from: datetime | None = None, valid_to: datetime | None = None, temporal_unknown_reason: str | None = None, source_version: str | None = None, source_version_unknown_reason: str | None = None, lineage: LineageEvidence | None = None, data_contract_key: str = VECTOR_GEOJSON_CONTRACT_KEY, data_contract_version: str = VECTOR_GEOJSON_CONTRACT_VERSION, ) -> DataAssetValidationInput: """Adapt a GeoJSON FeatureCollection to the generic vector contract input.""" raw_features = feature_collection.get("features") records = () if isinstance(raw_features, list): records = tuple( GeometryRecord( geometry=feature.get("geometry", {}), properties=feature.get("properties") if isinstance(feature.get("properties"), Mapping) else {}, identifier=str(feature.get("id")) if feature.get("id") is not None else None, ) for feature in raw_features if isinstance(feature, Mapping) ) merged_metadata = dict(metadata or {}) bounds = merged_metadata.get("bounds_json", merged_metadata.get("bounds")) return DataAssetValidationInput( asset_id=asset_id, data_contract_key=data_contract_key, data_contract_version=data_contract_version, kind=ContractKind.VECTOR, source_crs=source_crs, storage_crs=storage_crs, bounds=bounds, checksum_sha256=checksum_sha256, computed_checksum_sha256=computed_checksum_sha256, content=content, metadata=merged_metadata, units=units or {}, geometry_records=records, source_registry_id=source_registry_id, source_snapshot_id=source_snapshot_id, lineage=lineage or LineageEvidence(), imported_at=imported_at, observed_at=observed_at, valid_from=valid_from, valid_to=valid_to, temporal_unknown_reason=temporal_unknown_reason, source_version=source_version, source_version_unknown_reason=source_version_unknown_reason, ) def build_raster_ingest_input( *, asset_id: str, source_crs: str | None, storage_crs: str | None, raster_profile: Mapping[str, Any], bounds: BoundingBox | Mapping[str, Any] | Sequence[float] | None, resolution: Resolution | Mapping[str, Any] | Sequence[Any] | None, checksum_sha256: str | None, computed_checksum_sha256: str | None, source_registry_id: str | None, source_snapshot_id: str | None, imported_at: datetime | None, metadata: Mapping[str, Any] | None = None, content: bytes | None = None, units: Mapping[str, str] | None = None, observed_at: datetime | None = None, valid_from: datetime | None = None, valid_to: datetime | None = None, temporal_unknown_reason: str | None = None, source_version: str | None = None, source_version_unknown_reason: str | None = None, lineage: LineageEvidence | None = None, data_contract_key: str = RASTER_GEOTIFF_CONTRACT_KEY, data_contract_version: str = RASTER_GEOTIFF_CONTRACT_VERSION, ) -> DataAssetValidationInput: """Adapt extracted GeoTIFF metadata to the generic raster contract input.""" return DataAssetValidationInput( asset_id=asset_id, data_contract_key=data_contract_key, data_contract_version=data_contract_version, kind=ContractKind.RASTER, source_crs=source_crs, storage_crs=storage_crs, bounds=bounds, checksum_sha256=checksum_sha256, computed_checksum_sha256=computed_checksum_sha256, content=content, metadata=dict(metadata or {}), units=units or {}, resolution=resolution, raster_profile=dict(raster_profile), source_registry_id=source_registry_id, source_snapshot_id=source_snapshot_id, lineage=lineage or LineageEvidence(), imported_at=imported_at, observed_at=observed_at, valid_from=valid_from, valid_to=valid_to, temporal_unknown_reason=temporal_unknown_reason, source_version=source_version, source_version_unknown_reason=source_version_unknown_reason, ) def build_label_validation_input( *, asset_id: str, label_records: Sequence[Mapping[str, Any]], checksum_sha256: str | None, computed_checksum_sha256: str | None, source_registry_id: str | None, source_snapshot_id: str | None, imported_at: datetime | None, metadata: Mapping[str, Any] | None = None, content: bytes | None = None, label_mode: str = "objects", observed_at: datetime | None = None, temporal_unknown_reason: str | None = None, source_version: str | None = None, source_version_unknown_reason: str | None = None, lineage: LineageEvidence | None = None, data_contract_key: str = YOLO_LABEL_CONTRACT_KEY, data_contract_version: str = YOLO_LABEL_CONTRACT_VERSION, ) -> DataAssetValidationInput: """Build a strict YOLO-label validation input with explicit upstream lineage.""" normalized_metadata = dict(metadata or {}) normalized_metadata.setdefault("label_mode", label_mode) return DataAssetValidationInput( asset_id=asset_id, data_contract_key=data_contract_key, data_contract_version=data_contract_version, kind=ContractKind.LABEL, checksum_sha256=checksum_sha256, computed_checksum_sha256=computed_checksum_sha256, content=content, metadata=normalized_metadata, label_records=tuple(label_records), label_mode=label_mode, source_registry_id=source_registry_id, source_snapshot_id=source_snapshot_id, lineage=lineage or LineageEvidence(), imported_at=imported_at, observed_at=observed_at, temporal_unknown_reason=temporal_unknown_reason, source_version=source_version, source_version_unknown_reason=source_version_unknown_reason, ) def build_model_validation_input( *, asset_id: str, model_metadata: Mapping[str, Any], checksum_sha256: str | None, computed_checksum_sha256: str | None, source_registry_id: str | None, source_snapshot_id: str | None, imported_at: datetime | None, metadata: Mapping[str, Any] | None = None, content: bytes | None = None, source_version: str | None = None, lineage: LineageEvidence | None = None, data_contract_key: str = PYTORCH_MODEL_CONTRACT_KEY, data_contract_version: str = PYTORCH_MODEL_CONTRACT_VERSION, ) -> DataAssetValidationInput: """Build a model-asset validation input; model output is never inferred.""" return DataAssetValidationInput( asset_id=asset_id, data_contract_key=data_contract_key, data_contract_version=data_contract_version, kind=ContractKind.MODEL, checksum_sha256=checksum_sha256, computed_checksum_sha256=computed_checksum_sha256, content=content, metadata=dict(metadata or {}), model_metadata=dict(model_metadata), source_registry_id=source_registry_id, source_snapshot_id=source_snapshot_id, lineage=lineage or LineageEvidence(), imported_at=imported_at, source_version=source_version, )