Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
2082 lines
84 KiB
Python
2082 lines
84 KiB
Python
"""Versioned, fail-closed validation contracts for GeoIntel data assets.
|
|
|
|
This module intentionally has no ORM, route or storage dependency. Import
|
|
services build :class:`DataAssetValidationInput` from a staged artifact and
|
|
persist the report/decision in their own transaction. Keeping validation pure
|
|
makes it safe to run before an artifact is eligible for training, inference or
|
|
publication.
|
|
|
|
The contracts are deliberately explicit: an unknown contract version, an
|
|
unknown CRS, a missing checksum, incomplete lineage or an uncertain temporal
|
|
claim is a validation failure rather than a best-effort import.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timedelta, timezone
|
|
from enum import StrEnum
|
|
from hashlib import sha256
|
|
from math import isfinite
|
|
from numbers import Integral, Real
|
|
from typing import Any, Iterable, Mapping, Sequence
|
|
import json
|
|
import re
|
|
|
|
from pyproj import CRS
|
|
from shapely.geometry import shape
|
|
from shapely.geometry.base import BaseGeometry
|
|
from shapely.strtree import STRtree
|
|
|
|
|
|
_SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
|
|
|
|
|
|
class ContractKind(StrEnum):
|
|
"""The supported top-level artifact families."""
|
|
|
|
RASTER = "raster"
|
|
VECTOR = "vector"
|
|
LABEL = "label"
|
|
MODEL = "model"
|
|
|
|
|
|
class ValidationStatus(StrEnum):
|
|
"""Persisted validation status agreed for Phase 2 provenance fields."""
|
|
|
|
PASSED = "passed"
|
|
FAILED = "failed"
|
|
|
|
|
|
class ProvenanceStatus(StrEnum):
|
|
COMPLETE = "complete"
|
|
INCOMPLETE = "incomplete"
|
|
NOT_APPLICABLE = "not_applicable"
|
|
|
|
|
|
class LineageStatus(StrEnum):
|
|
COMPLETE = "complete"
|
|
INCOMPLETE = "incomplete"
|
|
NOT_APPLICABLE = "not_applicable"
|
|
|
|
|
|
class QuarantineStatus(StrEnum):
|
|
NOT_QUARANTINED = "not_quarantined"
|
|
QUARANTINED = "quarantined"
|
|
|
|
|
|
class IssueSeverity(StrEnum):
|
|
ERROR = "error"
|
|
WARNING = "warning"
|
|
|
|
|
|
class RequirementLevel(StrEnum):
|
|
REQUIRED = "required"
|
|
OPTIONAL = "optional"
|
|
NOT_APPLICABLE = "not_applicable"
|
|
UNKNOWN_WITH_REASON = "unknown_with_reason"
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class BoundingBox:
|
|
"""A numeric bounding box in the explicitly declared coordinate system."""
|
|
|
|
min_x: float
|
|
min_y: float
|
|
max_x: float
|
|
max_y: float
|
|
|
|
@classmethod
|
|
def from_value(cls, value: BoundingBox | Mapping[str, Any] | Sequence[float]) -> BoundingBox:
|
|
if isinstance(value, BoundingBox):
|
|
return value
|
|
if isinstance(value, Mapping):
|
|
try:
|
|
return cls(
|
|
min_x=float(value.get("min_x", value.get("minx"))),
|
|
min_y=float(value.get("min_y", value.get("miny"))),
|
|
max_x=float(value.get("max_x", value.get("maxx"))),
|
|
max_y=float(value.get("max_y", value.get("maxy"))),
|
|
)
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError("Bounding box mapping requires min/max x/y values") from exc
|
|
if isinstance(value, Sequence) and not isinstance(value, (str, bytes)) and len(value) == 4:
|
|
try:
|
|
return cls(*(float(item) for item in value))
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError("Bounding box values must be numeric") from exc
|
|
raise ValueError("Bounding box must be a four-value sequence or mapping")
|
|
|
|
def is_valid(self) -> bool:
|
|
values = (self.min_x, self.min_y, self.max_x, self.max_y)
|
|
return all(isfinite(value) for value in values) and self.min_x <= self.max_x and self.min_y <= self.max_y
|
|
|
|
def contains(self, other: BoundingBox, *, tolerance: float = 0.0) -> bool:
|
|
return (
|
|
self.min_x - tolerance <= other.min_x
|
|
and self.min_y - tolerance <= other.min_y
|
|
and self.max_x + tolerance >= other.max_x
|
|
and self.max_y + tolerance >= other.max_y
|
|
)
|
|
|
|
def nearly_equals(self, other: BoundingBox, *, tolerance: float) -> bool:
|
|
return all(
|
|
abs(left - right) <= tolerance
|
|
for left, right in zip(
|
|
(self.min_x, self.min_y, self.max_x, self.max_y),
|
|
(other.min_x, other.min_y, other.max_x, other.max_y),
|
|
strict=True,
|
|
)
|
|
)
|
|
|
|
def to_dict(self) -> dict[str, float]:
|
|
return {
|
|
"min_x": self.min_x,
|
|
"min_y": self.min_y,
|
|
"max_x": self.max_x,
|
|
"max_y": self.max_y,
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Resolution:
|
|
"""Explicit raster ground/sample resolution; no implicit unit conversion."""
|
|
|
|
x: float
|
|
y: float
|
|
unit: str
|
|
|
|
@classmethod
|
|
def from_value(cls, value: Resolution | Mapping[str, Any] | Sequence[Any]) -> Resolution:
|
|
if isinstance(value, Resolution):
|
|
return value
|
|
if isinstance(value, Mapping):
|
|
try:
|
|
return cls(float(value["x"]), float(value["y"]), str(value["unit"]).strip())
|
|
except (KeyError, TypeError, ValueError) as exc:
|
|
raise ValueError("Resolution mapping requires x, y and unit") from exc
|
|
if isinstance(value, Sequence) and not isinstance(value, (str, bytes)) and len(value) == 3:
|
|
try:
|
|
return cls(float(value[0]), float(value[1]), str(value[2]).strip())
|
|
except (TypeError, ValueError) as exc:
|
|
raise ValueError("Resolution values must contain numeric x/y and a unit") from exc
|
|
raise ValueError("Resolution must be a mapping or three-value sequence")
|
|
|
|
def is_valid(self) -> bool:
|
|
return isfinite(self.x) and isfinite(self.y) and self.x > 0.0 and self.y > 0.0 and bool(self.unit)
|
|
|
|
def to_dict(self) -> dict[str, float | str]:
|
|
return {"x": self.x, "y": self.y, "unit": self.unit}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AttributeRule:
|
|
"""An expected feature attribute and its portable JSON type contract."""
|
|
|
|
name: str
|
|
required: bool = True
|
|
nullable: bool = False
|
|
accepted_types: tuple[str, ...] = ("string",)
|
|
allowed_values: frozenset[Any] = frozenset()
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class GeometryRules:
|
|
allowed_geometry_types: frozenset[str] = frozenset()
|
|
attribute_rules: tuple[AttributeRule, ...] = ()
|
|
unique_attribute_fields: tuple[str, ...] = ()
|
|
require_features: bool = True
|
|
forbid_shared_area: bool = False
|
|
topology_max_features: int = 10_000
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RasterRules:
|
|
required_profile_fields: tuple[str, ...] = ("width", "height", "band_count", "dtype")
|
|
allowed_band_counts: frozenset[int] = frozenset()
|
|
allowed_dtypes: frozenset[str] = frozenset()
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class LabelRules:
|
|
allowed_class_ids: frozenset[int] = frozenset()
|
|
normalized_coordinates: bool = True
|
|
required_fields: tuple[str, ...] = ("class_id", "x_center", "y_center", "width", "height")
|
|
# Empty YOLO text files are not implicit negatives. A later contract
|
|
# version can permit them only when the caller declares and evidences a
|
|
# reviewed pure-background sample.
|
|
allow_empty_pure_background: bool = False
|
|
pure_background_required_metadata_fields: tuple[str, ...] = ()
|
|
allowed_pure_background_splits: frozenset[str] = frozenset()
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ModelRules:
|
|
required_fields: tuple[str, ...] = ("model_format", "framework", "class_mapping")
|
|
allowed_formats: frozenset[str] = frozenset()
|
|
minimum_class_count: int | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ResolutionRules:
|
|
required: bool = True
|
|
allowed_units: frozenset[str] = frozenset({"m"})
|
|
min_x: float | None = None
|
|
max_x: float | None = None
|
|
min_y: float | None = None
|
|
max_y: float | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class FreshnessRules:
|
|
observed_at: RequirementLevel = RequirementLevel.OPTIONAL
|
|
source_version: RequirementLevel = RequirementLevel.OPTIONAL
|
|
imported_at_required: bool = True
|
|
max_age: timedelta | None = None
|
|
allow_future_observation: bool = False
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class LineageRules:
|
|
"""The evidence required before an artifact can be considered traceable."""
|
|
|
|
require_source_registry: bool = True
|
|
require_source_snapshot: bool = True
|
|
require_upstream_assets: bool = False
|
|
require_transformation_when_crs_changes: bool = True
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class TransformationEvidence:
|
|
name: str
|
|
version: str
|
|
checksum_sha256: str
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class LineageEvidence:
|
|
upstream_asset_ids: tuple[str, ...] = ()
|
|
upstream_checksums_sha256: tuple[str, ...] = ()
|
|
transformations: tuple[TransformationEvidence, ...] = ()
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class GeometryRecord:
|
|
"""A geometry plus the source attributes needed for vector schema checks."""
|
|
|
|
geometry: BaseGeometry | Mapping[str, Any]
|
|
properties: Mapping[str, Any] = field(default_factory=dict)
|
|
identifier: str | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DataContract:
|
|
"""A schema and evidence contract identified by immutable key/version."""
|
|
|
|
key: str
|
|
version: str
|
|
kind: ContractKind
|
|
accepted_source_crs: frozenset[str] = frozenset()
|
|
canonical_storage_crs: str | None = None
|
|
require_storage_crs: bool = True
|
|
spatial_domain: BoundingBox | None = None
|
|
bounds_tolerance: float = 0.0
|
|
require_bounds: bool = False
|
|
require_checksum: bool = True
|
|
required_metadata_fields: tuple[str, ...] = ()
|
|
metadata_checksum_fields: tuple[str, ...] = ()
|
|
expected_units: Mapping[str, frozenset[str]] = field(default_factory=dict)
|
|
geometry_rules: GeometryRules | None = None
|
|
raster_rules: RasterRules | None = None
|
|
label_rules: LabelRules | None = None
|
|
model_rules: ModelRules | None = None
|
|
resolution_rules: ResolutionRules | None = None
|
|
freshness_rules: FreshnessRules = field(default_factory=FreshnessRules)
|
|
lineage_rules: LineageRules = field(default_factory=LineageRules)
|
|
quarantine_on_warning: bool = True
|
|
|
|
def fingerprint(self) -> str:
|
|
"""Return a deterministic hash of the schema semantics, not a filename."""
|
|
|
|
payload = {
|
|
"key": self.key,
|
|
"version": self.version,
|
|
"kind": self.kind.value,
|
|
"accepted_source_crs": sorted(self.accepted_source_crs),
|
|
"canonical_storage_crs": self.canonical_storage_crs,
|
|
"require_storage_crs": self.require_storage_crs,
|
|
"spatial_domain": self.spatial_domain.to_dict() if self.spatial_domain else None,
|
|
"bounds_tolerance": self.bounds_tolerance,
|
|
"require_bounds": self.require_bounds,
|
|
"require_checksum": self.require_checksum,
|
|
"required_metadata_fields": list(self.required_metadata_fields),
|
|
"metadata_checksum_fields": list(self.metadata_checksum_fields),
|
|
"expected_units": {key: sorted(value) for key, value in sorted(self.expected_units.items())},
|
|
"geometry_rules": _geometry_rules_payload(self.geometry_rules),
|
|
"raster_rules": _raster_rules_payload(self.raster_rules),
|
|
"label_rules": _label_rules_payload(self.label_rules),
|
|
"model_rules": _model_rules_payload(self.model_rules),
|
|
"resolution_rules": _resolution_rules_payload(self.resolution_rules),
|
|
"freshness_rules": _freshness_rules_payload(self.freshness_rules),
|
|
"lineage_rules": _lineage_rules_payload(self.lineage_rules),
|
|
"quarantine_on_warning": self.quarantine_on_warning,
|
|
}
|
|
return _stable_sha256(payload)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class DataAssetValidationInput:
|
|
"""Validated metadata from an already staged artifact.
|
|
|
|
``content`` is optional for large files. In that case the caller must
|
|
supply a trusted ``computed_checksum_sha256`` calculated while streaming
|
|
the staged bytes; a filename alone never satisfies checksum validation.
|
|
Geometry coordinates are expected in ``storage_crs``.
|
|
"""
|
|
|
|
asset_id: str
|
|
data_contract_key: str
|
|
data_contract_version: str
|
|
kind: ContractKind
|
|
source_crs: str | None = None
|
|
storage_crs: str | None = None
|
|
bounds: BoundingBox | Mapping[str, Any] | Sequence[float] | None = None
|
|
checksum_sha256: str | None = None
|
|
computed_checksum_sha256: str | None = None
|
|
content: bytes | None = None
|
|
metadata: Mapping[str, Any] = field(default_factory=dict)
|
|
units: Mapping[str, str] = field(default_factory=dict)
|
|
resolution: Resolution | Mapping[str, Any] | Sequence[Any] | None = None
|
|
# Vector checks make a bounds pass and a schema pass. Callers with a
|
|
# large partitioned source must therefore supply a *re-iterable*
|
|
# collection, not a one-shot generator. This permits bounded-memory
|
|
# validation without weakening any geometry or attribute checks.
|
|
geometry_records: Iterable[GeometryRecord] = ()
|
|
raster_profile: Mapping[str, Any] = field(default_factory=dict)
|
|
label_records: tuple[Mapping[str, Any], ...] = ()
|
|
label_mode: str = "objects"
|
|
model_metadata: Mapping[str, Any] = field(default_factory=dict)
|
|
source_registry_id: str | None = None
|
|
source_snapshot_id: str | None = None
|
|
lineage: LineageEvidence = field(default_factory=LineageEvidence)
|
|
imported_at: datetime | None = None
|
|
observed_at: datetime | None = None
|
|
valid_from: datetime | None = None
|
|
valid_to: datetime | None = None
|
|
temporal_unknown_reason: str | None = None
|
|
source_version: str | None = None
|
|
source_version_unknown_reason: str | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ValidationIssue:
|
|
code: str
|
|
category: str
|
|
message: str
|
|
severity: IssueSeverity = IssueSeverity.ERROR
|
|
field: str | None = None
|
|
expected: Any = None
|
|
observed: Any = None
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"code": self.code,
|
|
"category": self.category,
|
|
"message": self.message,
|
|
"severity": self.severity.value,
|
|
"field": self.field,
|
|
"expected": _json_safe(self.expected),
|
|
"observed": _json_safe(self.observed),
|
|
}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ValidationReport:
|
|
asset_id: str
|
|
data_contract_key: str
|
|
data_contract_version: str
|
|
contract_fingerprint_sha256: str | None
|
|
validation_status: ValidationStatus
|
|
provenance_status: ProvenanceStatus
|
|
lineage_status: LineageStatus
|
|
quarantine_status: QuarantineStatus
|
|
validation_scope: tuple[str, ...]
|
|
checked_at: datetime
|
|
issues: tuple[ValidationIssue, ...] = ()
|
|
|
|
@property
|
|
def report_sha256(self) -> str:
|
|
return _stable_sha256(self.to_dict(include_hash=False))
|
|
|
|
@property
|
|
def failed(self) -> bool:
|
|
return self.validation_status == ValidationStatus.FAILED
|
|
|
|
def to_dict(self, *, include_hash: bool = True) -> dict[str, Any]:
|
|
payload: dict[str, Any] = {
|
|
"asset_id": self.asset_id,
|
|
"data_contract_key": self.data_contract_key,
|
|
"data_contract_version": self.data_contract_version,
|
|
"contract_fingerprint_sha256": self.contract_fingerprint_sha256,
|
|
"validation_status": self.validation_status.value,
|
|
"provenance_status": self.provenance_status.value,
|
|
"lineage_status": self.lineage_status.value,
|
|
"quarantine_status": self.quarantine_status.value,
|
|
"validation_scope": list(self.validation_scope),
|
|
"checked_at": _datetime_payload(self.checked_at),
|
|
"issues": [issue.to_dict() for issue in self.issues],
|
|
}
|
|
if include_hash:
|
|
payload["report_sha256"] = self.report_sha256
|
|
return payload
|
|
|
|
def persistence_fields(self) -> dict[str, Any]:
|
|
"""Fields that map directly to the additive Phase 2 Dataset columns."""
|
|
|
|
return {
|
|
"data_contract_key": self.data_contract_key,
|
|
"data_contract_version": self.data_contract_version,
|
|
"validation_status": self.validation_status.value,
|
|
"validation_report_json": self.to_dict(),
|
|
"provenance_status": self.provenance_status.value,
|
|
"lineage_status": self.lineage_status.value,
|
|
"quarantine_status": self.quarantine_status.value,
|
|
}
|
|
|
|
|
|
class DataContractRegistry:
|
|
"""An in-memory exact-version registry; no implicit latest-version lookup."""
|
|
|
|
def __init__(self, contracts: Iterable[DataContract] = ()) -> None:
|
|
self._contracts: dict[tuple[str, str], DataContract] = {}
|
|
for contract in contracts:
|
|
self.register(contract)
|
|
|
|
def register(self, contract: DataContract) -> None:
|
|
key = _contract_identity(contract.key, contract.version)
|
|
if key in self._contracts:
|
|
raise ValueError(f"Data contract {contract.key}@{contract.version} is already registered")
|
|
self._contracts[key] = contract
|
|
|
|
def resolve(self, key: str, version: str) -> DataContract | None:
|
|
try:
|
|
identity = _contract_identity(key, version)
|
|
except (AttributeError, ValueError):
|
|
return None
|
|
return self._contracts.get(identity)
|
|
|
|
def registered_contracts(self) -> tuple[DataContract, ...]:
|
|
"""Return every exact contract identity in deterministic order.
|
|
|
|
Audit/evidence tooling may enumerate contracts, but callers still have
|
|
to resolve a concrete key/version to validate an asset. There is no
|
|
implicit "latest" policy.
|
|
"""
|
|
|
|
return tuple(
|
|
contract
|
|
for _identity, contract in sorted(self._contracts.items(), key=lambda item: item[0])
|
|
)
|
|
|
|
def validate(self, asset: DataAssetValidationInput, *, now: datetime | None = None) -> ValidationReport:
|
|
contract = self.resolve(asset.data_contract_key, asset.data_contract_version)
|
|
if contract is None:
|
|
issue = ValidationIssue(
|
|
code="DATA_CONTRACT_UNKNOWN",
|
|
category="contract",
|
|
field="data_contract_key",
|
|
message="No exact data contract version is registered for this asset.",
|
|
expected="registered key and version",
|
|
observed=f"{asset.data_contract_key}@{asset.data_contract_version}",
|
|
)
|
|
return _failed_unknown_contract_report(asset, issue, now=now)
|
|
return DataContractValidator.validate(contract, asset, now=now)
|
|
|
|
|
|
class DataContractValidator:
|
|
"""Pure validation engine used by staged imports and derived-artifact jobs."""
|
|
|
|
@classmethod
|
|
def validate(
|
|
cls,
|
|
contract: DataContract,
|
|
asset: DataAssetValidationInput,
|
|
*,
|
|
now: datetime | None = None,
|
|
) -> ValidationReport:
|
|
checked_at = _as_utc(now) or datetime.now(timezone.utc)
|
|
issues: list[ValidationIssue] = []
|
|
cls._check_identity(contract, asset, issues)
|
|
cls._check_checksum(contract, asset, issues)
|
|
cls._check_metadata(contract, asset, issues)
|
|
cls._check_temporal(contract, asset, checked_at, issues)
|
|
cls._check_provenance_and_lineage(contract, asset, issues)
|
|
cls._check_crs_and_bounds(contract, asset, issues)
|
|
cls._check_units(contract, asset, issues)
|
|
cls._check_resolution(contract, asset, issues)
|
|
|
|
if contract.kind == ContractKind.VECTOR:
|
|
cls._check_vector(contract, asset, issues)
|
|
elif contract.kind == ContractKind.RASTER:
|
|
cls._check_raster(contract, asset, issues)
|
|
elif contract.kind == ContractKind.LABEL:
|
|
cls._check_labels(contract, asset, issues)
|
|
elif contract.kind == ContractKind.MODEL:
|
|
cls._check_model(contract, asset, issues)
|
|
|
|
has_error = any(issue.severity == IssueSeverity.ERROR for issue in issues)
|
|
has_warning = any(issue.severity == IssueSeverity.WARNING for issue in issues)
|
|
quarantined = has_error or (has_warning and contract.quarantine_on_warning)
|
|
validation_status = ValidationStatus.FAILED if quarantined else ValidationStatus.PASSED
|
|
provenance_status = _provenance_status(contract, issues)
|
|
lineage_status = _lineage_status(contract, issues)
|
|
return ValidationReport(
|
|
asset_id=asset.asset_id,
|
|
data_contract_key=contract.key,
|
|
data_contract_version=contract.version,
|
|
contract_fingerprint_sha256=contract.fingerprint(),
|
|
validation_status=validation_status,
|
|
provenance_status=provenance_status,
|
|
lineage_status=lineage_status,
|
|
quarantine_status=(QuarantineStatus.QUARANTINED if quarantined else QuarantineStatus.NOT_QUARANTINED),
|
|
validation_scope=_validation_scope(contract),
|
|
checked_at=checked_at,
|
|
issues=tuple(issues),
|
|
)
|
|
|
|
@staticmethod
|
|
def _check_identity(
|
|
contract: DataContract,
|
|
asset: DataAssetValidationInput,
|
|
issues: list[ValidationIssue],
|
|
) -> None:
|
|
if not asset.asset_id.strip():
|
|
_issue(issues, "ASSET_ID_REQUIRED", "identity", "asset_id", "A non-empty asset id is required.")
|
|
if asset.data_contract_key != contract.key or asset.data_contract_version != contract.version:
|
|
_issue(
|
|
issues,
|
|
"DATA_CONTRACT_IDENTITY_MISMATCH",
|
|
"contract",
|
|
"data_contract_key",
|
|
"Asset contract identity does not match the supplied validator contract.",
|
|
expected=f"{contract.key}@{contract.version}",
|
|
observed=f"{asset.data_contract_key}@{asset.data_contract_version}",
|
|
)
|
|
if asset.kind != contract.kind:
|
|
_issue(
|
|
issues,
|
|
"DATA_KIND_MISMATCH",
|
|
"contract",
|
|
"kind",
|
|
"Asset kind does not match the selected data contract.",
|
|
expected=contract.kind.value,
|
|
observed=asset.kind.value,
|
|
)
|
|
|
|
@staticmethod
|
|
def _check_checksum(
|
|
contract: DataContract,
|
|
asset: DataAssetValidationInput,
|
|
issues: list[ValidationIssue],
|
|
) -> None:
|
|
declared = _normalise_checksum(asset.checksum_sha256)
|
|
computed = _normalise_checksum(asset.computed_checksum_sha256)
|
|
if asset.checksum_sha256 and declared is None:
|
|
_issue(issues, "CHECKSUM_FORMAT_INVALID", "checksum", "checksum_sha256", "Checksum must be a lowercase SHA-256 hex digest.")
|
|
if asset.computed_checksum_sha256 and computed is None:
|
|
_issue(
|
|
issues,
|
|
"COMPUTED_CHECKSUM_FORMAT_INVALID",
|
|
"checksum",
|
|
"computed_checksum_sha256",
|
|
"Computed checksum must be a lowercase SHA-256 hex digest.",
|
|
)
|
|
if asset.content is not None:
|
|
actual = sha256(asset.content).hexdigest()
|
|
if computed is not None and computed != actual:
|
|
_issue(
|
|
issues,
|
|
"COMPUTED_CHECKSUM_MISMATCH",
|
|
"checksum",
|
|
"computed_checksum_sha256",
|
|
"Provided computed checksum does not match staged bytes.",
|
|
expected=actual,
|
|
observed=computed,
|
|
)
|
|
computed = actual
|
|
if contract.require_checksum and (declared is None or computed is None):
|
|
_issue(
|
|
issues,
|
|
"CHECKSUM_EVIDENCE_REQUIRED",
|
|
"checksum",
|
|
"checksum_sha256",
|
|
"Both declared and computed checksum evidence are required before use.",
|
|
)
|
|
if declared is not None and computed is not None and declared != computed:
|
|
_issue(
|
|
issues,
|
|
"CHECKSUM_MISMATCH",
|
|
"checksum",
|
|
"checksum_sha256",
|
|
"Declared checksum does not match the staged artifact checksum.",
|
|
expected=computed,
|
|
observed=declared,
|
|
)
|
|
|
|
@staticmethod
|
|
def _check_metadata(
|
|
contract: DataContract,
|
|
asset: DataAssetValidationInput,
|
|
issues: list[ValidationIssue],
|
|
) -> None:
|
|
for field_name in contract.required_metadata_fields:
|
|
value = asset.metadata.get(field_name)
|
|
if value is None or (isinstance(value, str) and not value.strip()):
|
|
_issue(
|
|
issues,
|
|
"METADATA_FIELD_REQUIRED",
|
|
"metadata",
|
|
f"metadata.{field_name}",
|
|
"Required metadata field is missing or empty.",
|
|
expected=field_name,
|
|
observed=value,
|
|
)
|
|
for field_name in contract.metadata_checksum_fields:
|
|
value = asset.metadata.get(field_name)
|
|
if _normalise_checksum(value) is None:
|
|
_issue(
|
|
issues,
|
|
"METADATA_CHECKSUM_INVALID",
|
|
"checksum",
|
|
f"metadata.{field_name}",
|
|
"Metadata checksum must be a lowercase SHA-256 digest.",
|
|
observed=value,
|
|
)
|
|
|
|
@classmethod
|
|
def _check_temporal(
|
|
cls,
|
|
contract: DataContract,
|
|
asset: DataAssetValidationInput,
|
|
checked_at: datetime,
|
|
issues: list[ValidationIssue],
|
|
) -> None:
|
|
rules = contract.freshness_rules
|
|
observed_at = _as_utc(asset.observed_at)
|
|
imported_at = _as_utc(asset.imported_at)
|
|
valid_from = _as_utc(asset.valid_from)
|
|
valid_to = _as_utc(asset.valid_to)
|
|
|
|
if rules.imported_at_required and imported_at is None:
|
|
_issue(issues, "IMPORT_TIMESTAMP_REQUIRED", "temporal", "imported_at", "Import timestamp is required.")
|
|
if asset.imported_at is not None and imported_at is None:
|
|
_issue(issues, "IMPORT_TIMESTAMP_INVALID", "temporal", "imported_at", "Import timestamp must be timezone-aware.")
|
|
if asset.observed_at is not None and observed_at is None:
|
|
_issue(issues, "OBSERVATION_TIMESTAMP_INVALID", "temporal", "observed_at", "Observation timestamp must be timezone-aware.")
|
|
if rules.observed_at == RequirementLevel.REQUIRED and observed_at is None:
|
|
_issue(issues, "OBSERVATION_TIMESTAMP_REQUIRED", "temporal", "observed_at", "Observation timestamp is required by this contract.")
|
|
if rules.observed_at == RequirementLevel.UNKNOWN_WITH_REASON and observed_at is None and not _nonempty(asset.temporal_unknown_reason):
|
|
_issue(
|
|
issues,
|
|
"OBSERVATION_UNKNOWN_REASON_REQUIRED",
|
|
"temporal",
|
|
"temporal_unknown_reason",
|
|
"A documented reason is required when observation time is unknown.",
|
|
)
|
|
if rules.observed_at == RequirementLevel.NOT_APPLICABLE and observed_at is not None:
|
|
_issue(
|
|
issues,
|
|
"OBSERVATION_TIMESTAMP_NOT_APPLICABLE",
|
|
"temporal",
|
|
"observed_at",
|
|
"This contract does not permit an observation timestamp claim.",
|
|
)
|
|
if rules.source_version == RequirementLevel.REQUIRED and not _nonempty(asset.source_version):
|
|
_issue(issues, "SOURCE_VERSION_REQUIRED", "temporal", "source_version", "Source version is required by this contract.")
|
|
if rules.source_version == RequirementLevel.UNKNOWN_WITH_REASON and not _nonempty(asset.source_version) and not _nonempty(asset.source_version_unknown_reason):
|
|
_issue(
|
|
issues,
|
|
"SOURCE_VERSION_UNKNOWN_REASON_REQUIRED",
|
|
"temporal",
|
|
"source_version_unknown_reason",
|
|
"A documented reason is required when source version is unknown.",
|
|
)
|
|
if rules.source_version == RequirementLevel.NOT_APPLICABLE and _nonempty(asset.source_version):
|
|
_issue(
|
|
issues,
|
|
"SOURCE_VERSION_NOT_APPLICABLE",
|
|
"temporal",
|
|
"source_version",
|
|
"This contract does not permit a source version claim.",
|
|
)
|
|
if valid_from is not None and valid_to is not None and valid_to < valid_from:
|
|
_issue(
|
|
issues,
|
|
"VALIDITY_RANGE_INVALID",
|
|
"temporal",
|
|
"valid_to",
|
|
"valid_to must be on or after valid_from.",
|
|
expected=_datetime_payload(valid_from),
|
|
observed=_datetime_payload(valid_to),
|
|
)
|
|
if observed_at is not None and not rules.allow_future_observation and observed_at > checked_at:
|
|
_issue(
|
|
issues,
|
|
"OBSERVATION_IN_FUTURE",
|
|
"freshness",
|
|
"observed_at",
|
|
"Observation time cannot be in the future for this contract.",
|
|
expected=f"<= {_datetime_payload(checked_at)}",
|
|
observed=_datetime_payload(observed_at),
|
|
)
|
|
if rules.max_age is not None and observed_at is not None and checked_at - observed_at > rules.max_age:
|
|
_issue(
|
|
issues,
|
|
"FRESHNESS_EXCEEDED",
|
|
"freshness",
|
|
"observed_at",
|
|
"Observation evidence exceeds this contract's maximum age.",
|
|
expected=f"at most {rules.max_age.total_seconds()} seconds old",
|
|
observed=f"{(checked_at - observed_at).total_seconds()} seconds old",
|
|
)
|
|
|
|
@staticmethod
|
|
def _check_provenance_and_lineage(
|
|
contract: DataContract,
|
|
asset: DataAssetValidationInput,
|
|
issues: list[ValidationIssue],
|
|
) -> None:
|
|
rules = contract.lineage_rules
|
|
if rules.require_source_registry and not _nonempty(asset.source_registry_id):
|
|
_issue(
|
|
issues,
|
|
"SOURCE_REGISTRY_REQUIRED",
|
|
"provenance",
|
|
"source_registry_id",
|
|
"A server-attested source registry id is required.",
|
|
)
|
|
if rules.require_source_snapshot and not _nonempty(asset.source_snapshot_id):
|
|
_issue(
|
|
issues,
|
|
"SOURCE_SNAPSHOT_REQUIRED",
|
|
"provenance",
|
|
"source_snapshot_id",
|
|
"An immutable source snapshot id is required.",
|
|
)
|
|
if rules.require_upstream_assets:
|
|
if not asset.lineage.upstream_asset_ids or not asset.lineage.upstream_checksums_sha256:
|
|
_issue(
|
|
issues,
|
|
"UPSTREAM_LINEAGE_REQUIRED",
|
|
"lineage",
|
|
"lineage.upstream_asset_ids",
|
|
"Derived artifacts require upstream asset ids and checksums.",
|
|
)
|
|
elif len(asset.lineage.upstream_asset_ids) != len(asset.lineage.upstream_checksums_sha256):
|
|
_issue(
|
|
issues,
|
|
"UPSTREAM_LINEAGE_CARDINALITY_INVALID",
|
|
"lineage",
|
|
"lineage",
|
|
"Each upstream asset id must have one corresponding checksum.",
|
|
expected=len(asset.lineage.upstream_asset_ids),
|
|
observed=len(asset.lineage.upstream_checksums_sha256),
|
|
)
|
|
for checksum in asset.lineage.upstream_checksums_sha256:
|
|
if _normalise_checksum(checksum) is None:
|
|
_issue(
|
|
issues,
|
|
"UPSTREAM_CHECKSUM_FORMAT_INVALID",
|
|
"lineage",
|
|
"lineage.upstream_checksums_sha256",
|
|
"Each upstream checksum must be a lowercase SHA-256 digest.",
|
|
observed=checksum,
|
|
)
|
|
for transformation in asset.lineage.transformations:
|
|
if not _nonempty(transformation.name) or not _nonempty(transformation.version) or _normalise_checksum(transformation.checksum_sha256) is None:
|
|
_issue(
|
|
issues,
|
|
"TRANSFORMATION_EVIDENCE_INVALID",
|
|
"lineage",
|
|
"lineage.transformations",
|
|
"Every transformation requires name, version and checksum.",
|
|
observed={
|
|
"name": transformation.name,
|
|
"version": transformation.version,
|
|
"checksum_sha256": transformation.checksum_sha256,
|
|
},
|
|
)
|
|
|
|
@classmethod
|
|
def _check_crs_and_bounds(
|
|
cls,
|
|
contract: DataContract,
|
|
asset: DataAssetValidationInput,
|
|
issues: list[ValidationIssue],
|
|
) -> None:
|
|
source_crs = _normalise_crs(asset.source_crs)
|
|
storage_crs = _normalise_crs(asset.storage_crs)
|
|
if asset.source_crs and source_crs is None:
|
|
_issue(issues, "SOURCE_CRS_INVALID", "crs", "source_crs", "Source CRS is not parseable.", observed=asset.source_crs)
|
|
if asset.storage_crs and storage_crs is None:
|
|
_issue(issues, "STORAGE_CRS_INVALID", "crs", "storage_crs", "Storage CRS is not parseable.", observed=asset.storage_crs)
|
|
if contract.require_storage_crs and storage_crs is None:
|
|
_issue(
|
|
issues,
|
|
"STORAGE_CRS_REQUIRED",
|
|
"crs",
|
|
"storage_crs",
|
|
"An explicit storage CRS is required by this contract.",
|
|
)
|
|
if contract.accepted_source_crs:
|
|
expected = {_normalise_crs(value) for value in contract.accepted_source_crs}
|
|
if source_crs is None or source_crs not in expected:
|
|
_issue(
|
|
issues,
|
|
"SOURCE_CRS_NOT_ALLOWED",
|
|
"crs",
|
|
"source_crs",
|
|
"Source CRS is not allowed by this contract.",
|
|
expected=sorted(value for value in expected if value),
|
|
observed=source_crs or asset.source_crs,
|
|
)
|
|
if contract.canonical_storage_crs:
|
|
expected_storage_crs = _normalise_crs(contract.canonical_storage_crs)
|
|
if storage_crs != expected_storage_crs:
|
|
_issue(
|
|
issues,
|
|
"CANONICAL_STORAGE_CRS_REQUIRED",
|
|
"crs",
|
|
"storage_crs",
|
|
"Stored coordinates must use the contract's canonical CRS.",
|
|
expected=expected_storage_crs,
|
|
observed=storage_crs or asset.storage_crs,
|
|
)
|
|
if source_crs is not None and storage_crs is not None and source_crs != storage_crs and contract.lineage_rules.require_transformation_when_crs_changes:
|
|
if not asset.lineage.transformations:
|
|
_issue(
|
|
issues,
|
|
"CRS_TRANSFORMATION_LINEAGE_REQUIRED",
|
|
"lineage",
|
|
"lineage.transformations",
|
|
"A CRS change requires an explicit transformation record.",
|
|
expected=f"{source_crs} -> {storage_crs}",
|
|
)
|
|
|
|
bounds = _coerce_bounds(asset.bounds, issues)
|
|
observed_bounds = _geometry_bounds(asset.geometry_records)
|
|
effective_bounds = observed_bounds or bounds
|
|
if contract.require_bounds and effective_bounds is None:
|
|
_issue(issues, "BOUNDS_REQUIRED", "bounds", "bounds", "Spatial bounds are required by this contract.")
|
|
if bounds is not None and not bounds.is_valid():
|
|
_issue(issues, "BOUNDS_INVALID", "bounds", "bounds", "Bounds must be finite and ordered.", observed=bounds.to_dict())
|
|
if bounds is not None and observed_bounds is not None and not bounds.nearly_equals(observed_bounds, tolerance=contract.bounds_tolerance):
|
|
_issue(
|
|
issues,
|
|
"BOUNDS_GEOMETRY_MISMATCH",
|
|
"bounds",
|
|
"bounds",
|
|
"Declared bounds do not match the geometry envelope.",
|
|
expected=observed_bounds.to_dict(),
|
|
observed=bounds.to_dict(),
|
|
)
|
|
if contract.spatial_domain is not None and effective_bounds is not None and effective_bounds.is_valid():
|
|
if not contract.spatial_domain.contains(effective_bounds, tolerance=contract.bounds_tolerance):
|
|
_issue(
|
|
issues,
|
|
"CRS_COORDINATE_DOMAIN_VIOLATION",
|
|
"crs",
|
|
"bounds",
|
|
"Coordinates fall outside the contract's declared storage CRS domain.",
|
|
expected=contract.spatial_domain.to_dict(),
|
|
observed=effective_bounds.to_dict(),
|
|
)
|
|
|
|
@staticmethod
|
|
def _check_units(contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue]) -> None:
|
|
for field_name, allowed_units in contract.expected_units.items():
|
|
observed = asset.units.get(field_name)
|
|
normalised_allowed = {unit.strip().lower() for unit in allowed_units}
|
|
if not _nonempty(observed):
|
|
_issue(
|
|
issues,
|
|
"UNIT_REQUIRED",
|
|
"units",
|
|
f"units.{field_name}",
|
|
"A declared unit is required for this field.",
|
|
expected=sorted(normalised_allowed),
|
|
)
|
|
elif str(observed).strip().lower() not in normalised_allowed:
|
|
_issue(
|
|
issues,
|
|
"UNIT_NOT_ALLOWED",
|
|
"units",
|
|
f"units.{field_name}",
|
|
"Unit is not allowed by this contract; implicit conversion is forbidden.",
|
|
expected=sorted(normalised_allowed),
|
|
observed=observed,
|
|
)
|
|
|
|
@staticmethod
|
|
def _check_resolution(contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue]) -> None:
|
|
rules = contract.resolution_rules
|
|
if rules is None:
|
|
return
|
|
resolution = _coerce_resolution(asset.resolution, issues)
|
|
if resolution is None:
|
|
if rules.required:
|
|
_issue(issues, "RESOLUTION_REQUIRED", "resolution", "resolution", "Resolution is required by this contract.")
|
|
return
|
|
if not resolution.is_valid():
|
|
_issue(issues, "RESOLUTION_INVALID", "resolution", "resolution", "Resolution must be finite, positive and unit-labelled.", observed=resolution.to_dict())
|
|
return
|
|
allowed_units = {unit.strip().lower() for unit in rules.allowed_units}
|
|
if allowed_units and resolution.unit.strip().lower() not in allowed_units:
|
|
_issue(
|
|
issues,
|
|
"RESOLUTION_UNIT_NOT_ALLOWED",
|
|
"resolution",
|
|
"resolution.unit",
|
|
"Resolution unit is not allowed; no implicit conversion is applied.",
|
|
expected=sorted(allowed_units),
|
|
observed=resolution.unit,
|
|
)
|
|
for field_name, value, minimum, maximum in (
|
|
("x", resolution.x, rules.min_x, rules.max_x),
|
|
("y", resolution.y, rules.min_y, rules.max_y),
|
|
):
|
|
if minimum is not None and value < minimum or maximum is not None and value > maximum:
|
|
_issue(
|
|
issues,
|
|
"RESOLUTION_OUT_OF_RANGE",
|
|
"resolution",
|
|
f"resolution.{field_name}",
|
|
"Resolution is outside the contract's permitted range.",
|
|
expected={"min": minimum, "max": maximum},
|
|
observed=value,
|
|
)
|
|
|
|
@classmethod
|
|
def _check_vector(cls, contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue]) -> None:
|
|
rules = contract.geometry_rules
|
|
if rules is None:
|
|
_issue(issues, "VECTOR_RULES_REQUIRED", "schema", "geometry_rules", "Vector contracts require geometry rules.")
|
|
return
|
|
records = asset.geometry_records
|
|
has_records = False
|
|
# Topology checks necessarily need the complete geometry set. Default
|
|
# GeoJSON contracts do not prohibit overlapping source features, so
|
|
# keep large regional import validation streaming unless a stricter
|
|
# source-specific contract explicitly asks for that topology rule.
|
|
parsed: list[BaseGeometry] | None = [] if rules.forbid_shared_area else None
|
|
unique_values: dict[str, dict[Any, int]] = {
|
|
field_name: {} for field_name in rules.unique_attribute_fields
|
|
}
|
|
allowed_types = {value.lower() for value in rules.allowed_geometry_types}
|
|
for index, record in enumerate(records):
|
|
has_records = True
|
|
geometry = _coerce_geometry(record.geometry, index, issues)
|
|
if geometry is None:
|
|
continue
|
|
if geometry.is_empty:
|
|
_issue(issues, "GEOMETRY_EMPTY", "geometry", f"geometry_records[{index}]", "Geometry must not be empty.")
|
|
continue
|
|
if not geometry.is_valid:
|
|
_issue(
|
|
issues,
|
|
"GEOMETRY_INVALID",
|
|
"geometry",
|
|
f"geometry_records[{index}]",
|
|
"Geometry is invalid; this validator never silently repairs geometry.",
|
|
)
|
|
continue
|
|
if allowed_types and geometry.geom_type.lower() not in allowed_types:
|
|
_issue(
|
|
issues,
|
|
"GEOMETRY_TYPE_NOT_ALLOWED",
|
|
"geometry",
|
|
f"geometry_records[{index}]",
|
|
"Geometry type is not allowed by this contract.",
|
|
expected=sorted(rules.allowed_geometry_types),
|
|
observed=geometry.geom_type,
|
|
)
|
|
cls._check_attributes(rules.attribute_rules, record.properties, index, issues)
|
|
cls._check_unique_attribute_values(
|
|
rules.unique_attribute_fields,
|
|
record.properties,
|
|
index,
|
|
unique_values,
|
|
issues,
|
|
)
|
|
if parsed is not None:
|
|
parsed.append(geometry)
|
|
if rules.require_features and not has_records:
|
|
_issue(issues, "VECTOR_FEATURES_REQUIRED", "geometry", "geometry_records", "At least one vector feature is required.")
|
|
return
|
|
if parsed is not None:
|
|
cls._check_shared_area(parsed, rules, issues)
|
|
|
|
@staticmethod
|
|
def _check_attributes(
|
|
rules: tuple[AttributeRule, ...],
|
|
properties: Mapping[str, Any],
|
|
index: int,
|
|
issues: list[ValidationIssue],
|
|
) -> None:
|
|
for rule in rules:
|
|
present = rule.name in properties
|
|
value = properties.get(rule.name)
|
|
location = f"geometry_records[{index}].properties.{rule.name}"
|
|
if not present and rule.required:
|
|
_issue(issues, "ATTRIBUTE_REQUIRED", "attributes", location, "Required attribute is missing.", expected=rule.name)
|
|
continue
|
|
if not present:
|
|
continue
|
|
if value is None:
|
|
if not rule.nullable:
|
|
_issue(issues, "ATTRIBUTE_NULL_NOT_ALLOWED", "attributes", location, "Null is not allowed for this attribute.")
|
|
continue
|
|
observed_type = _json_value_type(value)
|
|
if rule.accepted_types and observed_type not in rule.accepted_types:
|
|
_issue(
|
|
issues,
|
|
"ATTRIBUTE_TYPE_INVALID",
|
|
"attributes",
|
|
location,
|
|
"Attribute type does not match the contract.",
|
|
expected=list(rule.accepted_types),
|
|
observed=observed_type,
|
|
)
|
|
if rule.allowed_values and value not in rule.allowed_values:
|
|
_issue(
|
|
issues,
|
|
"ATTRIBUTE_VALUE_NOT_ALLOWED",
|
|
"attributes",
|
|
location,
|
|
"Attribute value is not in the contract allowlist.",
|
|
expected=_sorted_json_values(rule.allowed_values),
|
|
observed=value,
|
|
)
|
|
|
|
@staticmethod
|
|
def _check_shared_area(
|
|
geometries: list[BaseGeometry], rules: GeometryRules, issues: list[ValidationIssue]) -> None:
|
|
if len(geometries) > rules.topology_max_features:
|
|
_issue(
|
|
issues,
|
|
"TOPOLOGY_CHECK_LIMIT_EXCEEDED",
|
|
"topology",
|
|
"geometry_records",
|
|
"Topology check was not run because the batch exceeds its declared safe limit.",
|
|
expected=f"<= {rules.topology_max_features} features",
|
|
observed=len(geometries),
|
|
)
|
|
return
|
|
tree = STRtree(geometries)
|
|
for index, geometry in enumerate(geometries):
|
|
for candidate_index in tree.query(geometry):
|
|
if not isinstance(candidate_index, Integral):
|
|
continue
|
|
if candidate_index <= index:
|
|
continue
|
|
candidate = geometries[int(candidate_index)]
|
|
if geometry.intersection(candidate).area > 0.0:
|
|
_issue(
|
|
issues,
|
|
"TOPOLOGY_SHARED_AREA",
|
|
"topology",
|
|
"geometry_records",
|
|
"Features share non-zero polygon area where this contract forbids overlap.",
|
|
observed={"left_index": index, "right_index": int(candidate_index)},
|
|
)
|
|
return
|
|
|
|
@staticmethod
|
|
def _check_unique_attribute_values(
|
|
field_names: tuple[str, ...],
|
|
properties: Mapping[str, Any],
|
|
index: int,
|
|
values_by_field: dict[str, dict[Any, int]],
|
|
issues: list[ValidationIssue],
|
|
) -> None:
|
|
for field_name in field_names:
|
|
value = properties.get(field_name)
|
|
if value is None:
|
|
continue
|
|
values = values_by_field[field_name]
|
|
try:
|
|
previous_index = values.get(value)
|
|
except TypeError:
|
|
_issue(
|
|
issues,
|
|
"ATTRIBUTE_UNIQUENESS_VALUE_UNHASHABLE",
|
|
"attributes",
|
|
f"geometry_records[{index}].properties.{field_name}",
|
|
"A unique attribute must have a scalar, hashable value.",
|
|
observed=value,
|
|
)
|
|
continue
|
|
if previous_index is not None:
|
|
_issue(
|
|
issues,
|
|
"ATTRIBUTE_UNIQUENESS_VIOLATION",
|
|
"attributes",
|
|
f"geometry_records[{index}].properties.{field_name}",
|
|
"A field declared unique has a duplicate value.",
|
|
observed={"value": value, "first_index": previous_index, "duplicate_index": index},
|
|
)
|
|
continue
|
|
values[value] = index
|
|
|
|
@staticmethod
|
|
def _check_raster(contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue]) -> None:
|
|
rules = contract.raster_rules
|
|
if rules is None:
|
|
_issue(issues, "RASTER_RULES_REQUIRED", "schema", "raster_rules", "Raster contracts require raster profile rules.")
|
|
return
|
|
profile = asset.raster_profile
|
|
for field_name in rules.required_profile_fields:
|
|
if profile.get(field_name) is None:
|
|
_issue(
|
|
issues,
|
|
"RASTER_PROFILE_FIELD_REQUIRED",
|
|
"raster",
|
|
f"raster_profile.{field_name}",
|
|
"Raster profile field is required.",
|
|
)
|
|
for field_name in ("width", "height", "band_count"):
|
|
value = profile.get(field_name)
|
|
if value is not None and (not isinstance(value, Integral) or isinstance(value, bool) or value <= 0):
|
|
_issue(
|
|
issues,
|
|
"RASTER_PROFILE_VALUE_INVALID",
|
|
"raster",
|
|
f"raster_profile.{field_name}",
|
|
"Raster dimensions and band count must be positive integers.",
|
|
observed=value,
|
|
)
|
|
band_count = profile.get("band_count")
|
|
if rules.allowed_band_counts and isinstance(band_count, Integral) and band_count not in rules.allowed_band_counts:
|
|
_issue(
|
|
issues,
|
|
"RASTER_BAND_COUNT_NOT_ALLOWED",
|
|
"raster",
|
|
"raster_profile.band_count",
|
|
"Raster band count is not allowed by this contract.",
|
|
expected=sorted(rules.allowed_band_counts),
|
|
observed=band_count,
|
|
)
|
|
dtype_values = profile.get("dtype")
|
|
dtypes = dtype_values if isinstance(dtype_values, (list, tuple, set)) else [dtype_values]
|
|
if rules.allowed_dtypes and any(dtype not in rules.allowed_dtypes for dtype in dtypes if dtype is not None):
|
|
_issue(
|
|
issues,
|
|
"RASTER_DTYPE_NOT_ALLOWED",
|
|
"raster",
|
|
"raster_profile.dtype",
|
|
"Raster dtype is not allowed by this contract.",
|
|
expected=sorted(rules.allowed_dtypes),
|
|
observed=list(dtypes),
|
|
)
|
|
|
|
@staticmethod
|
|
def _check_labels(contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue]) -> None:
|
|
rules = contract.label_rules
|
|
if rules is None:
|
|
_issue(issues, "LABEL_RULES_REQUIRED", "schema", "label_rules", "Label contracts require label rules.")
|
|
return
|
|
label_mode = str(asset.label_mode or "").strip().lower()
|
|
declared_mode = str(asset.metadata.get("label_mode") or "").strip().lower()
|
|
if declared_mode and declared_mode != label_mode:
|
|
_issue(
|
|
issues,
|
|
"LABEL_MODE_MISMATCH",
|
|
"labels",
|
|
"metadata.label_mode",
|
|
"The persisted label mode must match the validation input.",
|
|
expected=label_mode,
|
|
observed=declared_mode,
|
|
)
|
|
if not asset.label_records:
|
|
if not rules.allow_empty_pure_background:
|
|
_issue(issues, "LABEL_RECORDS_REQUIRED", "labels", "label_records", "At least one label record is required.")
|
|
return
|
|
if label_mode != "pure_background" or declared_mode != "pure_background":
|
|
_issue(
|
|
issues,
|
|
"PURE_BACKGROUND_MODE_REQUIRED",
|
|
"labels",
|
|
"metadata.label_mode",
|
|
"An empty YOLO label is valid only as explicitly declared pure_background evidence.",
|
|
expected="pure_background",
|
|
observed=declared_mode or label_mode or None,
|
|
)
|
|
return
|
|
DataContractValidator._check_pure_background_label(rules, asset, issues)
|
|
return
|
|
if label_mode != "objects":
|
|
_issue(
|
|
issues,
|
|
"LABEL_MODE_WITH_OBJECTS_INVALID",
|
|
"labels",
|
|
"label_mode",
|
|
"Non-empty YOLO labels must use the objects label mode.",
|
|
expected="objects",
|
|
observed=label_mode or None,
|
|
)
|
|
for index, record in enumerate(asset.label_records):
|
|
for field_name in rules.required_fields:
|
|
if field_name not in record:
|
|
_issue(
|
|
issues,
|
|
"LABEL_FIELD_REQUIRED",
|
|
"labels",
|
|
f"label_records[{index}].{field_name}",
|
|
"Label record field is required.",
|
|
)
|
|
class_id = record.get("class_id")
|
|
if not isinstance(class_id, Integral) or isinstance(class_id, bool):
|
|
_issue(
|
|
issues,
|
|
"LABEL_CLASS_ID_INVALID",
|
|
"labels",
|
|
f"label_records[{index}].class_id",
|
|
"Label class_id must be an integer.",
|
|
observed=class_id,
|
|
)
|
|
elif rules.allowed_class_ids and class_id not in rules.allowed_class_ids:
|
|
_issue(
|
|
issues,
|
|
"LABEL_CLASS_ID_NOT_ALLOWED",
|
|
"labels",
|
|
f"label_records[{index}].class_id",
|
|
"Label class_id is not present in the contract ontology.",
|
|
expected=sorted(rules.allowed_class_ids),
|
|
observed=class_id,
|
|
)
|
|
values: dict[str, float] = {}
|
|
for field_name in ("x_center", "y_center", "width", "height"):
|
|
value = record.get(field_name)
|
|
if not isinstance(value, Real) or isinstance(value, bool) or not isfinite(float(value)):
|
|
_issue(
|
|
issues,
|
|
"LABEL_COORDINATE_INVALID",
|
|
"labels",
|
|
f"label_records[{index}].{field_name}",
|
|
"Label coordinates must be finite numeric values.",
|
|
observed=value,
|
|
)
|
|
else:
|
|
values[field_name] = float(value)
|
|
if len(values) == 4 and rules.normalized_coordinates:
|
|
x_center, y_center, width, height = (values[field] for field in ("x_center", "y_center", "width", "height"))
|
|
if width <= 0.0 or height <= 0.0 or width > 1.0 or height > 1.0 or not 0.0 <= x_center <= 1.0 or not 0.0 <= y_center <= 1.0:
|
|
_issue(
|
|
issues,
|
|
"LABEL_NORMALIZED_COORDINATE_INVALID",
|
|
"labels",
|
|
f"label_records[{index}]",
|
|
"Normalized labels must have positive dimensions and coordinates within [0, 1].",
|
|
observed=values,
|
|
)
|
|
elif x_center - width / 2 < 0.0 or x_center + width / 2 > 1.0 or y_center - height / 2 < 0.0 or y_center + height / 2 > 1.0:
|
|
_issue(
|
|
issues,
|
|
"LABEL_BOX_OUTSIDE_IMAGE",
|
|
"labels",
|
|
f"label_records[{index}]",
|
|
"Label bounding box exceeds normalized image bounds.",
|
|
observed=values,
|
|
)
|
|
|
|
@staticmethod
|
|
def _check_pure_background_label(
|
|
rules: LabelRules,
|
|
asset: DataAssetValidationInput,
|
|
issues: list[ValidationIssue],
|
|
) -> None:
|
|
"""Require explicit provenance and human-review evidence for a zero-object label."""
|
|
|
|
metadata = asset.metadata
|
|
for field_name in rules.pure_background_required_metadata_fields:
|
|
value = metadata.get(field_name)
|
|
if value is None or (isinstance(value, str) and not value.strip()):
|
|
_issue(
|
|
issues,
|
|
"PURE_BACKGROUND_EVIDENCE_REQUIRED",
|
|
"labels",
|
|
f"metadata.{field_name}",
|
|
"Pure-background labels require explicit source, split and review evidence.",
|
|
expected=field_name,
|
|
observed=value,
|
|
)
|
|
|
|
split = str(metadata.get("split") or "").strip().lower()
|
|
if rules.allowed_pure_background_splits and split not in rules.allowed_pure_background_splits:
|
|
_issue(
|
|
issues,
|
|
"PURE_BACKGROUND_SPLIT_INVALID",
|
|
"labels",
|
|
"metadata.split",
|
|
"Pure-background label split is not allowed by this contract.",
|
|
expected=sorted(rules.allowed_pure_background_splits),
|
|
observed=split or None,
|
|
)
|
|
if metadata.get("review_decision") != "accepted":
|
|
_issue(
|
|
issues,
|
|
"PURE_BACKGROUND_REVIEW_NOT_ACCEPTED",
|
|
"labels",
|
|
"metadata.review_decision",
|
|
"A zero-object label must have an accepted human review decision.",
|
|
expected="accepted",
|
|
observed=metadata.get("review_decision"),
|
|
)
|
|
if _normalise_checksum(metadata.get("review_artifact_sha256")) is None:
|
|
_issue(
|
|
issues,
|
|
"PURE_BACKGROUND_REVIEW_ARTIFACT_CHECKSUM_INVALID",
|
|
"labels",
|
|
"metadata.review_artifact_sha256",
|
|
"A zero-object label must bind the reviewed artifact checksum.",
|
|
observed=metadata.get("review_artifact_sha256"),
|
|
)
|
|
reviewed_at = metadata.get("reviewed_at")
|
|
if not isinstance(reviewed_at, str) or not reviewed_at.strip():
|
|
_issue(
|
|
issues,
|
|
"PURE_BACKGROUND_REVIEW_TIMESTAMP_INVALID",
|
|
"labels",
|
|
"metadata.reviewed_at",
|
|
"A zero-object label must record a timezone-aware review timestamp.",
|
|
observed=reviewed_at,
|
|
)
|
|
else:
|
|
try:
|
|
timestamp = datetime.fromisoformat(reviewed_at.strip().replace("Z", "+00:00"))
|
|
except ValueError:
|
|
timestamp = None
|
|
if timestamp is None or timestamp.tzinfo is None:
|
|
_issue(
|
|
issues,
|
|
"PURE_BACKGROUND_REVIEW_TIMESTAMP_INVALID",
|
|
"labels",
|
|
"metadata.reviewed_at",
|
|
"A zero-object label must record a timezone-aware review timestamp.",
|
|
observed=reviewed_at,
|
|
)
|
|
|
|
@staticmethod
|
|
def _check_model(contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue]) -> None:
|
|
rules = contract.model_rules
|
|
if rules is None:
|
|
_issue(issues, "MODEL_RULES_REQUIRED", "schema", "model_rules", "Model contracts require model metadata rules.")
|
|
return
|
|
metadata = asset.model_metadata
|
|
for field_name in rules.required_fields:
|
|
value = metadata.get(field_name)
|
|
if value is None or (isinstance(value, str) and not value.strip()):
|
|
_issue(
|
|
issues,
|
|
"MODEL_METADATA_FIELD_REQUIRED",
|
|
"model",
|
|
f"model_metadata.{field_name}",
|
|
"Model metadata field is required.",
|
|
)
|
|
model_format = metadata.get("model_format")
|
|
if rules.allowed_formats and model_format not in rules.allowed_formats:
|
|
_issue(
|
|
issues,
|
|
"MODEL_FORMAT_NOT_ALLOWED",
|
|
"model",
|
|
"model_metadata.model_format",
|
|
"Model format is not allowed by this contract.",
|
|
expected=sorted(rules.allowed_formats),
|
|
observed=model_format,
|
|
)
|
|
class_mapping = metadata.get("class_mapping")
|
|
if rules.minimum_class_count is not None:
|
|
class_count = len(class_mapping) if isinstance(class_mapping, (Mapping, list, tuple)) else 0
|
|
if class_count < rules.minimum_class_count:
|
|
_issue(
|
|
issues,
|
|
"MODEL_CLASS_MAPPING_INCOMPLETE",
|
|
"model",
|
|
"model_metadata.class_mapping",
|
|
"Model class mapping does not meet the minimum ontology size.",
|
|
expected=rules.minimum_class_count,
|
|
observed=class_count,
|
|
)
|
|
|
|
|
|
def _failed_unknown_contract_report(
|
|
asset: DataAssetValidationInput,
|
|
issue: ValidationIssue,
|
|
*,
|
|
now: datetime | None,
|
|
) -> ValidationReport:
|
|
checked_at = _as_utc(now) or datetime.now(timezone.utc)
|
|
return ValidationReport(
|
|
asset_id=asset.asset_id,
|
|
data_contract_key=asset.data_contract_key,
|
|
data_contract_version=asset.data_contract_version,
|
|
contract_fingerprint_sha256=None,
|
|
validation_status=ValidationStatus.FAILED,
|
|
provenance_status=ProvenanceStatus.INCOMPLETE,
|
|
lineage_status=LineageStatus.INCOMPLETE,
|
|
quarantine_status=QuarantineStatus.QUARANTINED,
|
|
validation_scope=("contract",),
|
|
checked_at=checked_at,
|
|
issues=(issue,),
|
|
)
|
|
|
|
|
|
def _issue(
|
|
issues: list[ValidationIssue],
|
|
code: str,
|
|
category: str,
|
|
field: str | None,
|
|
message: str,
|
|
*,
|
|
expected: Any = None,
|
|
observed: Any = None,
|
|
severity: IssueSeverity = IssueSeverity.ERROR,
|
|
) -> None:
|
|
issues.append(
|
|
ValidationIssue(
|
|
code=code,
|
|
category=category,
|
|
field=field,
|
|
message=message,
|
|
expected=expected,
|
|
observed=observed,
|
|
severity=severity,
|
|
)
|
|
)
|
|
|
|
|
|
def _normalise_crs(value: str | None) -> str | None:
|
|
if not _nonempty(value):
|
|
return None
|
|
try:
|
|
crs = CRS.from_user_input(value)
|
|
except Exception:
|
|
return None
|
|
authority = crs.to_authority()
|
|
if authority:
|
|
return f"{authority[0].upper()}:{authority[1]}"
|
|
return crs.to_string()
|
|
|
|
|
|
def _coerce_bounds(value: Any, issues: list[ValidationIssue]) -> BoundingBox | None:
|
|
if value is None:
|
|
return None
|
|
try:
|
|
return BoundingBox.from_value(value)
|
|
except ValueError as exc:
|
|
_issue(issues, "BOUNDS_FORMAT_INVALID", "bounds", "bounds", str(exc), observed=value)
|
|
return None
|
|
|
|
|
|
def _coerce_resolution(value: Any, issues: list[ValidationIssue]) -> Resolution | None:
|
|
if value is None:
|
|
return None
|
|
try:
|
|
return Resolution.from_value(value)
|
|
except ValueError as exc:
|
|
_issue(issues, "RESOLUTION_FORMAT_INVALID", "resolution", "resolution", str(exc), observed=value)
|
|
return None
|
|
|
|
|
|
def _coerce_geometry(value: BaseGeometry | Mapping[str, Any], index: int, issues: list[ValidationIssue]) -> BaseGeometry | None:
|
|
if isinstance(value, BaseGeometry):
|
|
return value
|
|
try:
|
|
return shape(value)
|
|
except Exception:
|
|
_issue(
|
|
issues,
|
|
"GEOMETRY_PARSE_FAILED",
|
|
"geometry",
|
|
f"geometry_records[{index}]",
|
|
"Geometry cannot be parsed as GeoJSON/Shapely geometry.",
|
|
)
|
|
return None
|
|
|
|
|
|
def _geometry_bounds(records: Iterable[GeometryRecord]) -> BoundingBox | None:
|
|
min_x = min_y = max_x = max_y = None
|
|
for record in records:
|
|
if isinstance(record.geometry, BaseGeometry):
|
|
geometry = record.geometry
|
|
else:
|
|
try:
|
|
geometry = shape(record.geometry)
|
|
except Exception:
|
|
continue
|
|
if not geometry.is_empty:
|
|
record_min_x, record_min_y, record_max_x, record_max_y = (
|
|
float(value) for value in geometry.bounds
|
|
)
|
|
min_x = record_min_x if min_x is None else min(min_x, record_min_x)
|
|
min_y = record_min_y if min_y is None else min(min_y, record_min_y)
|
|
max_x = record_max_x if max_x is None else max(max_x, record_max_x)
|
|
max_y = record_max_y if max_y is None else max(max_y, record_max_y)
|
|
if min_x is None or min_y is None or max_x is None or max_y is None:
|
|
return None
|
|
return BoundingBox(min_x, min_y, max_x, max_y)
|
|
|
|
|
|
def _normalise_checksum(value: str | None) -> str | None:
|
|
if not _nonempty(value):
|
|
return None
|
|
normalised = str(value).strip().lower()
|
|
return normalised if _SHA256_RE.fullmatch(normalised) else None
|
|
|
|
|
|
def _as_utc(value: datetime | None) -> datetime | None:
|
|
if value is None or value.tzinfo is None:
|
|
return None
|
|
return value.astimezone(timezone.utc)
|
|
|
|
|
|
def _nonempty(value: Any) -> bool:
|
|
return value is not None and (not isinstance(value, str) or bool(value.strip()))
|
|
|
|
|
|
def _json_value_type(value: Any) -> str:
|
|
if value is None:
|
|
return "null"
|
|
if isinstance(value, bool):
|
|
return "boolean"
|
|
if isinstance(value, Integral):
|
|
return "integer"
|
|
if isinstance(value, Real):
|
|
return "number"
|
|
if isinstance(value, str):
|
|
return "string"
|
|
if isinstance(value, Mapping):
|
|
return "object"
|
|
if isinstance(value, (list, tuple)):
|
|
return "array"
|
|
return type(value).__name__
|
|
|
|
|
|
def _contract_identity(key: str, version: str) -> tuple[str, str]:
|
|
normalized_key = key.strip()
|
|
normalized_version = version.strip()
|
|
if not normalized_key or not normalized_version:
|
|
raise ValueError("Data contract key and version must be non-empty")
|
|
return normalized_key, normalized_version
|
|
|
|
|
|
def _stable_sha256(payload: Mapping[str, Any]) -> str:
|
|
encoded = json.dumps(_json_safe(payload), sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")
|
|
return sha256(encoded).hexdigest()
|
|
|
|
|
|
def _json_safe(value: Any) -> Any:
|
|
if isinstance(value, StrEnum):
|
|
return value.value
|
|
if isinstance(value, datetime):
|
|
return _datetime_payload(value)
|
|
if isinstance(value, BoundingBox):
|
|
return value.to_dict()
|
|
if isinstance(value, Resolution):
|
|
return value.to_dict()
|
|
if isinstance(value, Mapping):
|
|
return {str(key): _json_safe(item) for key, item in value.items()}
|
|
if isinstance(value, (list, tuple, set, frozenset)):
|
|
return [_json_safe(item) for item in value]
|
|
return value
|
|
|
|
|
|
def _datetime_payload(value: datetime) -> str:
|
|
return value.astimezone(timezone.utc).isoformat()
|
|
|
|
|
|
def _geometry_rules_payload(value: GeometryRules | None) -> dict[str, Any] | None:
|
|
if value is None:
|
|
return None
|
|
return {
|
|
"allowed_geometry_types": sorted(value.allowed_geometry_types),
|
|
"attribute_rules": [
|
|
{
|
|
"name": rule.name,
|
|
"required": rule.required,
|
|
"nullable": rule.nullable,
|
|
"accepted_types": list(rule.accepted_types),
|
|
"allowed_values": _sorted_json_values(rule.allowed_values),
|
|
}
|
|
for rule in value.attribute_rules
|
|
],
|
|
"unique_attribute_fields": list(value.unique_attribute_fields),
|
|
"require_features": value.require_features,
|
|
"forbid_shared_area": value.forbid_shared_area,
|
|
"topology_max_features": value.topology_max_features,
|
|
}
|
|
|
|
|
|
def _raster_rules_payload(value: RasterRules | None) -> dict[str, Any] | None:
|
|
if value is None:
|
|
return None
|
|
return {
|
|
"required_profile_fields": list(value.required_profile_fields),
|
|
"allowed_band_counts": sorted(value.allowed_band_counts),
|
|
"allowed_dtypes": sorted(value.allowed_dtypes),
|
|
}
|
|
|
|
|
|
def _label_rules_payload(value: LabelRules | None) -> dict[str, Any] | None:
|
|
if value is None:
|
|
return None
|
|
payload: dict[str, Any] = {
|
|
"allowed_class_ids": sorted(value.allowed_class_ids),
|
|
"normalized_coordinates": value.normalized_coordinates,
|
|
"required_fields": list(value.required_fields),
|
|
}
|
|
# Keep the historical v1.0.0 fingerprint stable. Pure-background support
|
|
# is introduced by a new exact contract version rather than silently
|
|
# widening the meaning of an already frozen label contract.
|
|
if value.allow_empty_pure_background:
|
|
payload.update(
|
|
{
|
|
"allow_empty_pure_background": True,
|
|
"pure_background_required_metadata_fields": list(value.pure_background_required_metadata_fields),
|
|
"allowed_pure_background_splits": sorted(value.allowed_pure_background_splits),
|
|
}
|
|
)
|
|
return payload
|
|
|
|
|
|
def _model_rules_payload(value: ModelRules | None) -> dict[str, Any] | None:
|
|
if value is None:
|
|
return None
|
|
return {
|
|
"required_fields": list(value.required_fields),
|
|
"allowed_formats": sorted(value.allowed_formats),
|
|
"minimum_class_count": value.minimum_class_count,
|
|
}
|
|
|
|
|
|
def _resolution_rules_payload(value: ResolutionRules | None) -> dict[str, Any] | None:
|
|
if value is None:
|
|
return None
|
|
return {
|
|
"required": value.required,
|
|
"allowed_units": sorted(value.allowed_units),
|
|
"min_x": value.min_x,
|
|
"max_x": value.max_x,
|
|
"min_y": value.min_y,
|
|
"max_y": value.max_y,
|
|
}
|
|
|
|
|
|
def _freshness_rules_payload(value: FreshnessRules) -> dict[str, Any]:
|
|
return {
|
|
"observed_at": value.observed_at.value,
|
|
"source_version": value.source_version.value,
|
|
"imported_at_required": value.imported_at_required,
|
|
"max_age_seconds": value.max_age.total_seconds() if value.max_age else None,
|
|
"allow_future_observation": value.allow_future_observation,
|
|
}
|
|
|
|
|
|
def _lineage_rules_payload(value: LineageRules) -> dict[str, Any]:
|
|
return {
|
|
"require_source_registry": value.require_source_registry,
|
|
"require_source_snapshot": value.require_source_snapshot,
|
|
"require_upstream_assets": value.require_upstream_assets,
|
|
"require_transformation_when_crs_changes": value.require_transformation_when_crs_changes,
|
|
}
|
|
|
|
|
|
def _sorted_json_values(values: Iterable[Any]) -> list[Any]:
|
|
serialised = [_json_safe(value) for value in values]
|
|
return sorted(serialised, key=lambda value: json.dumps(value, sort_keys=True, ensure_ascii=True))
|
|
|
|
|
|
def _provenance_status(contract: DataContract, issues: list[ValidationIssue]) -> ProvenanceStatus:
|
|
categories = {"provenance", "checksum", "temporal", "freshness", "crs", "bounds", "units", "resolution", "metadata"}
|
|
if any(issue.category in categories for issue in issues):
|
|
return ProvenanceStatus.INCOMPLETE
|
|
if not contract.lineage_rules.require_source_registry and not contract.lineage_rules.require_source_snapshot:
|
|
return ProvenanceStatus.NOT_APPLICABLE
|
|
return ProvenanceStatus.COMPLETE
|
|
|
|
|
|
def _lineage_status(contract: DataContract, issues: list[ValidationIssue]) -> LineageStatus:
|
|
if any(issue.category == "lineage" for issue in issues):
|
|
return LineageStatus.INCOMPLETE
|
|
if not contract.lineage_rules.require_upstream_assets and not contract.lineage_rules.require_transformation_when_crs_changes:
|
|
return LineageStatus.NOT_APPLICABLE
|
|
return LineageStatus.COMPLETE
|
|
|
|
|
|
def _validation_scope(contract: DataContract) -> tuple[str, ...]:
|
|
checks = ["contract", "checksum", "metadata", "temporal", "provenance", "lineage", "crs", "bounds", "units"]
|
|
if contract.resolution_rules is not None:
|
|
checks.append("resolution")
|
|
checks.append(contract.kind.value)
|
|
return tuple(checks)
|
|
|
|
|
|
# The generic contracts below are deliberately narrow in evidence requirements
|
|
# but broad in legitimate Belgian source CRSs. A source-specific registry may
|
|
# register an additional, stricter version; ingestion must always choose an
|
|
# explicit key/version and may never silently choose a "latest" contract.
|
|
VECTOR_GEOJSON_CONTRACT_KEY = "geointel.vector.geojson"
|
|
VECTOR_GEOJSON_CONTRACT_VERSION = "1.0.0"
|
|
RASTER_GEOTIFF_CONTRACT_KEY = "geointel.raster.geotiff"
|
|
RASTER_GEOTIFF_CONTRACT_VERSION = "1.0.0"
|
|
YOLO_LABEL_CONTRACT_KEY = "geointel.label.yolo"
|
|
YOLO_LABEL_LEGACY_CONTRACT_VERSION = "1.0.0"
|
|
YOLO_LABEL_CONTRACT_VERSION = "1.1.0"
|
|
PYTORCH_MODEL_CONTRACT_KEY = "geointel.model.pytorch"
|
|
PYTORCH_MODEL_CONTRACT_VERSION = "1.0.0"
|
|
|
|
_BELGIUM_AND_NORTH_SEA_WGS84_DOMAIN = BoundingBox(min_x=1.5, min_y=48.5, max_x=7.5, max_y=52.5)
|
|
_BELGIAN_SOURCE_CRS = frozenset({"EPSG:4326", "EPSG:31370", "EPSG:3812"})
|
|
|
|
|
|
def build_default_data_contract_registry() -> DataContractRegistry:
|
|
"""Build the concrete exact-version registry used by generic ingestion.
|
|
|
|
The defaults are not a trust registry. They validate a safely staged
|
|
artifact only after the caller supplies server-attested source registry and
|
|
snapshot identities. GRB/PICC/UrbIS and source-specific semantic rules are
|
|
intentionally supplied by stricter source-registry contracts.
|
|
"""
|
|
|
|
vector_contract = DataContract(
|
|
key=VECTOR_GEOJSON_CONTRACT_KEY,
|
|
version=VECTOR_GEOJSON_CONTRACT_VERSION,
|
|
kind=ContractKind.VECTOR,
|
|
accepted_source_crs=_BELGIAN_SOURCE_CRS,
|
|
canonical_storage_crs="EPSG:4326",
|
|
spatial_domain=_BELGIUM_AND_NORTH_SEA_WGS84_DOMAIN,
|
|
require_bounds=True,
|
|
required_metadata_fields=("license",),
|
|
geometry_rules=GeometryRules(require_features=True),
|
|
freshness_rules=FreshnessRules(
|
|
observed_at=RequirementLevel.UNKNOWN_WITH_REASON,
|
|
source_version=RequirementLevel.UNKNOWN_WITH_REASON,
|
|
),
|
|
)
|
|
raster_contract = DataContract(
|
|
key=RASTER_GEOTIFF_CONTRACT_KEY,
|
|
version=RASTER_GEOTIFF_CONTRACT_VERSION,
|
|
kind=ContractKind.RASTER,
|
|
accepted_source_crs=_BELGIAN_SOURCE_CRS,
|
|
require_bounds=True,
|
|
required_metadata_fields=("license",),
|
|
raster_rules=RasterRules(),
|
|
resolution_rules=ResolutionRules(
|
|
allowed_units=frozenset({"m", "degree"}),
|
|
min_x=0.000001,
|
|
max_x=10_000.0,
|
|
min_y=0.000001,
|
|
max_y=10_000.0,
|
|
),
|
|
freshness_rules=FreshnessRules(
|
|
observed_at=RequirementLevel.UNKNOWN_WITH_REASON,
|
|
source_version=RequirementLevel.UNKNOWN_WITH_REASON,
|
|
),
|
|
lineage_rules=LineageRules(require_transformation_when_crs_changes=False),
|
|
)
|
|
legacy_label_contract = DataContract(
|
|
key=YOLO_LABEL_CONTRACT_KEY,
|
|
version=YOLO_LABEL_LEGACY_CONTRACT_VERSION,
|
|
kind=ContractKind.LABEL,
|
|
require_storage_crs=False,
|
|
required_metadata_fields=("image_checksum_sha256", "class_ontology_version", "tile_manifest_sha256"),
|
|
metadata_checksum_fields=("image_checksum_sha256", "tile_manifest_sha256"),
|
|
label_rules=LabelRules(allowed_class_ids=frozenset({0})),
|
|
freshness_rules=FreshnessRules(
|
|
observed_at=RequirementLevel.UNKNOWN_WITH_REASON,
|
|
source_version=RequirementLevel.UNKNOWN_WITH_REASON,
|
|
),
|
|
lineage_rules=LineageRules(
|
|
require_source_registry=True,
|
|
require_source_snapshot=True,
|
|
require_upstream_assets=True,
|
|
require_transformation_when_crs_changes=False,
|
|
),
|
|
)
|
|
label_contract = DataContract(
|
|
key=YOLO_LABEL_CONTRACT_KEY,
|
|
version=YOLO_LABEL_CONTRACT_VERSION,
|
|
kind=ContractKind.LABEL,
|
|
require_storage_crs=False,
|
|
required_metadata_fields=(
|
|
"image_checksum_sha256",
|
|
"class_ontology_version",
|
|
"source_corpus_manifest_sha256",
|
|
"label_mode",
|
|
),
|
|
metadata_checksum_fields=("image_checksum_sha256", "source_corpus_manifest_sha256"),
|
|
label_rules=LabelRules(
|
|
allowed_class_ids=frozenset({0}),
|
|
allow_empty_pure_background=True,
|
|
pure_background_required_metadata_fields=(
|
|
"sample_slug",
|
|
"split",
|
|
"raster_dataset_id",
|
|
"reference_dataset_id",
|
|
"review_decision",
|
|
"reviewer_id",
|
|
"reviewed_at",
|
|
"review_artifact_sha256",
|
|
),
|
|
allowed_pure_background_splits=frozenset({"train", "val"}),
|
|
),
|
|
freshness_rules=FreshnessRules(
|
|
observed_at=RequirementLevel.UNKNOWN_WITH_REASON,
|
|
source_version=RequirementLevel.UNKNOWN_WITH_REASON,
|
|
),
|
|
lineage_rules=LineageRules(
|
|
require_source_registry=True,
|
|
require_source_snapshot=True,
|
|
require_upstream_assets=True,
|
|
require_transformation_when_crs_changes=False,
|
|
),
|
|
)
|
|
model_contract = DataContract(
|
|
key=PYTORCH_MODEL_CONTRACT_KEY,
|
|
version=PYTORCH_MODEL_CONTRACT_VERSION,
|
|
kind=ContractKind.MODEL,
|
|
require_storage_crs=False,
|
|
required_metadata_fields=("training_manifest_sha256", "runtime_manifest_sha256"),
|
|
metadata_checksum_fields=("training_manifest_sha256", "runtime_manifest_sha256"),
|
|
model_rules=ModelRules(allowed_formats=frozenset({"pytorch", "ultralytics"}), minimum_class_count=1),
|
|
freshness_rules=FreshnessRules(
|
|
observed_at=RequirementLevel.NOT_APPLICABLE,
|
|
source_version=RequirementLevel.REQUIRED,
|
|
),
|
|
lineage_rules=LineageRules(
|
|
require_source_registry=True,
|
|
require_source_snapshot=True,
|
|
require_upstream_assets=True,
|
|
require_transformation_when_crs_changes=False,
|
|
),
|
|
)
|
|
return DataContractRegistry((vector_contract, raster_contract, legacy_label_contract, label_contract, model_contract))
|
|
|
|
|
|
def validate_registered_asset(
|
|
asset: DataAssetValidationInput,
|
|
*,
|
|
registry: DataContractRegistry | None = None,
|
|
now: datetime | None = None,
|
|
) -> ValidationReport:
|
|
"""Validate an explicitly versioned asset against a supplied/default registry."""
|
|
|
|
active_registry = registry or build_default_data_contract_registry()
|
|
return active_registry.validate(asset, now=now)
|
|
|
|
|
|
def build_vector_ingest_input(
|
|
*,
|
|
asset_id: str,
|
|
source_crs: str | None,
|
|
storage_crs: str | None,
|
|
feature_collection: Mapping[str, Any],
|
|
checksum_sha256: str | None,
|
|
computed_checksum_sha256: str | None,
|
|
source_registry_id: str | None,
|
|
source_snapshot_id: str | None,
|
|
imported_at: datetime | None,
|
|
metadata: Mapping[str, Any] | None = None,
|
|
content: bytes | None = None,
|
|
units: Mapping[str, str] | None = None,
|
|
observed_at: datetime | None = None,
|
|
valid_from: datetime | None = None,
|
|
valid_to: datetime | None = None,
|
|
temporal_unknown_reason: str | None = None,
|
|
source_version: str | None = None,
|
|
source_version_unknown_reason: str | None = None,
|
|
lineage: LineageEvidence | None = None,
|
|
data_contract_key: str = VECTOR_GEOJSON_CONTRACT_KEY,
|
|
data_contract_version: str = VECTOR_GEOJSON_CONTRACT_VERSION,
|
|
) -> DataAssetValidationInput:
|
|
"""Adapt a GeoJSON FeatureCollection to the generic vector contract input."""
|
|
|
|
raw_features = feature_collection.get("features")
|
|
records = ()
|
|
if isinstance(raw_features, list):
|
|
records = tuple(
|
|
GeometryRecord(
|
|
geometry=feature.get("geometry", {}),
|
|
properties=feature.get("properties") if isinstance(feature.get("properties"), Mapping) else {},
|
|
identifier=str(feature.get("id")) if feature.get("id") is not None else None,
|
|
)
|
|
for feature in raw_features
|
|
if isinstance(feature, Mapping)
|
|
)
|
|
merged_metadata = dict(metadata or {})
|
|
bounds = merged_metadata.get("bounds_json", merged_metadata.get("bounds"))
|
|
return DataAssetValidationInput(
|
|
asset_id=asset_id,
|
|
data_contract_key=data_contract_key,
|
|
data_contract_version=data_contract_version,
|
|
kind=ContractKind.VECTOR,
|
|
source_crs=source_crs,
|
|
storage_crs=storage_crs,
|
|
bounds=bounds,
|
|
checksum_sha256=checksum_sha256,
|
|
computed_checksum_sha256=computed_checksum_sha256,
|
|
content=content,
|
|
metadata=merged_metadata,
|
|
units=units or {},
|
|
geometry_records=records,
|
|
source_registry_id=source_registry_id,
|
|
source_snapshot_id=source_snapshot_id,
|
|
lineage=lineage or LineageEvidence(),
|
|
imported_at=imported_at,
|
|
observed_at=observed_at,
|
|
valid_from=valid_from,
|
|
valid_to=valid_to,
|
|
temporal_unknown_reason=temporal_unknown_reason,
|
|
source_version=source_version,
|
|
source_version_unknown_reason=source_version_unknown_reason,
|
|
)
|
|
|
|
|
|
def build_raster_ingest_input(
|
|
*,
|
|
asset_id: str,
|
|
source_crs: str | None,
|
|
storage_crs: str | None,
|
|
raster_profile: Mapping[str, Any],
|
|
bounds: BoundingBox | Mapping[str, Any] | Sequence[float] | None,
|
|
resolution: Resolution | Mapping[str, Any] | Sequence[Any] | None,
|
|
checksum_sha256: str | None,
|
|
computed_checksum_sha256: str | None,
|
|
source_registry_id: str | None,
|
|
source_snapshot_id: str | None,
|
|
imported_at: datetime | None,
|
|
metadata: Mapping[str, Any] | None = None,
|
|
content: bytes | None = None,
|
|
units: Mapping[str, str] | None = None,
|
|
observed_at: datetime | None = None,
|
|
valid_from: datetime | None = None,
|
|
valid_to: datetime | None = None,
|
|
temporal_unknown_reason: str | None = None,
|
|
source_version: str | None = None,
|
|
source_version_unknown_reason: str | None = None,
|
|
lineage: LineageEvidence | None = None,
|
|
data_contract_key: str = RASTER_GEOTIFF_CONTRACT_KEY,
|
|
data_contract_version: str = RASTER_GEOTIFF_CONTRACT_VERSION,
|
|
) -> DataAssetValidationInput:
|
|
"""Adapt extracted GeoTIFF metadata to the generic raster contract input."""
|
|
|
|
return DataAssetValidationInput(
|
|
asset_id=asset_id,
|
|
data_contract_key=data_contract_key,
|
|
data_contract_version=data_contract_version,
|
|
kind=ContractKind.RASTER,
|
|
source_crs=source_crs,
|
|
storage_crs=storage_crs,
|
|
bounds=bounds,
|
|
checksum_sha256=checksum_sha256,
|
|
computed_checksum_sha256=computed_checksum_sha256,
|
|
content=content,
|
|
metadata=dict(metadata or {}),
|
|
units=units or {},
|
|
resolution=resolution,
|
|
raster_profile=dict(raster_profile),
|
|
source_registry_id=source_registry_id,
|
|
source_snapshot_id=source_snapshot_id,
|
|
lineage=lineage or LineageEvidence(),
|
|
imported_at=imported_at,
|
|
observed_at=observed_at,
|
|
valid_from=valid_from,
|
|
valid_to=valid_to,
|
|
temporal_unknown_reason=temporal_unknown_reason,
|
|
source_version=source_version,
|
|
source_version_unknown_reason=source_version_unknown_reason,
|
|
)
|
|
|
|
|
|
def build_label_validation_input(
|
|
*,
|
|
asset_id: str,
|
|
label_records: Sequence[Mapping[str, Any]],
|
|
checksum_sha256: str | None,
|
|
computed_checksum_sha256: str | None,
|
|
source_registry_id: str | None,
|
|
source_snapshot_id: str | None,
|
|
imported_at: datetime | None,
|
|
metadata: Mapping[str, Any] | None = None,
|
|
content: bytes | None = None,
|
|
label_mode: str = "objects",
|
|
observed_at: datetime | None = None,
|
|
temporal_unknown_reason: str | None = None,
|
|
source_version: str | None = None,
|
|
source_version_unknown_reason: str | None = None,
|
|
lineage: LineageEvidence | None = None,
|
|
data_contract_key: str = YOLO_LABEL_CONTRACT_KEY,
|
|
data_contract_version: str = YOLO_LABEL_CONTRACT_VERSION,
|
|
) -> DataAssetValidationInput:
|
|
"""Build a strict YOLO-label validation input with explicit upstream lineage."""
|
|
|
|
normalized_metadata = dict(metadata or {})
|
|
normalized_metadata.setdefault("label_mode", label_mode)
|
|
|
|
return DataAssetValidationInput(
|
|
asset_id=asset_id,
|
|
data_contract_key=data_contract_key,
|
|
data_contract_version=data_contract_version,
|
|
kind=ContractKind.LABEL,
|
|
checksum_sha256=checksum_sha256,
|
|
computed_checksum_sha256=computed_checksum_sha256,
|
|
content=content,
|
|
metadata=normalized_metadata,
|
|
label_records=tuple(label_records),
|
|
label_mode=label_mode,
|
|
source_registry_id=source_registry_id,
|
|
source_snapshot_id=source_snapshot_id,
|
|
lineage=lineage or LineageEvidence(),
|
|
imported_at=imported_at,
|
|
observed_at=observed_at,
|
|
temporal_unknown_reason=temporal_unknown_reason,
|
|
source_version=source_version,
|
|
source_version_unknown_reason=source_version_unknown_reason,
|
|
)
|
|
|
|
|
|
def build_model_validation_input(
|
|
*,
|
|
asset_id: str,
|
|
model_metadata: Mapping[str, Any],
|
|
checksum_sha256: str | None,
|
|
computed_checksum_sha256: str | None,
|
|
source_registry_id: str | None,
|
|
source_snapshot_id: str | None,
|
|
imported_at: datetime | None,
|
|
metadata: Mapping[str, Any] | None = None,
|
|
content: bytes | None = None,
|
|
source_version: str | None = None,
|
|
lineage: LineageEvidence | None = None,
|
|
data_contract_key: str = PYTORCH_MODEL_CONTRACT_KEY,
|
|
data_contract_version: str = PYTORCH_MODEL_CONTRACT_VERSION,
|
|
) -> DataAssetValidationInput:
|
|
"""Build a model-asset validation input; model output is never inferred."""
|
|
|
|
return DataAssetValidationInput(
|
|
asset_id=asset_id,
|
|
data_contract_key=data_contract_key,
|
|
data_contract_version=data_contract_version,
|
|
kind=ContractKind.MODEL,
|
|
checksum_sha256=checksum_sha256,
|
|
computed_checksum_sha256=computed_checksum_sha256,
|
|
content=content,
|
|
metadata=dict(metadata or {}),
|
|
model_metadata=dict(model_metadata),
|
|
source_registry_id=source_registry_id,
|
|
source_snapshot_id=source_snapshot_id,
|
|
lineage=lineage or LineageEvidence(),
|
|
imported_at=imported_at,
|
|
source_version=source_version,
|
|
)
|