"""Server-owned source identity, snapshot and lineage persistence primitives. This service deliberately does not inspect caller-provided ``source_name`` metadata. A governed adapter must select one of the static definitions below, create an immutable snapshot and bind that exact snapshot to a dataset/version. """ from __future__ import annotations from dataclasses import dataclass from datetime import datetime import re from typing import Any from uuid import UUID from sqlalchemy.orm import Session from app.core.errors import AppError from app.models import ( Dataset, DatasetLineageEdge, DatasetQuarantine, DatasetVersion, SourceRegistry, SourceSnapshot, ) _CHECKSUM = re.compile(r"^[a-fA-F0-9]{64}$") _VALIDATION_STATUSES = {"not_validated", "passed", "failed"} _PROVENANCE_STATUSES = {"complete", "incomplete", "not_applicable"} _LINEAGE_STATUSES = {"complete", "incomplete", "not_applicable"} _FRESHNESS_STATUSES = { "unknown", "current", "due", "stale", "not_applicable", "review_required", } _SNAPSHOT_INGEST_STATUSES = { "registered", "configured", "not_configured", "available", "ingested", "failed", "quarantined", "legacy_unverified", } _MAX_LINEAGE_GRAPH_NODES = 50_000 def _usage_policy( *, ground_truth_allowed: bool = False, training_allowed: bool = False, allowed_tasks: tuple[str, ...] = (), validation_authority: dict[str, str] | None = None, ) -> dict[str, Any]: return { "automatic_ground_truth": False, "ground_truth_allowed": ground_truth_allowed, "training_allowed": training_allowed, "allowed_tasks": list(allowed_tasks), "validation_authority": validation_authority or {}, } @dataclass(frozen=True) class SourceRegistryDefinition: source_key: str display_name: str classification: str authority_name: str authority_scope: dict[str, Any] default_crs: str = "unknown" default_units: str = "unknown" provider_adapter_key: str | None = None source_url: str | None = None license_name: str = "Provider terms must be verified for each immutable snapshot." usage_restrictions: str = ( "Use only according to the source-specific snapshot terms and attribution." ) spatial_resolution: dict[str, Any] | None = None temporal_coverage: dict[str, Any] | None = None geographic_coverage: dict[str, Any] | None = None expected_geometry_types: tuple[str, ...] = () expected_attributes: dict[str, Any] | None = None usage_policy: dict[str, Any] | None = None freshness_status: str = "unknown" ingest_status: str = "registered" known_limitations: tuple[str, ...] = () def as_model_values(self) -> dict[str, Any]: return { "source_key": self.source_key, "display_name": self.display_name, "classification": self.classification, "authority_name": self.authority_name, "authority_scope_json": dict(self.authority_scope), "provider_adapter_key": self.provider_adapter_key, "source_url": self.source_url, "license_name": self.license_name, "usage_restrictions": self.usage_restrictions, "default_crs": self.default_crs, "default_units": self.default_units, "spatial_resolution_json": dict( self.spatial_resolution or {"status": "unknown"} ), "temporal_coverage_json": dict( self.temporal_coverage or {"status": "unknown"} ), "geographic_coverage_json": dict( self.geographic_coverage or {"status": "unknown"} ), "expected_geometry_types_json": list(self.expected_geometry_types), "expected_attributes_json": dict( self.expected_attributes or {"status": "unknown"} ), "usage_policy_json": dict(self.usage_policy or _usage_policy()), "freshness_status": self.freshness_status, "ingest_status": self.ingest_status, "known_limitations_json": list( self.known_limitations or ( "No authority, ground-truth, freshness or training claim is allowed without a governed snapshot and passed contract.", ) ), "registry_metadata_json": { "registry_owner": "server", "definition_version": "phase2-v1", }, } def _definition( source_key: str, display_name: str, classification: str, authority_name: str, authority_scope: dict[str, Any], **kwargs: Any, ) -> SourceRegistryDefinition: return SourceRegistryDefinition( source_key=source_key, display_name=display_name, classification=classification, authority_name=authority_name, authority_scope=authority_scope, **kwargs, ) _VECTOR_CONTEXT = _usage_policy(allowed_tasks=("reference_context",)) _IMAGERY_CONTEXT = _usage_policy( training_allowed=True, allowed_tasks=("imagery", "training_input", "visual_context"), ) _REGIONAL_BUILDING_LABELS = _usage_policy( ground_truth_allowed=True, training_allowed=True, allowed_tasks=("building_validation", "building_labels"), validation_authority={"building_validation": "regional_primary_pending_contract"}, ) SERVER_OWNED_SOURCE_DEFINITIONS: dict[str, SourceRegistryDefinition] = { definition.source_key: definition for definition in ( _definition( "grb", "Grootschalig Referentie Bestand", "authoritative", "Digitaal Vlaanderen", {"zone": "Flanders", "themes": ["buildings", "roads", "water", "parcels"]}, provider_adapter_key="grb", source_url="https://www.vlaanderen.be/datavindplaats/catalogus/basiskaart-vlaanderen-grb", default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=( "Polygon", "MultiPolygon", "LineString", "MultiLineString", ), expected_attributes={ "required": ["id"], "layers": ["GBG", "Wegsegment", "WTZ", "WLAS", "WGR", "ADP"], }, usage_policy=_usage_policy( ground_truth_allowed=True, training_allowed=True, allowed_tasks=( "building_validation", "building_labels", "reference_context", ), validation_authority={"building_validation": "primary"}, ), ingest_status="configured", known_limitations=( "GRB building geometry is authoritative only for a governed, versioned snapshot within Flanders.", "GRB does not independently establish imagery-time alignment or national model validation.", ), ), _definition( "digitaal_vlaanderen", "Digitaal Vlaanderen (bronportaal)", "authoritative", "Digitaal Vlaanderen", {"zone": "Flanders", "role": "umbrella_catalogue_and_adapter_authority"}, provider_adapter_key="digitaal_vlaanderen", source_url="https://www.vlaanderen.be/datavindplaats", default_crs="product_specific", default_units="product_specific", geographic_coverage={"zone": "Flanders"}, usage_policy=_usage_policy( allowed_tasks=("source_catalogue", "reference_context") ), ingest_status="configured", known_limitations=( "This umbrella authority is not a product-level ground-truth source.", "A governed import must use a product-specific source key whenever one is available.", ), ), _definition( "digitaal_vlaanderen_buildings_addresses_register", "Gebouwen- en adressenregister", "authoritative", "Digitaal Vlaanderen", {"zone": "Flanders", "theme": "buildings_addresses"}, provider_adapter_key="buildings_addresses_register", default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=("Point", "Polygon", "MultiPolygon"), expected_attributes={ "required": ["id"], "role": "administrative_corroboration", }, usage_policy=_usage_policy( allowed_tasks=( "building_validation", "address_corroboration", "building_register_validation", ), validation_authority={ "building_validation": "corroborative", "building_register_validation": "primary", }, ), known_limitations=( "Administrative records do not replace a governed footprint-label contract.", ), ), _definition( "sentinel_2", "Sentinel-2", "contextual", "Copernicus Programme", {"scope": "Belgium and Belgian North Sea", "role": "multispectral_context"}, provider_adapter_key="sentinel_2", default_crs="product_specific", default_units="reflectance", spatial_resolution={"metres": [10, 20, 60]}, temporal_coverage={"cadence_days": 5, "status": "product_specific"}, geographic_coverage={"scope": "Belgium and Belgian North Sea"}, expected_attributes={"required": ["product_id", "sensing_time"]}, usage_policy=_usage_policy( training_allowed=True, allowed_tasks=("imagery_context", "change_context"), ), ingest_status="not_configured", known_limitations=( "Sentinel-2 is contextual imagery, never automatic building ground truth.", ), ), _definition( "digitaal_vlaanderen_dhmv", "Digitaal Hoogtemodel Vlaanderen", "authoritative", "Digitaal Vlaanderen", {"zone": "Flanders", "role": "terrain_height_corroboration"}, provider_adapter_key="dhmv", default_crs="EPSG:31370", default_units="m TAW", spatial_resolution={"metres": 1}, temporal_coverage={"period": "2013-2015", "status": "product_specific"}, geographic_coverage={"zone": "Flanders"}, expected_attributes={"bands": 1, "nodata_required": True}, usage_policy=_usage_policy( training_allowed=True, allowed_tasks=( "terrain_context", "height_corroboration", "elevation_validation", ), validation_authority={ "building_validation": "corroborative", "elevation_validation": "primary", }, ), ingest_status="configured", known_limitations=( "DHMV is height context and cannot independently establish building labels.", ), ), _definition( "osm", "OpenStreetMap", "contextual", "OpenStreetMap contributors", {"scope": "community-maintained", "role": "contextual"}, provider_adapter_key="osm", source_url="https://www.openstreetmap.org", license_name="ODbL", usage_restrictions="OpenStreetMap attribution and ODbL obligations apply.", default_crs="EPSG:4326", default_units="mixed", geographic_coverage={"scope": "global"}, expected_geometry_types=("Point", "LineString", "Polygon", "MultiPolygon"), expected_attributes={"status": "community_tags"}, usage_policy=_usage_policy( allowed_tasks=("context", "candidate_discovery") ), ingest_status="not_configured", known_limitations=( "OSM is never automatic ground truth for GeoIntel validation or labels.", ), ), _definition( "manual", "Handmatige upload", "experimental", "Operator supplied", {"scope": "operator_supplied", "trust": "unverified"}, known_limitations=( "Manual uploads remain untrusted until a passed contract and governed provenance are attached.", ), ingest_status="configured", ), _definition( "fixture", "Test- en demo fixture", "experimental", "GeoIntel test fixture", {"scope": "test_only"}, default_crs="fixture_specific", default_units="fixture_specific", known_limitations=( "Fixtures must never be presented as official data or used for production training/promotion.", ), ingest_status="configured", ), _definition( "map_selection", "Afgeleide kaartselectie", "derived", "GeoIntel derived operation", {"scope": "derived_from_registered_input"}, default_crs="EPSG:4326", default_units="source_dependent", known_limitations=( "Derived selections inherit no authority beyond complete source snapshots and lineage edges.", ), ), _definition( "derived", "Afgeleide dataset", "derived", "GeoIntel derived operation", {"scope": "derived_from_registered_input"}, ), _definition( "training_label", "Afgeleide trainingslabels", "derived", "GeoIntel reviewed label pipeline", {"scope": "derived_from_reviewed_source_snapshots"}, known_limitations=( "Training labels require complete source lineage and human-review evidence; they inherit no automatic authority.", ), ), _definition( "model", "Model artifact", "experimental", "GeoIntel model pipeline", {"scope": "internal_model_artifact"}, freshness_status="current", ingest_status="configured", known_limitations=( "A model artifact is not a validated capability or promotion decision without its model card and evaluation evidence.", ), ), _definition( "experimental", "Experimentele bron", "experimental", "Unverified", {"scope": "unverified"}, ), _definition( "legacy_unknown", "Niet-geclassificeerde historische bron", "experimental", "Legacy import — unverified", {"scope": "legacy", "trust": "unverified"}, ingest_status="legacy_unverified", known_limitations=( "Historical source identity is descriptive only until re-ingested through a governed adapter.", ), ), _definition( "ngi_adminvector", "NGI AdminVector", "authoritative", "Nationaal Geografisch Instituut", {"scope": "Belgium"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"scope": "Belgium"}, expected_geometry_types=("Polygon", "MultiPolygon"), usage_policy=_VECTOR_CONTEXT, ), _definition( "rbins_marine_reporting_units", "RBINS mariene rapportage-eenheden", "authoritative", "RBINS", {"zone": "Belgian North Sea"}, default_crs="EPSG:4326", default_units="degrees", geographic_coverage={"zone": "Belgian North Sea"}, expected_geometry_types=("Polygon", "MultiPolygon"), usage_policy=_VECTOR_CONTEXT, ), _definition( "rbins_msp_2026", "Belgisch Marien Ruimtelijk Plan 2026-2034", "authoritative", "RBINS", {"zone": "Belgian North Sea", "edition": "2026-2034"}, default_crs="EPSG:4326", default_units="degrees", geographic_coverage={"zone": "Belgian North Sea"}, expected_geometry_types=("Polygon", "MultiPolygon"), usage_policy=_VECTOR_CONTEXT, ), _definition( "vrbg", "Vlaams Wegenregister", "authoritative", "Digitaal Vlaanderen", {"zone": "Flanders", "theme": "roads"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=("LineString", "MultiLineString"), usage_policy=_VECTOR_CONTEXT, ), _definition( "digitaal_vlaanderen_orthophoto", "Orthofoto Vlaanderen", "contextual", "Digitaal Vlaanderen", {"zone": "Flanders", "role": "imagery"}, default_crs="EPSG:31370", default_units="pixel", spatial_resolution={"metres": 0.25}, geographic_coverage={"zone": "Flanders"}, usage_policy=_IMAGERY_CONTEXT, ingest_status="configured", ), _definition( "spw_orthophoto", "Orthofoto Wallonië", "contextual", "Service public de Wallonie", {"zone": "Wallonia", "role": "imagery"}, license_name="CC BY 4.0", default_crs="EPSG:31370", default_units="pixel", spatial_resolution={"metres": 0.25}, geographic_coverage={"zone": "Wallonia"}, usage_policy=_IMAGERY_CONTEXT, ingest_status="configured", ), _definition( "urbis_orthophoto", "Orthofoto Brussel", "contextual", "UrbIS / Brussels Region", {"zone": "Brussels-Capital Region", "role": "imagery"}, license_name="CC0", default_crs="EPSG:31370", default_units="pixel", spatial_resolution={"metres": 0.25}, geographic_coverage={"zone": "Brussels-Capital Region"}, usage_policy=_IMAGERY_CONTEXT, ingest_status="configured", ), _definition( "agentschap_landbouw_zeevisserij_agricultural_parcels", "Landbouwgebruikspercelen", "authoritative", "Agentschap Landbouw en Zeevisserij", {"zone": "Flanders", "theme": "agricultural_parcels"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=("Polygon", "MultiPolygon"), usage_policy=_VECTOR_CONTEXT, ), _definition( "department_omgeving_land_use", "Landgebruik Vlaanderen", "authoritative", "Departement Omgeving", {"zone": "Flanders", "theme": "land_use"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=("Polygon", "MultiPolygon"), usage_policy=_VECTOR_CONTEXT, ), _definition( "inbo_bwk_natura2000", "BWK en Natura 2000", "authoritative", "INBO", {"zone": "Flanders", "theme": "nature"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=("Polygon", "MultiPolygon"), usage_policy=_VECTOR_CONTEXT, ), _definition( "statbel", "Statbel bevolking", "authoritative", "Statbel", {"scope": "Belgium", "theme": "population"}, default_crs="EPSG:31370", default_units="persons", geographic_coverage={"scope": "Belgium"}, expected_geometry_types=("Polygon", "MultiPolygon"), usage_policy=_VECTOR_CONTEXT, ), _definition( "waterinfo", "Waterinfo", "authoritative", "Waterinfo Vlaanderen", {"zone": "Flanders", "theme": "water"}, default_crs="EPSG:31370", default_units="source_specific", geographic_coverage={"zone": "Flanders"}, usage_policy=_VECTOR_CONTEXT, ), _definition( "department_omgeving_thematic_raster", "Omgeving thematische rasters", "authoritative", "Departement Omgeving", {"zone": "Flanders", "theme": "thematic_raster"}, default_crs="EPSG:31370", default_units="source_specific", geographic_coverage={"zone": "Flanders"}, usage_policy=_VECTOR_CONTEXT, ), _definition( "dov_soil_map", "DOV bodemkaart", "authoritative", "Databank Ondergrond Vlaanderen", {"zone": "Flanders", "theme": "soil"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=("Polygon", "MultiPolygon"), usage_policy=_VECTOR_CONTEXT, ), _definition( "vmm_flood_hazard", "VMM overstromingskaarten", "authoritative", "Vlaamse Milieumaatschappij", {"zone": "Flanders", "theme": "flood_hazard"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, usage_policy=_VECTOR_CONTEXT, ), _definition( "vmm_vha_bathymetry_profiles", "VHA bathymetrieprofielen", "authoritative", "Vlaamse Milieumaatschappij", {"zone": "Flanders", "theme": "bathymetry_profiles"}, default_crs="EPSG:31370", default_units="m TAW", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=("Point",), usage_policy=_VECTOR_CONTEXT, ), _definition( "historical_landuse", "Historisch landgebruik", "corroborative", "Historical archive provider", {"scope": "Belgium", "theme": "historical_land_use"}, default_crs="source_specific", default_units="source_specific", usage_policy=_VECTOR_CONTEXT, ), _definition( "spw_geoportail", "SPW Geoportail (bronportaal)", "authoritative", "Service public de Wallonie", {"zone": "Wallonia", "role": "umbrella_catalogue_and_adapter_authority"}, provider_adapter_key="spw_geoportail", source_url="https://geoportail.wallonie.be/catalogue", default_crs="product_specific", default_units="product_specific", geographic_coverage={"zone": "Wallonia"}, usage_policy=_usage_policy( allowed_tasks=("source_catalogue", "reference_context") ), ingest_status="configured", known_limitations=( "This umbrella authority is not a product-level ground-truth source.", "A governed import must use PICC, WALOUS, terrain, flood, orthophoto or another product-specific key when available.", ), ), _definition( "spw_picc", "PICC", "authoritative", "Service public de Wallonie", {"zone": "Wallonia", "theme": "topography_buildings"}, license_name="CC BY 4.0", default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Wallonia"}, expected_geometry_types=( "Polygon", "MultiPolygon", "LineString", "MultiLineString", ), usage_policy=_REGIONAL_BUILDING_LABELS, ), _definition( "urbis", "UrbIS", "authoritative", "Brussels Region", {"zone": "Brussels-Capital Region", "theme": "topography_buildings"}, license_name="CC0", default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Brussels-Capital Region"}, expected_geometry_types=( "Polygon", "MultiPolygon", "LineString", "MultiLineString", ), usage_policy=_REGIONAL_BUILDING_LABELS, ), _definition( "spw_walous_land_cover", "WALOUS landbedekking", "authoritative", "Service public de Wallonie", {"zone": "Wallonia", "theme": "land_cover"}, license_name="CC BY 4.0", default_crs="EPSG:3812", default_units="class_code", spatial_resolution={"metres": 1}, geographic_coverage={"zone": "Wallonia"}, usage_policy=_VECTOR_CONTEXT, ), _definition( "spw_bathymetry", "SPW bathymetrie", "authoritative", "Service public de Wallonie", {"zone": "Wallonia", "theme": "bathymetry"}, default_crs="EPSG:3812", default_units="mDNG", geographic_coverage={"zone": "Wallonia"}, usage_policy=_VECTOR_CONTEXT, ), _definition( "spw_terrain", "SPW terreinmodel", "corroborative", "Service public de Wallonie", {"zone": "Wallonia", "theme": "terrain"}, default_crs="EPSG:3812", default_units="metres", spatial_resolution={"metres": 1}, geographic_coverage={"zone": "Wallonia"}, usage_policy=_VECTOR_CONTEXT, ), _definition( "spw_flood_hazard", "SPW overstromingsgevaar", "authoritative", "Service public de Wallonie", {"zone": "Wallonia", "theme": "flood_hazard"}, default_crs="EPSG:3812", default_units="metres", geographic_coverage={"zone": "Wallonia"}, usage_policy=_VECTOR_CONTEXT, ), _definition( "mdk_bathymetry", "MDK bathymetrie", "authoritative", "Maritieme Dienstverlening en Kust", {"zone": "Belgian North Sea", "theme": "bathymetry"}, default_crs="EPSG:3812", default_units="metres", geographic_coverage={"zone": "Belgian North Sea"}, usage_policy=_VECTOR_CONTEXT, ), _definition( "mdk_bcp_bathymetry", "MDK BCP bathymetrie-probe en verwerving", "authoritative", "Maritieme Dienstverlening en Kust", { "zone": "Belgian North Sea", "theme": "bathymetry", "role": "coverage_probe_and_governed_acquisition", }, provider_adapter_key="mdk_bcp_bathymetry", source_url="https://www.vlaanderen.be/datavindplaats", default_crs="EPSG:3812", default_units="metres", geographic_coverage={"zone": "Belgian North Sea"}, usage_policy=_VECTOR_CONTEXT, ingest_status="not_configured", known_limitations=( "A BCP coverage probe is discovery evidence, not a usable bathymetry dataset.", "Only a successful governed acquisition with an immutable response checksum may create a source snapshot.", ), ), ) } class SourceRegistryService: """Fail-closed registry operations for governed importers and validators. All methods intentionally flush but do not commit. The caller owns the dataset/import transaction, so a source snapshot, validation result and dataset write can be rolled back together. """ @staticmethod def normalize_source_key(source_key: str) -> str: normalized = source_key.strip().lower() if not normalized or len(normalized) > 120: raise AppError( code="SOURCE_REGISTRY_KEY_INVALID", message="Source registry key must be a non-empty value up to 120 characters", status_code=422, ) return normalized @classmethod def definition_for(cls, source_key: str) -> SourceRegistryDefinition: normalized = cls.normalize_source_key(source_key) definition = SERVER_OWNED_SOURCE_DEFINITIONS.get(normalized) if definition is None: raise AppError( code="SOURCE_REGISTRY_ENTRY_NOT_FOUND", message="Source is not registered as a server-owned source", details={"source_key": normalized}, status_code=422, ) return definition @classmethod def ensure_server_owned_source(cls, db: Session, source_key: str) -> SourceRegistry: definition = cls.definition_for(source_key) existing = ( db.query(SourceRegistry) .filter(SourceRegistry.source_key == definition.source_key) .one_or_none() ) if existing is not None: return existing source = SourceRegistry(**definition.as_model_values()) db.add(source) db.flush() return source @staticmethod def normalize_ingest_key(ingest_key: str) -> str: normalized = ingest_key.strip() if not normalized or len(normalized) > 255: raise AppError( code="INGEST_KEY_INVALID", message="Ingest key must be a non-empty value up to 255 characters", status_code=422, ) return normalized @classmethod def find_dataset_by_ingest_key( cls, db: Session, project_id: UUID, ingest_key: str ) -> Dataset | None: normalized = cls.normalize_ingest_key(ingest_key) return ( db.query(Dataset) .filter(Dataset.project_id == project_id, Dataset.ingest_key == normalized) .one_or_none() ) @classmethod def find_dataset_version_by_ingest_key( cls, db: Session, dataset_id: UUID, ingest_key: str, ) -> DatasetVersion | None: normalized = cls.normalize_ingest_key(ingest_key) return ( db.query(DatasetVersion) .filter( DatasetVersion.dataset_id == dataset_id, DatasetVersion.ingest_key == normalized, ) .one_or_none() ) @staticmethod def _validate_status(value: str, allowed: set[str], field_name: str) -> str: normalized = value.strip().lower() if normalized not in allowed: raise AppError( code="SOURCE_REGISTRY_STATUS_INVALID", message=f"Unsupported {field_name}", details={ "field": field_name, "value": value, "allowed": sorted(allowed), }, status_code=422, ) return normalized @staticmethod def _validate_checksum(checksum_sha256: str) -> str: normalized = checksum_sha256.strip().lower() if not _CHECKSUM.fullmatch(normalized): raise AppError( code="SOURCE_SNAPSHOT_CHECKSUM_INVALID", message="Source snapshot checksum must be a SHA-256 hex digest", status_code=422, ) return normalized @classmethod def record_snapshot( cls, db: Session, *, source_key: str, snapshot_key: str, checksum_sha256: str, source_version: str | None = None, snapshot_at: datetime | None = None, fetched_at: datetime | None = None, source_url: str | None = None, crs: str | None = None, units: str | None = None, spatial_resolution: dict[str, Any] | None = None, temporal_coverage: dict[str, Any] | None = None, geographic_coverage: dict[str, Any] | None = None, observed_schema: dict[str, Any] | None = None, freshness_status: str = "unknown", ingest_status: str = "ingested", known_limitations: list[str] | None = None, snapshot_metadata: dict[str, Any] | None = None, reuse_existing_snapshot: bool = False, ) -> SourceSnapshot: """Record immutable source evidence, or reuse an identical snapshot. ``fetched_at`` belongs to the immutable source snapshot, whereas a Dataset's ``imported_at`` records each local ingestion event. A governed importer may therefore replay an already-known source snapshot for another project. In that narrow replay mode the existing ``fetched_at`` is retained; every other evidence field is still required to be identical and the row is never updated. """ normalized_key = snapshot_key.strip() if not normalized_key or len(normalized_key) > 255: raise AppError( code="SOURCE_SNAPSHOT_KEY_INVALID", message="Snapshot key must be a non-empty value up to 255 characters", status_code=422, ) source = cls.ensure_server_owned_source(db, source_key) checksum = cls._validate_checksum(checksum_sha256) normalized_freshness = cls._validate_status( freshness_status, _FRESHNESS_STATUSES, "freshness_status" ) normalized_ingest = cls._validate_status( ingest_status, _SNAPSHOT_INGEST_STATUSES, "ingest_status" ) existing = ( db.query(SourceSnapshot) .filter( SourceSnapshot.source_registry_id == source.id, SourceSnapshot.snapshot_key == normalized_key, ) .one_or_none() ) if existing is not None: immutable_values = { "checksum_sha256": checksum, "source_version": source_version.strip() if source_version else None, "snapshot_at": snapshot_at, "source_url": source_url.strip() if source_url else None, "crs": crs.strip() if crs else None, "units": units.strip() if units else None, "spatial_resolution_json": ( dict(spatial_resolution or {"status": "unknown"}) if spatial_resolution is not None else None ), "temporal_coverage_json": ( dict(temporal_coverage or {"status": "unknown"}) if temporal_coverage is not None else None ), "geographic_coverage_json": ( dict(geographic_coverage or {"status": "unknown"}) if geographic_coverage is not None else None ), "observed_schema_json": ( dict(observed_schema or {"status": "unknown"}) if observed_schema is not None else None ), "known_limitations_json": list(known_limitations) if known_limitations is not None else None, "snapshot_metadata_json": dict(snapshot_metadata) if snapshot_metadata is not None else None, } if not reuse_existing_snapshot: immutable_values["fetched_at"] = fetched_at conflicts = { field_name: { "existing": getattr(existing, field_name), "incoming": incoming, } for field_name, incoming in immutable_values.items() if incoming is not None and getattr(existing, field_name) != incoming } if conflicts: raise AppError( code="SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT", message="Existing source snapshot key has different immutable evidence", details={ "source_key": source.source_key, "snapshot_key": normalized_key, "conflicting_fields": sorted(conflicts), }, status_code=409, ) return existing snapshot = SourceSnapshot( source_registry_id=source.id, snapshot_key=normalized_key, source_version=source_version.strip() if source_version else None, snapshot_at=snapshot_at, fetched_at=fetched_at, source_url=source_url.strip() if source_url else None, checksum_sha256=checksum, crs=crs.strip() if crs else None, units=units.strip() if units else None, spatial_resolution_json=dict(spatial_resolution or {"status": "unknown"}), temporal_coverage_json=dict(temporal_coverage or {"status": "unknown"}), geographic_coverage_json=dict(geographic_coverage or {"status": "unknown"}), observed_schema_json=dict(observed_schema or {"status": "unknown"}), freshness_status=normalized_freshness, ingest_status=normalized_ingest, known_limitations_json=list(known_limitations or []), snapshot_metadata_json=dict(snapshot_metadata or {}), ) db.add(snapshot) db.flush() return snapshot @classmethod def bind_dataset_provenance( cls, dataset: Dataset, *, source: SourceRegistry, snapshot: SourceSnapshot, data_contract_key: str, data_contract_version: str, validation_status: str, provenance_status: str, lineage_status: str, ) -> Dataset: cls._validate_binding( source=source, snapshot=snapshot, data_contract_key=data_contract_key, data_contract_version=data_contract_version, validation_status=validation_status, provenance_status=provenance_status, lineage_status=lineage_status, ) dataset.source_registry_id = source.id dataset.source_snapshot_id = snapshot.id dataset.data_contract_key = data_contract_key.strip() dataset.data_contract_version = data_contract_version.strip() dataset.validation_status = validation_status.strip().lower() dataset.provenance_status = provenance_status.strip().lower() dataset.lineage_status = lineage_status.strip().lower() return dataset @classmethod def bind_dataset_version_provenance( cls, dataset_version: DatasetVersion, *, source: SourceRegistry, snapshot: SourceSnapshot, data_contract_key: str, data_contract_version: str, validation_status: str, provenance_status: str, lineage_status: str, ) -> DatasetVersion: cls._validate_binding( source=source, snapshot=snapshot, data_contract_key=data_contract_key, data_contract_version=data_contract_version, validation_status=validation_status, provenance_status=provenance_status, lineage_status=lineage_status, ) dataset_version.source_registry_id = source.id dataset_version.source_snapshot_id = snapshot.id dataset_version.data_contract_key = data_contract_key.strip() dataset_version.data_contract_version = data_contract_version.strip() dataset_version.validation_status = validation_status.strip().lower() dataset_version.provenance_status = provenance_status.strip().lower() dataset_version.lineage_status = lineage_status.strip().lower() return dataset_version @classmethod def _validate_binding( cls, *, source: SourceRegistry, snapshot: SourceSnapshot, data_contract_key: str, data_contract_version: str, validation_status: str, provenance_status: str, lineage_status: str, ) -> None: if source.id != snapshot.source_registry_id: raise AppError( code="SOURCE_SNAPSHOT_REGISTRY_MISMATCH", message="Source snapshot does not belong to the selected source registry entry", status_code=409, ) if not data_contract_key.strip() or not data_contract_version.strip(): raise AppError( code="DATA_CONTRACT_IDENTITY_REQUIRED", message="Dataset provenance binding requires a contract key and version", status_code=422, ) cls._validate_status( validation_status, _VALIDATION_STATUSES, "validation_status" ) cls._validate_status( provenance_status, _PROVENANCE_STATUSES, "provenance_status" ) cls._validate_status(lineage_status, _LINEAGE_STATUSES, "lineage_status") @staticmethod def record_lineage_edge( db: Session, *, parent_dataset_id: UUID, child_dataset_id: UUID, relation_type: str, transformation_name: str, parent_dataset_version_id: UUID | None = None, child_dataset_version_id: UUID | None = None, transformation_version: str | None = None, parameters: dict[str, Any] | None = None, input_checksum_sha256: str | None = None, output_checksum_sha256: str | None = None, ) -> DatasetLineageEdge: if parent_dataset_id == child_dataset_id: raise AppError( code="DATASET_LINEAGE_SELF_REFERENCE", message="A dataset cannot be its own lineage parent", status_code=422, ) normalized_relation = relation_type.strip() normalized_transform = transformation_name.strip() if not normalized_relation or not normalized_transform: raise AppError( code="DATASET_LINEAGE_IDENTITY_REQUIRED", message="Lineage relation type and transformation name are required", status_code=422, ) normalized_input_checksum = SourceRegistryService._optional_checksum( input_checksum_sha256 ) normalized_output_checksum = SourceRegistryService._optional_checksum( output_checksum_sha256 ) existing = ( db.query(DatasetLineageEdge) .filter( DatasetLineageEdge.parent_dataset_id == parent_dataset_id, DatasetLineageEdge.child_dataset_id == child_dataset_id, DatasetLineageEdge.relation_type == normalized_relation, DatasetLineageEdge.transformation_name == normalized_transform, ) .one_or_none() ) if existing is not None: if ( existing.input_checksum_sha256 != normalized_input_checksum or existing.output_checksum_sha256 != normalized_output_checksum ): raise AppError( code="DATASET_LINEAGE_IMMUTABILITY_CONFLICT", message="Existing lineage edge has different artifact checksums", status_code=409, ) return existing if SourceRegistryService._would_create_lineage_cycle( db, parent_dataset_id=parent_dataset_id, child_dataset_id=child_dataset_id, ): raise AppError( code="DATASET_LINEAGE_CYCLE_DETECTED", message="The proposed lineage edge would make the dataset lineage graph cyclic", details={ "parent_dataset_id": str(parent_dataset_id), "child_dataset_id": str(child_dataset_id), }, status_code=409, ) edge = DatasetLineageEdge( parent_dataset_id=parent_dataset_id, child_dataset_id=child_dataset_id, parent_dataset_version_id=parent_dataset_version_id, child_dataset_version_id=child_dataset_version_id, relation_type=normalized_relation, transformation_name=normalized_transform, transformation_version=transformation_version.strip() if transformation_version else None, parameters_json=dict(parameters or {}), input_checksum_sha256=normalized_input_checksum, output_checksum_sha256=normalized_output_checksum, ) db.add(edge) db.flush() return edge @staticmethod def _would_create_lineage_cycle( db: Session, *, parent_dataset_id: UUID, child_dataset_id: UUID, ) -> bool: """Return whether ``parent -> child`` would close an existing DAG path. A lineage edge is directed from an input/parent Dataset to its derived child. Adding ``parent -> child`` is unsafe precisely when ``parent`` is already reachable downstream from ``child``. Querying one indexed parent frontier at a time avoids loading unrelated lineage history and the visited set makes a pre-existing corrupt cycle finite to inspect. The graph-size limit is itself fail-closed: a graph too large to audit may not receive a new edge until it is investigated. """ frontier = {child_dataset_id} visited: set[UUID] = set() while frontier: if parent_dataset_id in frontier: return True current = frontier - visited if not current: return False visited.update(current) if len(visited) > _MAX_LINEAGE_GRAPH_NODES: raise AppError( code="DATASET_LINEAGE_GRAPH_LIMIT_EXCEEDED", message="Dataset lineage graph exceeds the safe traversal limit", details={"max_nodes": _MAX_LINEAGE_GRAPH_NODES}, status_code=409, ) edges = ( db.query(DatasetLineageEdge) .filter(DatasetLineageEdge.parent_dataset_id.in_(current)) .all() ) frontier = { edge.child_dataset_id for edge in edges if edge.child_dataset_id not in visited } return False @classmethod def _lineage_descendant_dataset_ids( cls, db: Session, *, root_dataset_ids: set[UUID], ) -> set[UUID]: """Return every reachable child Dataset, including the supplied roots. A quarantine is a lineage safety event, not merely a status update on the immediately observed asset. Traversal follows the same directed parent-to-child relation used by the cycle guard and remains bounded so a corrupt graph cannot make a quarantine operation unobservable. PostgreSQL enforces the equivalent recursive propagation for all persisted writes; this application-side traversal keeps the service fail-closed for normal ORM callers and focused in-memory test doubles. """ affected = set(root_dataset_ids) frontier = set(root_dataset_ids) visited: set[UUID] = set() while frontier: current = frontier - visited if not current: break visited.update(current) if len(visited) > _MAX_LINEAGE_GRAPH_NODES: raise AppError( code="DATASET_LINEAGE_GRAPH_LIMIT_EXCEEDED", message="Dataset lineage graph exceeds the safe traversal limit during quarantine propagation", details={"max_nodes": _MAX_LINEAGE_GRAPH_NODES}, status_code=409, ) edges = ( db.query(DatasetLineageEdge) .filter(DatasetLineageEdge.parent_dataset_id.in_(current)) .all() ) frontier = { edge.child_dataset_id for edge in edges if edge.child_dataset_id not in visited } affected.update(frontier) return affected @staticmethod def _mark_dataset_quarantined(dataset: Dataset) -> None: """Invalidate every Dataset-level consumption gate in one place.""" dataset.status = "quarantined" dataset.quarantine_status = "quarantined" dataset.validation_status = "failed" dataset.provenance_status = "incomplete" dataset.lineage_status = "incomplete" @staticmethod def _mark_dataset_version_quarantined(dataset_version: DatasetVersion) -> None: """Invalidate a version that belongs to a quarantined Dataset lineage.""" dataset_version.validation_status = "failed" dataset_version.provenance_status = "incomplete" dataset_version.lineage_status = "incomplete" @classmethod def _propagate_lineage_quarantine( cls, db: Session, *, root_datasets: tuple[Dataset, ...], ) -> tuple[Dataset, ...]: """Quarantine root datasets and all immutable downstream derivatives. A source snapshot can be bound by more than one Dataset, so callers supply every directly affected root. Derived descendants retain their own source snapshot evidence, but their Dataset and DatasetVersion state becomes non-consumable until a governed re-ingest establishes a new valid lineage. """ datasets_by_id: dict[UUID, Dataset] = {} transient_roots: list[Dataset] = [] for candidate in root_datasets: dataset_id = getattr(candidate, "id", None) if dataset_id is None: transient_roots.append(candidate) else: datasets_by_id[dataset_id] = candidate affected_ids = ( cls._lineage_descendant_dataset_ids( db, root_dataset_ids=set(datasets_by_id), ) if datasets_by_id else set() ) if affected_ids: for candidate in ( db.query(Dataset).filter(Dataset.id.in_(affected_ids)).all() ): datasets_by_id[candidate.id] = candidate affected_datasets = tuple((*datasets_by_id.values(), *transient_roots)) for candidate in affected_datasets: cls._mark_dataset_quarantined(candidate) if affected_ids: for dataset_version in ( db.query(DatasetVersion) .filter(DatasetVersion.dataset_id.in_(affected_ids)) .all() ): cls._mark_dataset_version_quarantined(dataset_version) return affected_datasets @classmethod def quarantine_dataset( cls, db: Session, *, stage: str, reason_code: str, dataset: Dataset | None = None, dataset_version: DatasetVersion | None = None, source_snapshot: SourceSnapshot | None = None, details: dict[str, Any] | None = None, artifact_path: str | None = None, artifact_checksum_sha256: str | None = None, ) -> DatasetQuarantine: normalized_stage = stage.strip() normalized_reason = reason_code.strip() if not normalized_stage or not normalized_reason: raise AppError( code="QUARANTINE_REASON_REQUIRED", message="Quarantine stage and reason code are required", status_code=422, ) if dataset is None and dataset_version is None and source_snapshot is None: raise AppError( code="QUARANTINE_TARGET_REQUIRED", message="Quarantine requires a dataset, dataset version or source snapshot", status_code=422, ) if dataset is None and dataset_version is not None: dataset = cls._dataset_for_version(db, dataset_version) if dataset is None: # A version-only quarantine without the owning Dataset would # leave the Dataset consumable: every current consumption # boundary evaluates Dataset, not DatasetVersion. Refuse the # partial state rather than quietly recording an ineffective # quarantine. raise AppError( code="QUARANTINE_PARENT_DATASET_NOT_FOUND", message="A version quarantine requires its owning Dataset so the quarantine can propagate.", details={ "dataset_version_id": str(dataset_version.id), "dataset_id": str(dataset_version.dataset_id), }, status_code=409, ) root_datasets: list[Dataset] = [dataset] if dataset is not None else [] if source_snapshot is not None: # A snapshot is immutable shared evidence. Its direct bindings are # roots too, so invalidating one artifact cannot leave a sibling or # any downstream derivative consumable through a stale lineage. root_datasets.extend( db.query(Dataset) .filter(Dataset.source_snapshot_id == source_snapshot.id) .all() ) cls._propagate_lineage_quarantine(db, root_datasets=tuple(root_datasets)) if dataset_version is not None: cls._mark_dataset_version_quarantined(dataset_version) if source_snapshot is not None: # A quarantined source artifact must no longer satisfy the # authoritative-validation eligibility check for any linked data. source_snapshot.ingest_status = "quarantined" record = DatasetQuarantine( dataset_id=dataset.id if dataset is not None else None, dataset_version_id=dataset_version.id if dataset_version is not None else None, source_snapshot_id=source_snapshot.id if source_snapshot is not None else None, stage=normalized_stage, reason_code=normalized_reason, details_json=dict(details or {}), artifact_path=artifact_path, artifact_checksum_sha256=cls._optional_checksum(artifact_checksum_sha256), status="quarantined", ) db.add(record) db.flush() return record @staticmethod def _dataset_for_version( db: Session, dataset_version: DatasetVersion ) -> Dataset | None: """Resolve the Dataset that must share a version's quarantine state.""" related = getattr(dataset_version, "dataset", None) if related is not None: return related getter = getattr(db, "get", None) if callable(getter): resolved = getter(Dataset, dataset_version.dataset_id) if resolved is not None: return resolved # Use a normal ORM query as a final path for sessions where the # relationship is deliberately not loaded. This also keeps bounded # in-memory persistence fixtures representative of production. return ( db.query(Dataset) .filter(Dataset.id == dataset_version.dataset_id) .one_or_none() ) @staticmethod def validation_authority_for_task(source: SourceRegistry, task: str) -> str | None: policy = ( source.usage_policy_json if isinstance(source.usage_policy_json, dict) else {} ) authority = policy.get("validation_authority") if not isinstance(authority, dict): return None value = authority.get(task) return str(value) if value else None @classmethod def is_dataset_eligible_for_authoritative_validation( cls, dataset: Dataset, *, source: SourceRegistry, snapshot: SourceSnapshot, task: str, ) -> bool: policy = ( source.usage_policy_json if isinstance(source.usage_policy_json, dict) else {} ) return bool( source.classification == "authoritative" and policy.get("ground_truth_allowed") is True and cls.validation_authority_for_task(source, task) == "primary" and dataset.status == "ready" and dataset.quarantine_status == "not_quarantined" and dataset.validation_status == "passed" and dataset.provenance_status == "complete" and dataset.lineage_status in {"complete", "not_applicable"} and dataset.source_registry_id == source.id and dataset.source_snapshot_id == snapshot.id and snapshot.source_registry_id == source.id and snapshot.ingest_status == "ingested" and bool( snapshot.checksum_sha256 and _CHECKSUM.fullmatch(snapshot.checksum_sha256) ) ) @classmethod def _optional_checksum(cls, checksum_sha256: str | None) -> str | None: if checksum_sha256 is None: return None return cls._validate_checksum(checksum_sha256)