"""Add server-owned source registry, source snapshots and dataset provenance gates. Legacy datasets are retained untouched. They receive an identity link when one is available, but remain explicitly incomplete/not_validated until a governed ingest writes an immutable source snapshot and contract result. """ from __future__ import annotations import uuid import json from alembic import op import sqlalchemy as sa revision = "202608010001" down_revision = "202607260001" branch_labels = None depends_on = None _SOURCE_NAMESPACE = uuid.UUID("6e5f9c11-bc7f-4f5f-ae23-f3eddb4d0149") _DEFAULT_LICENSE = "Provider terms must be verified for each immutable snapshot." _DEFAULT_LIMITATION = "No authority, ground-truth, freshness or training claim is allowed without a governed snapshot and passed contract." _DEFAULT_USAGE_POLICY = { "automatic_ground_truth": False, "ground_truth_allowed": False, "training_allowed": False, "allowed_tasks": [], "validation_authority": {}, } def _source_seed( source_key: str, display_name: str, classification: str, authority_name: str, *, authority_scope: dict | None = None, provider_adapter_key: str | None = None, source_url: str | None = None, license_name: str = _DEFAULT_LICENSE, usage_restrictions: str = "Use only according to the source-specific snapshot terms and attribution.", default_crs: str = "unknown", default_units: str = "unknown", spatial_resolution: dict | None = None, temporal_coverage: dict | None = None, geographic_coverage: dict | None = None, expected_geometry_types: list[str] | None = None, expected_attributes: dict | None = None, usage_policy: dict | None = None, freshness_status: str = "unknown", ingest_status: str = "registered", known_limitations: list[str] | None = None, ) -> dict: return { "id": uuid.uuid5(_SOURCE_NAMESPACE, source_key), "source_key": source_key, "display_name": display_name, "classification": classification, "authority_name": authority_name, "authority_scope_json": authority_scope or {"status": "unknown"}, "provider_adapter_key": provider_adapter_key, "source_url": source_url, "license_name": license_name, "license_url": None, "usage_restrictions": usage_restrictions, "default_crs": default_crs, "default_units": default_units, "spatial_resolution_json": spatial_resolution or {"status": "unknown"}, "temporal_coverage_json": temporal_coverage or {"status": "unknown"}, "geographic_coverage_json": geographic_coverage or {"status": "unknown"}, "expected_geometry_types_json": expected_geometry_types or [], "expected_attributes_json": expected_attributes or {"status": "unknown"}, "usage_policy_json": usage_policy or dict(_DEFAULT_USAGE_POLICY), "freshness_status": freshness_status, "ingest_status": ingest_status, "known_limitations_json": known_limitations or [_DEFAULT_LIMITATION], "registry_metadata_json": {"registry_owner": "server", "seeded_by": revision}, } def _seed_rows() -> list[dict]: governed_vector_policy = { "automatic_ground_truth": False, "ground_truth_allowed": False, "training_allowed": False, "allowed_tasks": ["reference_context"], "validation_authority": {}, } governed_imagery_policy = { "automatic_ground_truth": False, "ground_truth_allowed": False, "training_allowed": True, "allowed_tasks": ["imagery", "training_input", "visual_context"], "validation_authority": {}, } regional_building_policy = { "automatic_ground_truth": False, "ground_truth_allowed": True, "training_allowed": True, "allowed_tasks": ["building_validation", "building_labels"], "validation_authority": {"building_validation": "regional_primary_pending_contract"}, } return [ _source_seed( "grb", "Grootschalig Referentie Bestand", "authoritative", "Digitaal Vlaanderen", authority_scope={"zone": "Flanders", "themes": ["buildings", "roads", "water", "parcels"]}, provider_adapter_key="grb", source_url="https://www.vlaanderen.be/datavindplaats/catalogus/basiskaart-vlaanderen-grb", usage_restrictions="Use governed GRB acquisition evidence and retain Digitaal Vlaanderen attribution.", default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=["Polygon", "MultiPolygon", "LineString", "MultiLineString"], expected_attributes={"required": ["id"], "layers": ["GBG", "Wegsegment", "WTZ", "WLAS", "WGR", "ADP"]}, usage_policy={ "automatic_ground_truth": False, "ground_truth_allowed": True, "training_allowed": True, "allowed_tasks": ["building_validation", "building_labels", "reference_context"], "validation_authority": {"building_validation": "primary"}, }, ingest_status="configured", known_limitations=[ "GRB building geometry is authoritative only for a governed, versioned snapshot within Flanders.", "GRB does not establish imagery-time alignment or national model validation by itself.", ], ), _source_seed( "digitaal_vlaanderen", "Digitaal Vlaanderen (bronportaal)", "authoritative", "Digitaal Vlaanderen", authority_scope={"zone": "Flanders", "role": "umbrella_catalogue_and_adapter_authority"}, provider_adapter_key="digitaal_vlaanderen", source_url="https://www.vlaanderen.be/datavindplaats", default_crs="product_specific", default_units="product_specific", geographic_coverage={"zone": "Flanders"}, usage_policy={ "automatic_ground_truth": False, "ground_truth_allowed": False, "training_allowed": False, "allowed_tasks": ["source_catalogue", "reference_context"], "validation_authority": {}, }, ingest_status="configured", known_limitations=[ "This umbrella authority is not a product-level ground-truth source.", "A governed import must use a product-specific source key whenever one is available.", ], ), _source_seed( "digitaal_vlaanderen_buildings_addresses_register", "Gebouwen- en adressenregister", "authoritative", "Digitaal Vlaanderen", authority_scope={"zone": "Flanders", "theme": "buildings_addresses"}, provider_adapter_key="buildings_addresses_register", default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=["Point", "Polygon", "MultiPolygon"], expected_attributes={"required": ["id"], "role": "administrative_corroboration"}, usage_policy={ "automatic_ground_truth": False, "ground_truth_allowed": False, "training_allowed": False, "allowed_tasks": ["building_validation", "address_corroboration", "building_register_validation"], "validation_authority": { "building_validation": "corroborative", "building_register_validation": "primary", }, }, known_limitations=["Administrative building/address records do not replace a governed reference-footprint contract."], ), _source_seed( "sentinel_2", "Sentinel-2", "contextual", "Copernicus Programme", authority_scope={"scope": "Belgium and Belgian North Sea", "role": "multispectral_context"}, provider_adapter_key="sentinel_2", default_crs="product_specific", default_units="reflectance", spatial_resolution={"metres": [10, 20, 60]}, temporal_coverage={"cadence_days": 5, "status": "product_specific"}, geographic_coverage={"scope": "Belgium and Belgian North Sea"}, expected_geometry_types=[], expected_attributes={"required": ["product_id", "sensing_time"]}, usage_policy={ "automatic_ground_truth": False, "ground_truth_allowed": False, "training_allowed": True, "allowed_tasks": ["imagery_context", "change_context"], "validation_authority": {}, }, ingest_status="not_configured", known_limitations=["Sentinel-2 is contextual imagery, not automatic ground truth for building labels."], ), _source_seed( "digitaal_vlaanderen_dhmv", "Digitaal Hoogtemodel Vlaanderen", "authoritative", "Digitaal Vlaanderen", authority_scope={"zone": "Flanders", "role": "terrain_height_corroboration"}, provider_adapter_key="dhmv", default_crs="EPSG:31370", default_units="m TAW", spatial_resolution={"metres": 1}, temporal_coverage={"period": "2013-2015", "status": "product_specific"}, geographic_coverage={"zone": "Flanders"}, expected_attributes={"bands": 1, "nodata_required": True}, usage_policy={ "automatic_ground_truth": False, "ground_truth_allowed": False, "training_allowed": True, "allowed_tasks": ["terrain_context", "height_corroboration", "elevation_validation"], "validation_authority": { "building_validation": "corroborative", "elevation_validation": "primary", }, }, ingest_status="configured", known_limitations=["DHMV is height context and cannot independently establish building labels."], ), _source_seed( "osm", "OpenStreetMap", "contextual", "OpenStreetMap contributors", authority_scope={"scope": "community-maintained", "role": "contextual"}, provider_adapter_key="osm", source_url="https://www.openstreetmap.org", license_name="ODbL", usage_restrictions="OpenStreetMap attribution and ODbL obligations apply.", default_crs="EPSG:4326", default_units="mixed", geographic_coverage={"scope": "global"}, expected_geometry_types=["Point", "LineString", "Polygon", "MultiPolygon"], expected_attributes={"status": "community_tags"}, usage_policy={ "automatic_ground_truth": False, "ground_truth_allowed": False, "training_allowed": False, "allowed_tasks": ["context", "candidate_discovery"], "validation_authority": {}, }, ingest_status="not_configured", known_limitations=["OSM is never automatic ground truth for GeoIntel validation or labels."], ), _source_seed( "manual", "Handmatige upload", "experimental", "Operator supplied", authority_scope={"scope": "operator supplied", "trust": "unverified"}, default_crs="unknown", default_units="unknown", usage_policy=dict(_DEFAULT_USAGE_POLICY), ingest_status="configured", known_limitations=["Manual uploads remain untrusted until a passed contract and governed provenance are attached."], ), _source_seed( "fixture", "Test- en demo fixture", "experimental", "GeoIntel test fixture", authority_scope={"scope": "test_only"}, default_crs="fixture_specific", default_units="fixture_specific", usage_policy=dict(_DEFAULT_USAGE_POLICY), ingest_status="configured", known_limitations=["Fixtures must never be presented as official data or used for production training/promotion."], ), _source_seed( "map_selection", "Afgeleide kaartselectie", "derived", "GeoIntel derived operation", authority_scope={"scope": "derived_from_registered_input"}, default_crs="EPSG:4326", default_units="source_dependent", usage_policy=dict(_DEFAULT_USAGE_POLICY), known_limitations=["Derived selections inherit no authority beyond their complete lineage edge and source snapshot."], ), _source_seed( "derived", "Afgeleide dataset", "derived", "GeoIntel derived operation", authority_scope={"scope": "derived_from_registered_input"}, usage_policy=dict(_DEFAULT_USAGE_POLICY), ), _source_seed( "training_label", "Afgeleide trainingslabels", "derived", "GeoIntel reviewed label pipeline", authority_scope={"scope": "derived_from_reviewed_source_snapshots"}, usage_policy=dict(_DEFAULT_USAGE_POLICY), known_limitations=["Training labels require complete source lineage and human-review evidence; they inherit no automatic authority."], ), _source_seed( "model", "Model artifact", "experimental", "GeoIntel model pipeline", authority_scope={"scope": "internal_model_artifact"}, usage_policy=dict(_DEFAULT_USAGE_POLICY), known_limitations=["A model artifact is not a validated capability or promotion decision without its model card and evaluation evidence."], ), _source_seed( "experimental", "Experimentele bron", "experimental", "Unverified", authority_scope={"scope": "unverified"}, usage_policy=dict(_DEFAULT_USAGE_POLICY), ), _source_seed( "legacy_unknown", "Niet-geclassificeerde historische bron", "experimental", "Legacy import — unverified", authority_scope={"scope": "legacy", "trust": "unverified"}, usage_policy=dict(_DEFAULT_USAGE_POLICY), ingest_status="legacy_unverified", known_limitations=["Historical source identity is retained descriptively but has no governed authority until re-ingested."], ), _source_seed("ngi_adminvector", "NGI AdminVector", "authoritative", "Nationaal Geografisch Instituut", authority_scope={"scope": "Belgium"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"scope": "Belgium"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy), _source_seed("rbins_marine_reporting_units", "RBINS mariene rapportage-eenheden", "authoritative", "RBINS", authority_scope={"zone": "Belgian North Sea"}, default_crs="EPSG:4326", default_units="degrees", geographic_coverage={"zone": "Belgian North Sea"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy), _source_seed("rbins_msp_2026", "Belgisch Marien Ruimtelijk Plan 2026-2034", "authoritative", "RBINS", authority_scope={"zone": "Belgian North Sea", "edition": "2026-2034"}, default_crs="EPSG:4326", default_units="degrees", geographic_coverage={"zone": "Belgian North Sea"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy), _source_seed("vrbg", "Vlaams Wegenregister", "authoritative", "Digitaal Vlaanderen", authority_scope={"zone": "Flanders", "theme": "roads"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=["LineString", "MultiLineString"], usage_policy=governed_vector_policy), _source_seed("digitaal_vlaanderen_orthophoto", "Orthofoto Vlaanderen", "contextual", "Digitaal Vlaanderen", authority_scope={"zone": "Flanders", "role": "imagery"}, default_crs="EPSG:31370", default_units="pixel", spatial_resolution={"metres": 0.25}, geographic_coverage={"zone": "Flanders"}, usage_policy=governed_imagery_policy, ingest_status="configured"), _source_seed("spw_orthophoto", "Orthofoto Wallonië", "contextual", "Service public de Wallonie", authority_scope={"zone": "Wallonia", "role": "imagery"}, license_name="CC BY 4.0", default_crs="EPSG:31370", default_units="pixel", spatial_resolution={"metres": 0.25}, geographic_coverage={"zone": "Wallonia"}, usage_policy=governed_imagery_policy, ingest_status="configured"), _source_seed("urbis_orthophoto", "Orthofoto Brussel", "contextual", "UrbIS / Brussels Region", authority_scope={"zone": "Brussels-Capital Region", "role": "imagery"}, license_name="CC0", default_crs="EPSG:31370", default_units="pixel", spatial_resolution={"metres": 0.25}, geographic_coverage={"zone": "Brussels-Capital Region"}, usage_policy=governed_imagery_policy, ingest_status="configured"), _source_seed("agentschap_landbouw_zeevisserij_agricultural_parcels", "Landbouwgebruikspercelen", "authoritative", "Agentschap Landbouw en Zeevisserij", authority_scope={"zone": "Flanders", "theme": "agricultural_parcels"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy), _source_seed("department_omgeving_land_use", "Landgebruik Vlaanderen", "authoritative", "Departement Omgeving", authority_scope={"zone": "Flanders", "theme": "land_use"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy), _source_seed("inbo_bwk_natura2000", "BWK en Natura 2000", "authoritative", "INBO", authority_scope={"zone": "Flanders", "theme": "nature"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy), _source_seed("statbel", "Statbel bevolking", "authoritative", "Statbel", authority_scope={"scope": "Belgium", "theme": "population"}, default_crs="EPSG:31370", default_units="persons", geographic_coverage={"scope": "Belgium"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy), _source_seed("waterinfo", "Waterinfo", "authoritative", "Waterinfo Vlaanderen", authority_scope={"zone": "Flanders", "theme": "water"}, default_crs="EPSG:31370", default_units="source_specific", geographic_coverage={"zone": "Flanders"}, usage_policy=governed_vector_policy), _source_seed("department_omgeving_thematic_raster", "Omgeving thematische rasters", "authoritative", "Departement Omgeving", authority_scope={"zone": "Flanders", "theme": "thematic_raster"}, default_crs="EPSG:31370", default_units="source_specific", geographic_coverage={"zone": "Flanders"}, usage_policy=governed_vector_policy), _source_seed("dov_soil_map", "DOV bodemkaart", "authoritative", "Databank Ondergrond Vlaanderen", authority_scope={"zone": "Flanders", "theme": "soil"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy), _source_seed("vmm_flood_hazard", "VMM overstromingskaarten", "authoritative", "Vlaamse Milieumaatschappij", authority_scope={"zone": "Flanders", "theme": "flood_hazard"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, usage_policy=governed_vector_policy), _source_seed("vmm_vha_bathymetry_profiles", "VHA bathymetrieprofielen", "authoritative", "Vlaamse Milieumaatschappij", authority_scope={"zone": "Flanders", "theme": "bathymetry_profiles"}, default_crs="EPSG:31370", default_units="m TAW", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=["Point"], usage_policy=governed_vector_policy), _source_seed("historical_landuse", "Historisch landgebruik", "corroborative", "Historical archive provider", authority_scope={"scope": "Belgium", "theme": "historical_land_use"}, default_crs="source_specific", default_units="source_specific", usage_policy=governed_vector_policy), _source_seed( "spw_geoportail", "SPW Geoportail (bronportaal)", "authoritative", "Service public de Wallonie", authority_scope={"zone": "Wallonia", "role": "umbrella_catalogue_and_adapter_authority"}, provider_adapter_key="spw_geoportail", source_url="https://geoportail.wallonie.be/catalogue", default_crs="product_specific", default_units="product_specific", geographic_coverage={"zone": "Wallonia"}, usage_policy={ "automatic_ground_truth": False, "ground_truth_allowed": False, "training_allowed": False, "allowed_tasks": ["source_catalogue", "reference_context"], "validation_authority": {}, }, ingest_status="configured", known_limitations=[ "This umbrella authority is not a product-level ground-truth source.", "A governed import must use PICC, WALOUS, terrain, flood, orthophoto or another product-specific key when available.", ], ), _source_seed("spw_picc", "PICC", "authoritative", "Service public de Wallonie", authority_scope={"zone": "Wallonia", "theme": "topography_buildings"}, license_name="CC BY 4.0", default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Wallonia"}, expected_geometry_types=["Polygon", "MultiPolygon", "LineString", "MultiLineString"], usage_policy=regional_building_policy), _source_seed("urbis", "UrbIS", "authoritative", "Brussels Region", authority_scope={"zone": "Brussels-Capital Region", "theme": "topography_buildings"}, license_name="CC0", default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Brussels-Capital Region"}, expected_geometry_types=["Polygon", "MultiPolygon", "LineString", "MultiLineString"], usage_policy=regional_building_policy), _source_seed("spw_walous_land_cover", "WALOUS landbedekking", "authoritative", "Service public de Wallonie", authority_scope={"zone": "Wallonia", "theme": "land_cover"}, license_name="CC BY 4.0", default_crs="EPSG:3812", default_units="class_code", spatial_resolution={"metres": 1}, geographic_coverage={"zone": "Wallonia"}, usage_policy=governed_vector_policy), _source_seed("spw_bathymetry", "SPW bathymetrie", "authoritative", "Service public de Wallonie", authority_scope={"zone": "Wallonia", "theme": "bathymetry"}, default_crs="EPSG:3812", default_units="mDNG", geographic_coverage={"zone": "Wallonia"}, usage_policy=governed_vector_policy), _source_seed("spw_terrain", "SPW terreinmodel", "corroborative", "Service public de Wallonie", authority_scope={"zone": "Wallonia", "theme": "terrain"}, default_crs="EPSG:3812", default_units="metres", spatial_resolution={"metres": 1}, geographic_coverage={"zone": "Wallonia"}, usage_policy=governed_vector_policy), _source_seed("spw_flood_hazard", "SPW overstromingsgevaar", "authoritative", "Service public de Wallonie", authority_scope={"zone": "Wallonia", "theme": "flood_hazard"}, default_crs="EPSG:3812", default_units="metres", geographic_coverage={"zone": "Wallonia"}, usage_policy=governed_vector_policy), _source_seed("mdk_bathymetry", "MDK bathymetrie", "authoritative", "Maritieme Dienstverlening en Kust", authority_scope={"zone": "Belgian North Sea", "theme": "bathymetry"}, default_crs="EPSG:3812", default_units="metres", geographic_coverage={"zone": "Belgian North Sea"}, usage_policy=governed_vector_policy), _source_seed( "mdk_bcp_bathymetry", "MDK BCP bathymetrie-probe en verwerving", "authoritative", "Maritieme Dienstverlening en Kust", authority_scope={ "zone": "Belgian North Sea", "theme": "bathymetry", "role": "coverage_probe_and_governed_acquisition", }, provider_adapter_key="mdk_bcp_bathymetry", source_url="https://www.vlaanderen.be/datavindplaats", default_crs="EPSG:3812", default_units="metres", geographic_coverage={"zone": "Belgian North Sea"}, usage_policy=governed_vector_policy, ingest_status="not_configured", known_limitations=[ "A BCP coverage probe is discovery evidence, not a usable bathymetry dataset.", "Only a successful governed acquisition with an immutable response checksum may create a source snapshot.", ], ), ] def _source_registry_table() -> sa.Table: return sa.table( "source_registry", sa.column("id", sa.UUID()), sa.column("source_key", sa.String()), sa.column("display_name", sa.String()), sa.column("classification", sa.String()), sa.column("authority_name", sa.String()), sa.column("authority_scope_json", sa.JSON()), sa.column("provider_adapter_key", sa.String()), sa.column("source_url", sa.Text()), sa.column("license_name", sa.String()), sa.column("license_url", sa.Text()), sa.column("usage_restrictions", sa.Text()), sa.column("default_crs", sa.String()), sa.column("default_units", sa.String()), sa.column("spatial_resolution_json", sa.JSON()), sa.column("temporal_coverage_json", sa.JSON()), sa.column("geographic_coverage_json", sa.JSON()), sa.column("expected_geometry_types_json", sa.JSON()), sa.column("expected_attributes_json", sa.JSON()), sa.column("usage_policy_json", sa.JSON()), sa.column("freshness_status", sa.String()), sa.column("ingest_status", sa.String()), sa.column("known_limitations_json", sa.JSON()), sa.column("registry_metadata_json", sa.JSON()), ) def _offline_safe_seed_rows() -> list[dict]: json_columns = { "authority_scope_json", "spatial_resolution_json", "temporal_coverage_json", "geographic_coverage_json", "expected_geometry_types_json", "expected_attributes_json", "usage_policy_json", "known_limitations_json", "registry_metadata_json", } rows: list[dict] = [] for seed in _seed_rows(): row = dict(seed) row["id"] = op.inline_literal(str(seed["id"]), type_=sa.String()) for column_name in json_columns: row[column_name] = op.inline_literal( json.dumps(seed[column_name], sort_keys=True, separators=(",", ":")), type_=sa.String(), ) rows.append(row) return rows def upgrade() -> None: op.create_table( "source_registry", sa.Column("id", sa.UUID(), primary_key=True), sa.Column("source_key", sa.String(length=120), nullable=False), sa.Column("display_name", sa.String(length=255), nullable=False), sa.Column("classification", sa.String(length=32), nullable=False), sa.Column("authority_name", sa.String(length=255), nullable=False, server_default="unknown"), sa.Column("authority_scope_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), sa.Column("provider_adapter_key", sa.String(length=120), nullable=True), sa.Column("source_url", sa.Text(), nullable=True), sa.Column("license_name", sa.String(length=255), nullable=False, server_default="unknown"), sa.Column("license_url", sa.Text(), nullable=True), sa.Column("usage_restrictions", sa.Text(), nullable=False, server_default="unknown"), sa.Column("default_crs", sa.String(length=64), nullable=False, server_default="unknown"), sa.Column("default_units", sa.String(length=120), nullable=False, server_default="unknown"), sa.Column("spatial_resolution_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), sa.Column("temporal_coverage_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), sa.Column("geographic_coverage_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), sa.Column("expected_geometry_types_json", sa.JSON(), nullable=False, server_default=sa.text("'[]'::json")), sa.Column("expected_attributes_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), sa.Column("usage_policy_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), sa.Column("freshness_status", sa.String(length=32), nullable=False, server_default="unknown"), sa.Column("ingest_status", sa.String(length=32), nullable=False, server_default="registered"), sa.Column("known_limitations_json", sa.JSON(), nullable=False, server_default=sa.text("'[]'::json")), sa.Column("registry_metadata_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()), sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()), sa.UniqueConstraint("source_key", name="uq_source_registry_source_key"), sa.CheckConstraint( "classification IN ('authoritative', 'corroborative', 'contextual', 'derived', 'experimental')", name="ck_source_registry_classification", ), sa.CheckConstraint( "freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')", name="ck_source_registry_freshness_status", ), sa.CheckConstraint( "ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', " "'failed', 'quarantined', 'legacy_unverified')", name="ck_source_registry_ingest_status", ), ) op.create_table( "source_snapshots", sa.Column("id", sa.UUID(), primary_key=True), sa.Column("source_registry_id", sa.UUID(), sa.ForeignKey("source_registry.id", ondelete="CASCADE"), nullable=False), sa.Column("snapshot_key", sa.String(length=255), nullable=False), sa.Column("source_version", sa.String(length=120), nullable=True), sa.Column("snapshot_at", sa.DateTime(timezone=True), nullable=True), sa.Column("fetched_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), sa.Column("source_url", sa.Text(), nullable=True), sa.Column("checksum_sha256", sa.String(length=64), nullable=False), sa.Column("crs", sa.String(length=64), nullable=True), sa.Column("units", sa.String(length=120), nullable=True), sa.Column("spatial_resolution_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), sa.Column("temporal_coverage_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), sa.Column("geographic_coverage_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), sa.Column("observed_schema_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), sa.Column("freshness_status", sa.String(length=32), nullable=False, server_default="unknown"), sa.Column("ingest_status", sa.String(length=32), nullable=False, server_default="registered"), sa.Column("known_limitations_json", sa.JSON(), nullable=False, server_default=sa.text("'[]'::json")), sa.Column("snapshot_metadata_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()), sa.UniqueConstraint("source_registry_id", "snapshot_key", name="uq_source_snapshots_registry_key"), sa.CheckConstraint( "freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')", name="ck_source_snapshots_freshness_status", ), sa.CheckConstraint( "ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', " "'failed', 'quarantined', 'legacy_unverified')", name="ck_source_snapshots_ingest_status", ), sa.CheckConstraint( "checksum_sha256 = lower(checksum_sha256) AND checksum_sha256 ~ '^[0-9a-f]{64}$'", name="ck_source_snapshots_checksum_sha256", ), ) op.create_index("ix_source_snapshots_registry_fetched", "source_snapshots", ["source_registry_id", "fetched_at"]) op.create_index("ix_source_snapshots_checksum", "source_snapshots", ["checksum_sha256"]) op.add_column("datasets", sa.Column("source_registry_id", sa.UUID(), nullable=True)) op.add_column("datasets", sa.Column("source_snapshot_id", sa.UUID(), nullable=True)) op.add_column("datasets", sa.Column("ingest_key", sa.String(length=255), nullable=True)) op.add_column("datasets", sa.Column("data_contract_key", sa.String(length=120), nullable=True)) op.add_column("datasets", sa.Column("data_contract_version", sa.String(length=64), nullable=True)) op.add_column( "datasets", sa.Column( "validation_status", sa.String(length=32), nullable=False, server_default="not_validated", comment="not_validated | passed | failed", ), ) op.add_column("datasets", sa.Column("validation_report_json", sa.JSON(), nullable=True)) op.add_column( "datasets", sa.Column( "provenance_status", sa.String(length=32), nullable=False, server_default="incomplete", comment="complete | incomplete | not_applicable", ), ) op.add_column( "datasets", sa.Column( "lineage_status", sa.String(length=32), nullable=False, server_default="incomplete", comment="complete | incomplete | not_applicable", ), ) op.add_column( "datasets", sa.Column( "quarantine_status", sa.String(length=32), nullable=False, server_default="not_quarantined", comment="not_quarantined | quarantined", ), ) op.add_column("dataset_versions", sa.Column("source_registry_id", sa.UUID(), nullable=True)) op.add_column("dataset_versions", sa.Column("source_snapshot_id", sa.UUID(), nullable=True)) op.add_column("dataset_versions", sa.Column("ingest_key", sa.String(length=255), nullable=True)) op.add_column("dataset_versions", sa.Column("data_contract_key", sa.String(length=120), nullable=True)) op.add_column("dataset_versions", sa.Column("data_contract_version", sa.String(length=64), nullable=True)) op.add_column( "dataset_versions", sa.Column( "validation_status", sa.String(length=32), nullable=False, server_default="not_validated", comment="not_validated | passed | failed", ), ) op.add_column("dataset_versions", sa.Column("validation_report_json", sa.JSON(), nullable=True)) op.add_column( "dataset_versions", sa.Column( "provenance_status", sa.String(length=32), nullable=False, server_default="incomplete", comment="complete | incomplete | not_applicable", ), ) op.add_column( "dataset_versions", sa.Column( "lineage_status", sa.String(length=32), nullable=False, server_default="incomplete", comment="complete | incomplete | not_applicable", ), ) op.create_foreign_key("fk_datasets_source_registry", "datasets", "source_registry", ["source_registry_id"], ["id"], ondelete="SET NULL") op.create_foreign_key("fk_datasets_source_snapshot", "datasets", "source_snapshots", ["source_snapshot_id"], ["id"], ondelete="SET NULL") op.create_foreign_key("fk_dataset_versions_source_registry", "dataset_versions", "source_registry", ["source_registry_id"], ["id"], ondelete="SET NULL") op.create_foreign_key("fk_dataset_versions_source_snapshot", "dataset_versions", "source_snapshots", ["source_snapshot_id"], ["id"], ondelete="SET NULL") op.create_index("ix_datasets_source_registry_snapshot", "datasets", ["source_registry_id", "source_snapshot_id"]) op.create_index("ix_dataset_versions_source_registry_snapshot", "dataset_versions", ["source_registry_id", "source_snapshot_id"]) op.create_unique_constraint("uq_datasets_project_ingest_key", "datasets", ["project_id", "ingest_key"]) op.create_unique_constraint("uq_dataset_versions_dataset_ingest_key", "dataset_versions", ["dataset_id", "ingest_key"]) op.create_check_constraint("ck_datasets_validation_status", "datasets", "validation_status IN ('not_validated', 'passed', 'failed')") op.create_check_constraint("ck_datasets_provenance_status", "datasets", "provenance_status IN ('complete', 'incomplete', 'not_applicable')") op.create_check_constraint("ck_datasets_lineage_status", "datasets", "lineage_status IN ('complete', 'incomplete', 'not_applicable')") op.create_check_constraint("ck_datasets_quarantine_status", "datasets", "quarantine_status IN ('not_quarantined', 'quarantined')") op.create_check_constraint("ck_datasets_ingest_key_not_blank", "datasets", "ingest_key IS NULL OR btrim(ingest_key) <> ''") op.create_check_constraint("ck_dataset_versions_validation_status", "dataset_versions", "validation_status IN ('not_validated', 'passed', 'failed')") op.create_check_constraint("ck_dataset_versions_provenance_status", "dataset_versions", "provenance_status IN ('complete', 'incomplete', 'not_applicable')") op.create_check_constraint("ck_dataset_versions_lineage_status", "dataset_versions", "lineage_status IN ('complete', 'incomplete', 'not_applicable')") op.create_check_constraint("ck_dataset_versions_ingest_key_not_blank", "dataset_versions", "ingest_key IS NULL OR btrim(ingest_key) <> ''") op.create_table( "dataset_lineage_edges", sa.Column("id", sa.UUID(), primary_key=True), sa.Column("parent_dataset_id", sa.UUID(), sa.ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False), sa.Column("child_dataset_id", sa.UUID(), sa.ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False), sa.Column("parent_dataset_version_id", sa.UUID(), sa.ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True), sa.Column("child_dataset_version_id", sa.UUID(), sa.ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True), sa.Column("relation_type", sa.String(length=64), nullable=False), sa.Column("transformation_name", sa.String(length=255), nullable=False), sa.Column("transformation_version", sa.String(length=120), nullable=True), sa.Column("parameters_json", sa.JSON(), nullable=True), sa.Column("input_checksum_sha256", sa.String(length=64), nullable=True), sa.Column("output_checksum_sha256", sa.String(length=64), nullable=True), sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()), sa.CheckConstraint("parent_dataset_id <> child_dataset_id", name="ck_dataset_lineage_edges_distinct_datasets"), sa.UniqueConstraint("parent_dataset_id", "child_dataset_id", "relation_type", "transformation_name", name="uq_dataset_lineage_edges_relation"), ) op.create_index("ix_dataset_lineage_edges_parent", "dataset_lineage_edges", ["parent_dataset_id"]) op.create_index("ix_dataset_lineage_edges_child", "dataset_lineage_edges", ["child_dataset_id"]) op.create_table( "dataset_quarantines", sa.Column("id", sa.UUID(), primary_key=True), sa.Column("dataset_id", sa.UUID(), sa.ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True), sa.Column("dataset_version_id", sa.UUID(), sa.ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True), sa.Column("source_snapshot_id", sa.UUID(), sa.ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True), sa.Column("stage", sa.String(length=64), nullable=False), sa.Column("reason_code", sa.String(length=120), nullable=False), sa.Column("details_json", sa.JSON(), nullable=True), sa.Column("artifact_path", sa.Text(), nullable=True), sa.Column("artifact_checksum_sha256", sa.String(length=64), nullable=True), sa.Column("status", sa.String(length=32), nullable=False, server_default="quarantined"), sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()), sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True), sa.Column("resolved_by", sa.String(length=120), nullable=True), sa.CheckConstraint( "dataset_id IS NOT NULL OR dataset_version_id IS NOT NULL OR source_snapshot_id IS NOT NULL", name="ck_dataset_quarantines_target_present", ), sa.CheckConstraint("status IN ('quarantined', 'released', 'rejected')", name="ck_dataset_quarantines_status"), ) op.create_index("ix_dataset_quarantines_dataset_status", "dataset_quarantines", ["dataset_id", "status"]) op.create_index("ix_dataset_quarantines_snapshot_status", "dataset_quarantines", ["source_snapshot_id", "status"]) op.bulk_insert(_source_registry_table(), _offline_safe_seed_rows(), multiinsert=False) op.execute( sa.text( """ UPDATE datasets SET validation_status = 'not_validated', provenance_status = 'incomplete', lineage_status = 'incomplete', quarantine_status = 'not_quarantined' """ ) ) op.execute( sa.text( """ UPDATE datasets AS dataset SET source_registry_id = registry.id FROM source_registry AS registry WHERE registry.source_key = COALESCE( NULLIF(lower(btrim(dataset.source_name)), ''), NULLIF(lower(btrim(dataset.source)), ''), '__unregistered_legacy_source__' ) """ ) ) op.execute( sa.text( """ UPDATE dataset_versions AS version SET source_registry_id = dataset.source_registry_id, validation_status = 'not_validated', provenance_status = 'incomplete', lineage_status = 'incomplete' FROM datasets AS dataset WHERE version.dataset_id = dataset.id """ ) ) # A Dataset has separate foreign keys to its source registry and immutable # snapshot. Those independent FKs alone cannot prove the snapshot belongs # to the selected registry. Keep legacy rows with no snapshot valid, but # reject every new or mutated mismatched pair at the database boundary. # This mirrors the fail-closed application consumption gate and also # protects maintenance scripts that bypass the ORM. op.execute( sa.text( """ CREATE FUNCTION geointel_phase2_snapshot_registry_guard() RETURNS trigger AS $$ DECLARE snapshot_registry uuid; BEGIN IF NEW.source_snapshot_id IS NULL THEN RETURN NEW; END IF; SELECT source_registry_id INTO snapshot_registry FROM source_snapshots WHERE id = NEW.source_snapshot_id; IF snapshot_registry IS NULL OR NEW.source_registry_id IS NULL OR NEW.source_registry_id <> snapshot_registry THEN RAISE EXCEPTION 'source snapshot % does not belong to source registry %', NEW.source_snapshot_id, NEW.source_registry_id USING ERRCODE = '23514'; END IF; RETURN NEW; END; $$ LANGUAGE plpgsql; CREATE TRIGGER trg_datasets_snapshot_registry_guard BEFORE INSERT OR UPDATE OF source_registry_id, source_snapshot_id ON datasets FOR EACH ROW EXECUTE FUNCTION geointel_phase2_snapshot_registry_guard(); CREATE TRIGGER trg_dataset_versions_snapshot_registry_guard BEFORE INSERT OR UPDATE OF source_registry_id, source_snapshot_id ON dataset_versions FOR EACH ROW EXECUTE FUNCTION geointel_phase2_snapshot_registry_guard(); CREATE FUNCTION geointel_phase2_snapshot_registry_immutable_guard() RETURNS trigger AS $$ BEGIN IF NEW.source_registry_id IS DISTINCT FROM OLD.source_registry_id THEN RAISE EXCEPTION 'source snapshot registry identity is immutable' USING ERRCODE = '23514'; END IF; RETURN NEW; END; $$ LANGUAGE plpgsql; CREATE TRIGGER trg_source_snapshots_registry_immutable BEFORE UPDATE OF source_registry_id ON source_snapshots FOR EACH ROW EXECUTE FUNCTION geointel_phase2_snapshot_registry_immutable_guard(); CREATE FUNCTION geointel_phase2_source_registry_write_guard() RETURNS trigger AS $$ BEGIN RAISE EXCEPTION 'source registry entries are server-owned and immutable' USING ERRCODE = '23514'; END; $$ LANGUAGE plpgsql; CREATE TRIGGER trg_source_registry_write_guard BEFORE UPDATE OR DELETE ON source_registry FOR EACH ROW EXECUTE FUNCTION geointel_phase2_source_registry_write_guard(); CREATE FUNCTION geointel_phase2_snapshot_evidence_immutable_guard() RETURNS trigger AS $$ BEGIN IF TG_OP = 'DELETE' THEN RAISE EXCEPTION 'source snapshots are immutable evidence and cannot be deleted' USING ERRCODE = '23514'; END IF; IF NEW.source_registry_id IS DISTINCT FROM OLD.source_registry_id OR NEW.snapshot_key IS DISTINCT FROM OLD.snapshot_key OR NEW.source_version IS DISTINCT FROM OLD.source_version OR NEW.snapshot_at IS DISTINCT FROM OLD.snapshot_at OR NEW.fetched_at IS DISTINCT FROM OLD.fetched_at OR NEW.source_url IS DISTINCT FROM OLD.source_url OR NEW.checksum_sha256 IS DISTINCT FROM OLD.checksum_sha256 OR NEW.crs IS DISTINCT FROM OLD.crs OR NEW.units IS DISTINCT FROM OLD.units OR NEW.spatial_resolution_json::text IS DISTINCT FROM OLD.spatial_resolution_json::text OR NEW.temporal_coverage_json::text IS DISTINCT FROM OLD.temporal_coverage_json::text OR NEW.geographic_coverage_json::text IS DISTINCT FROM OLD.geographic_coverage_json::text OR NEW.observed_schema_json::text IS DISTINCT FROM OLD.observed_schema_json::text OR NEW.freshness_status IS DISTINCT FROM OLD.freshness_status OR NEW.known_limitations_json::text IS DISTINCT FROM OLD.known_limitations_json::text OR NEW.snapshot_metadata_json::text IS DISTINCT FROM OLD.snapshot_metadata_json::text THEN RAISE EXCEPTION 'source snapshot evidence is immutable' USING ERRCODE = '23514'; END IF; IF NEW.ingest_status IS DISTINCT FROM OLD.ingest_status AND NEW.ingest_status <> 'quarantined' THEN RAISE EXCEPTION 'source snapshot lifecycle may only transition to quarantined' USING ERRCODE = '23514'; END IF; RETURN NEW; END; $$ LANGUAGE plpgsql; CREATE TRIGGER trg_source_snapshots_evidence_immutable BEFORE UPDATE OR DELETE ON source_snapshots FOR EACH ROW EXECUTE FUNCTION geointel_phase2_snapshot_evidence_immutable_guard(); CREATE FUNCTION geointel_phase2_contract_report_guard() RETURNS trigger AS $$ DECLARE snapshot_checksum text; BEGIN IF NEW.validation_status = 'passed' THEN IF NEW.data_contract_key IS NULL OR btrim(NEW.data_contract_key) = '' OR NEW.data_contract_version IS NULL OR btrim(NEW.data_contract_version) = '' OR NEW.source_registry_id IS NULL OR NEW.source_snapshot_id IS NULL OR NEW.validation_report_json IS NULL OR json_typeof(NEW.validation_report_json) <> 'object' OR COALESCE(NEW.validation_report_json ->> 'validation_status', '') <> 'passed' OR COALESCE(NEW.validation_report_json ->> 'data_contract_key', '') <> NEW.data_contract_key OR COALESCE(NEW.validation_report_json ->> 'data_contract_version', '') <> NEW.data_contract_version OR COALESCE(NEW.validation_report_json ->> 'provenance_status', '') <> 'complete' OR COALESCE(NEW.validation_report_json ->> 'lineage_status', '') NOT IN ('complete', 'not_applicable') OR COALESCE(NEW.validation_report_json ->> 'quarantine_status', '') <> 'not_quarantined' OR COALESCE(NEW.validation_report_json ->> 'contract_fingerprint_sha256', '') !~ '^[0-9a-f]{64}$' OR COALESCE(NEW.validation_report_json ->> 'report_sha256', '') !~ '^[0-9a-f]{64}$' OR NEW.provenance_status <> 'complete' OR NEW.lineage_status NOT IN ('complete', 'not_applicable') THEN RAISE EXCEPTION 'passed dataset contract state requires a matching complete validation report' USING ERRCODE = '23514'; END IF; IF NEW.quarantine_status <> 'not_quarantined' THEN RAISE EXCEPTION 'passed dataset cannot be quarantined' USING ERRCODE = '23514'; END IF; SELECT checksum_sha256 INTO snapshot_checksum FROM source_snapshots WHERE id = NEW.source_snapshot_id; IF NEW.checksum_sha256 IS NULL OR NEW.checksum_sha256 <> lower(NEW.checksum_sha256) OR NEW.checksum_sha256 !~ '^[0-9a-f]{64}$' OR snapshot_checksum IS NULL OR NEW.checksum_sha256 <> snapshot_checksum THEN RAISE EXCEPTION 'passed dataset checksum must be a canonical SHA-256 bound to its source snapshot' USING ERRCODE = '23514'; END IF; END IF; -- A passed record is bound to exact bytes and the source/time -- evidence that was validated with those bytes. A caller may -- invalidate it, but cannot atomically replace any evidence -- while doing so: invalidate first, then governed re-ingest. IF TG_OP = 'UPDATE' AND OLD.validation_status = 'passed' AND ( NEW.source IS DISTINCT FROM OLD.source OR NEW.dataset_type IS DISTINCT FROM OLD.dataset_type OR NEW.storage_path IS DISTINCT FROM OLD.storage_path OR NEW.original_filename IS DISTINCT FROM OLD.original_filename OR NEW.stored_filename IS DISTINCT FROM OLD.stored_filename OR NEW.content_type IS DISTINCT FROM OLD.content_type OR NEW.size_bytes IS DISTINCT FROM OLD.size_bytes OR NEW.checksum_sha256 IS DISTINCT FROM OLD.checksum_sha256 OR NEW.ingest_key IS DISTINCT FROM OLD.ingest_key OR NEW.derived_from_dataset_id IS DISTINCT FROM OLD.derived_from_dataset_id OR NEW.crs IS DISTINCT FROM OLD.crs OR NEW.bounds_json::text IS DISTINCT FROM OLD.bounds_json::text OR NEW.resolution_json::text IS DISTINCT FROM OLD.resolution_json::text OR NEW.bands_json::text IS DISTINCT FROM OLD.bands_json::text OR NEW.metadata_json::text IS DISTINCT FROM OLD.metadata_json::text OR NEW.dataset_role IS DISTINCT FROM OLD.dataset_role OR NEW.source_name IS DISTINCT FROM OLD.source_name OR NEW.reference_layer_name IS DISTINCT FROM OLD.reference_layer_name OR NEW.source_metadata::text IS DISTINCT FROM OLD.source_metadata::text OR NEW.provenance_metadata::text IS DISTINCT FROM OLD.provenance_metadata::text OR NEW.source_registry_id IS DISTINCT FROM OLD.source_registry_id OR NEW.source_snapshot_id IS DISTINCT FROM OLD.source_snapshot_id OR NEW.data_contract_key IS DISTINCT FROM OLD.data_contract_key OR NEW.data_contract_version IS DISTINCT FROM OLD.data_contract_version OR NEW.validation_report_json::text IS DISTINCT FROM OLD.validation_report_json::text OR NEW.temporal_series_key IS DISTINCT FROM OLD.temporal_series_key OR NEW.observed_at IS DISTINCT FROM OLD.observed_at OR NEW.valid_from IS DISTINCT FROM OLD.valid_from OR NEW.valid_to IS DISTINCT FROM OLD.valid_to OR NEW.temporal_granularity IS DISTINCT FROM OLD.temporal_granularity OR NEW.source_version IS DISTINCT FROM OLD.source_version ) THEN RAISE EXCEPTION 'accepted dataset artifact and provenance evidence is immutable; invalidate it before a governed re-ingest' USING ERRCODE = '23514'; END IF; IF TG_OP = 'UPDATE' AND OLD.quarantine_status = 'quarantined' AND NEW.quarantine_status <> 'quarantined' THEN RAISE EXCEPTION 'quarantined dataset cannot be released in place; a governed re-ingest is required' USING ERRCODE = '23514'; END IF; RETURN NEW; END; $$ LANGUAGE plpgsql; CREATE FUNCTION geointel_phase2_dataset_version_contract_report_guard() RETURNS trigger AS $$ DECLARE snapshot_checksum text; BEGIN IF NEW.validation_status = 'passed' THEN IF NEW.data_contract_key IS NULL OR btrim(NEW.data_contract_key) = '' OR NEW.data_contract_version IS NULL OR btrim(NEW.data_contract_version) = '' OR NEW.source_registry_id IS NULL OR NEW.source_snapshot_id IS NULL OR NEW.validation_report_json IS NULL OR json_typeof(NEW.validation_report_json) <> 'object' OR COALESCE(NEW.validation_report_json ->> 'validation_status', '') <> 'passed' OR COALESCE(NEW.validation_report_json ->> 'data_contract_key', '') <> NEW.data_contract_key OR COALESCE(NEW.validation_report_json ->> 'data_contract_version', '') <> NEW.data_contract_version OR COALESCE(NEW.validation_report_json ->> 'provenance_status', '') <> 'complete' OR COALESCE(NEW.validation_report_json ->> 'lineage_status', '') NOT IN ('complete', 'not_applicable') OR COALESCE(NEW.validation_report_json ->> 'quarantine_status', '') <> 'not_quarantined' OR COALESCE(NEW.validation_report_json ->> 'contract_fingerprint_sha256', '') !~ '^[0-9a-f]{64}$' OR COALESCE(NEW.validation_report_json ->> 'report_sha256', '') !~ '^[0-9a-f]{64}$' OR NEW.provenance_status <> 'complete' OR NEW.lineage_status NOT IN ('complete', 'not_applicable') THEN RAISE EXCEPTION 'passed dataset version contract state requires a matching complete validation report' USING ERRCODE = '23514'; END IF; SELECT checksum_sha256 INTO snapshot_checksum FROM source_snapshots WHERE id = NEW.source_snapshot_id; IF NEW.checksum_sha256 IS NULL OR NEW.checksum_sha256 <> lower(NEW.checksum_sha256) OR NEW.checksum_sha256 !~ '^[0-9a-f]{64}$' OR snapshot_checksum IS NULL OR NEW.checksum_sha256 <> snapshot_checksum THEN RAISE EXCEPTION 'passed dataset version checksum must be a canonical SHA-256 bound to its source snapshot' USING ERRCODE = '23514'; END IF; END IF; IF TG_OP = 'UPDATE' AND OLD.validation_status = 'passed' AND ( NEW.dataset_id IS DISTINCT FROM OLD.dataset_id OR NEW.version IS DISTINCT FROM OLD.version OR NEW.storage_path IS DISTINCT FROM OLD.storage_path OR NEW.source_version IS DISTINCT FROM OLD.source_version OR NEW.observed_at IS DISTINCT FROM OLD.observed_at OR NEW.valid_from IS DISTINCT FROM OLD.valid_from OR NEW.valid_to IS DISTINCT FROM OLD.valid_to OR NEW.checksum_sha256 IS DISTINCT FROM OLD.checksum_sha256 OR NEW.ingest_key IS DISTINCT FROM OLD.ingest_key OR NEW.source_metadata::text IS DISTINCT FROM OLD.source_metadata::text OR NEW.provenance_metadata::text IS DISTINCT FROM OLD.provenance_metadata::text OR NEW.source_registry_id IS DISTINCT FROM OLD.source_registry_id OR NEW.source_snapshot_id IS DISTINCT FROM OLD.source_snapshot_id OR NEW.data_contract_key IS DISTINCT FROM OLD.data_contract_key OR NEW.data_contract_version IS DISTINCT FROM OLD.data_contract_version OR NEW.validation_report_json::text IS DISTINCT FROM OLD.validation_report_json::text ) THEN RAISE EXCEPTION 'accepted dataset-version artifact and provenance evidence is immutable; invalidate it before a governed re-ingest' USING ERRCODE = '23514'; END IF; RETURN NEW; END; $$ LANGUAGE plpgsql; CREATE TRIGGER trg_datasets_contract_report_guard BEFORE INSERT OR UPDATE OF source, dataset_type, storage_path, original_filename, stored_filename, content_type, size_bytes, checksum_sha256, ingest_key, derived_from_dataset_id, crs, bounds_json, resolution_json, bands_json, metadata_json, dataset_role, source_name, reference_layer_name, source_metadata, provenance_metadata, source_registry_id, source_snapshot_id, data_contract_key, data_contract_version, validation_status, validation_report_json, provenance_status, lineage_status, quarantine_status, temporal_series_key, observed_at, valid_from, valid_to, temporal_granularity, source_version ON datasets FOR EACH ROW EXECUTE FUNCTION geointel_phase2_contract_report_guard(); CREATE TRIGGER trg_dataset_versions_contract_report_guard BEFORE INSERT OR UPDATE OF dataset_id, version, storage_path, source_version, observed_at, valid_from, valid_to, checksum_sha256, ingest_key, source_metadata, provenance_metadata, source_registry_id, source_snapshot_id, data_contract_key, data_contract_version, validation_status, validation_report_json, provenance_status, lineage_status ON dataset_versions FOR EACH ROW EXECUTE FUNCTION geointel_phase2_dataset_version_contract_report_guard(); CREATE FUNCTION geointel_phase2_lineage_cycle_guard() RETURNS trigger AS $$ BEGIN IF NEW.parent_dataset_id = NEW.child_dataset_id THEN RAISE EXCEPTION 'a dataset cannot be its own lineage parent' USING ERRCODE = '23514'; END IF; IF EXISTS ( WITH RECURSIVE descendants(dataset_id) AS ( SELECT edge.child_dataset_id FROM dataset_lineage_edges AS edge WHERE edge.parent_dataset_id = NEW.child_dataset_id AND (TG_OP <> 'UPDATE' OR edge.id <> NEW.id) UNION SELECT edge.child_dataset_id FROM dataset_lineage_edges AS edge JOIN descendants ON edge.parent_dataset_id = descendants.dataset_id WHERE TG_OP <> 'UPDATE' OR edge.id <> NEW.id ) SELECT 1 FROM descendants WHERE dataset_id = NEW.parent_dataset_id ) THEN RAISE EXCEPTION 'lineage edge % -> % would create a cycle', NEW.parent_dataset_id, NEW.child_dataset_id USING ERRCODE = '23514'; END IF; RETURN NEW; END; $$ LANGUAGE plpgsql; CREATE TRIGGER trg_dataset_lineage_edges_cycle_guard BEFORE INSERT ON dataset_lineage_edges FOR EACH ROW EXECUTE FUNCTION geointel_phase2_lineage_cycle_guard(); CREATE FUNCTION geointel_phase2_lineage_edge_immutable_guard() RETURNS trigger AS $$ BEGIN RAISE EXCEPTION 'dataset lineage edges are immutable evidence and cannot be updated or deleted' USING ERRCODE = '23514'; END; $$ LANGUAGE plpgsql; CREATE TRIGGER trg_dataset_lineage_edges_immutable BEFORE UPDATE OR DELETE ON dataset_lineage_edges FOR EACH ROW EXECUTE FUNCTION geointel_phase2_lineage_edge_immutable_guard(); CREATE FUNCTION geointel_phase2_quarantine_lineage_descendants( root_dataset_id uuid, root_snapshot_id uuid ) RETURNS void AS $$ BEGIN -- A shared source snapshot has several direct Dataset roots; -- each root and every descendant must be made non-consumable. -- UNION (rather than UNION ALL) keeps corrupt historic cycles -- finite while the immutable cycle guard prevents new ones. WITH RECURSIVE quarantine_roots(dataset_id) AS ( SELECT root_dataset_id WHERE root_dataset_id IS NOT NULL UNION SELECT dataset.id FROM datasets AS dataset WHERE root_snapshot_id IS NOT NULL AND dataset.source_snapshot_id = root_snapshot_id ), descendants(dataset_id) AS ( SELECT dataset_id FROM quarantine_roots UNION SELECT edge.child_dataset_id FROM dataset_lineage_edges AS edge JOIN descendants AS upstream ON edge.parent_dataset_id = upstream.dataset_id ) UPDATE dataset_versions AS version SET validation_status = 'failed', provenance_status = 'incomplete', lineage_status = 'incomplete' WHERE version.dataset_id IN (SELECT dataset_id FROM descendants); WITH RECURSIVE quarantine_roots(dataset_id) AS ( SELECT root_dataset_id WHERE root_dataset_id IS NOT NULL UNION SELECT dataset.id FROM datasets AS dataset WHERE root_snapshot_id IS NOT NULL AND dataset.source_snapshot_id = root_snapshot_id ), descendants(dataset_id) AS ( SELECT dataset_id FROM quarantine_roots UNION SELECT edge.child_dataset_id FROM dataset_lineage_edges AS edge JOIN descendants AS upstream ON edge.parent_dataset_id = upstream.dataset_id ) UPDATE datasets AS dataset SET status = 'quarantined', quarantine_status = 'quarantined', validation_status = 'failed', provenance_status = 'incomplete', lineage_status = 'incomplete' WHERE dataset.id IN (SELECT dataset_id FROM descendants); END; $$ LANGUAGE plpgsql; CREATE FUNCTION geointel_phase2_quarantine_state_guard() RETURNS trigger AS $$ DECLARE owner_dataset uuid; owner_snapshot uuid; dataset_snapshot uuid; BEGIN IF NEW.status <> 'quarantined' THEN RETURN NEW; END IF; IF NEW.dataset_version_id IS NOT NULL THEN SELECT dataset_id, source_snapshot_id INTO owner_dataset, owner_snapshot FROM dataset_versions WHERE id = NEW.dataset_version_id; IF owner_dataset IS NULL THEN RAISE EXCEPTION 'quarantine dataset version % has no owning dataset', NEW.dataset_version_id USING ERRCODE = '23514'; END IF; IF NEW.dataset_id IS NOT NULL AND NEW.dataset_id <> owner_dataset THEN RAISE EXCEPTION 'quarantine dataset and dataset version do not belong together' USING ERRCODE = '23514'; END IF; UPDATE dataset_versions SET validation_status = 'failed', provenance_status = 'incomplete', lineage_status = 'incomplete' WHERE id = NEW.dataset_version_id; END IF; owner_dataset := COALESCE(NEW.dataset_id, owner_dataset); IF owner_dataset IS NOT NULL THEN SELECT source_snapshot_id INTO dataset_snapshot FROM datasets WHERE id = owner_dataset; IF NEW.dataset_version_id IS NULL AND NEW.source_snapshot_id IS NOT NULL AND dataset_snapshot IS NOT NULL AND NEW.source_snapshot_id <> dataset_snapshot THEN RAISE EXCEPTION 'quarantine dataset and source snapshot do not belong together' USING ERRCODE = '23514'; END IF; IF NEW.dataset_version_id IS NOT NULL AND NEW.source_snapshot_id IS NOT NULL AND owner_snapshot IS NOT NULL AND NEW.source_snapshot_id <> owner_snapshot THEN RAISE EXCEPTION 'quarantine dataset version and source snapshot do not belong together' USING ERRCODE = '23514'; END IF; UPDATE datasets SET status = 'quarantined', quarantine_status = 'quarantined', validation_status = 'failed', provenance_status = 'incomplete', lineage_status = 'incomplete' WHERE id = owner_dataset; END IF; owner_snapshot := COALESCE(NEW.source_snapshot_id, owner_snapshot, dataset_snapshot); IF owner_snapshot IS NOT NULL THEN UPDATE source_snapshots SET ingest_status = 'quarantined' WHERE id = owner_snapshot; -- A concrete snapshot is shared immutable evidence. Once -- quarantined, every Dataset/Version bound to it must be -- non-consumable as well; a consumption gate must never -- depend on callers having included source_snapshot_id in -- the quarantine record. UPDATE dataset_versions SET validation_status = 'failed', provenance_status = 'incomplete', lineage_status = 'incomplete' WHERE source_snapshot_id = owner_snapshot; UPDATE datasets SET status = 'quarantined', quarantine_status = 'quarantined', validation_status = 'failed', provenance_status = 'incomplete', lineage_status = 'incomplete' WHERE source_snapshot_id = owner_snapshot; END IF; -- Direct quarantine and shared-snapshot fan-out are not -- enough: every downstream derivative inherits the unsafe -- lineage and must fail all Dataset consumption boundaries. PERFORM geointel_phase2_quarantine_lineage_descendants(owner_dataset, owner_snapshot); RETURN NEW; END; $$ LANGUAGE plpgsql; CREATE TRIGGER trg_dataset_quarantines_state_guard AFTER INSERT OR UPDATE OF status, dataset_id, dataset_version_id, source_snapshot_id ON dataset_quarantines FOR EACH ROW EXECUTE FUNCTION geointel_phase2_quarantine_state_guard(); """ ) ) def downgrade() -> None: op.execute( sa.text( """ DROP TRIGGER IF EXISTS trg_datasets_snapshot_registry_guard ON datasets; DROP TRIGGER IF EXISTS trg_dataset_versions_snapshot_registry_guard ON dataset_versions; DROP TRIGGER IF EXISTS trg_source_snapshots_registry_immutable ON source_snapshots; DROP TRIGGER IF EXISTS trg_source_registry_write_guard ON source_registry; DROP TRIGGER IF EXISTS trg_source_snapshots_evidence_immutable ON source_snapshots; DROP TRIGGER IF EXISTS trg_datasets_contract_report_guard ON datasets; DROP TRIGGER IF EXISTS trg_dataset_versions_contract_report_guard ON dataset_versions; DROP TRIGGER IF EXISTS trg_dataset_lineage_edges_cycle_guard ON dataset_lineage_edges; DROP TRIGGER IF EXISTS trg_dataset_lineage_edges_immutable ON dataset_lineage_edges; DROP TRIGGER IF EXISTS trg_dataset_quarantines_state_guard ON dataset_quarantines; DROP FUNCTION IF EXISTS geointel_phase2_snapshot_registry_guard(); DROP FUNCTION IF EXISTS geointel_phase2_snapshot_registry_immutable_guard(); DROP FUNCTION IF EXISTS geointel_phase2_source_registry_write_guard(); DROP FUNCTION IF EXISTS geointel_phase2_snapshot_evidence_immutable_guard(); DROP FUNCTION IF EXISTS geointel_phase2_contract_report_guard(); DROP FUNCTION IF EXISTS geointel_phase2_dataset_version_contract_report_guard(); DROP FUNCTION IF EXISTS geointel_phase2_lineage_cycle_guard(); DROP FUNCTION IF EXISTS geointel_phase2_lineage_edge_immutable_guard(); DROP FUNCTION IF EXISTS geointel_phase2_quarantine_lineage_descendants(uuid, uuid); DROP FUNCTION IF EXISTS geointel_phase2_quarantine_state_guard(); """ ) ) op.drop_table("dataset_quarantines") op.drop_table("dataset_lineage_edges") op.drop_constraint("ck_dataset_versions_ingest_key_not_blank", "dataset_versions", type_="check") op.drop_constraint("ck_dataset_versions_lineage_status", "dataset_versions", type_="check") op.drop_constraint("ck_dataset_versions_provenance_status", "dataset_versions", type_="check") op.drop_constraint("ck_dataset_versions_validation_status", "dataset_versions", type_="check") op.drop_constraint("ck_datasets_ingest_key_not_blank", "datasets", type_="check") op.drop_constraint("ck_datasets_quarantine_status", "datasets", type_="check") op.drop_constraint("ck_datasets_lineage_status", "datasets", type_="check") op.drop_constraint("ck_datasets_provenance_status", "datasets", type_="check") op.drop_constraint("ck_datasets_validation_status", "datasets", type_="check") op.drop_constraint("uq_dataset_versions_dataset_ingest_key", "dataset_versions", type_="unique") op.drop_constraint("uq_datasets_project_ingest_key", "datasets", type_="unique") op.drop_index("ix_dataset_versions_source_registry_snapshot", table_name="dataset_versions") op.drop_index("ix_datasets_source_registry_snapshot", table_name="datasets") op.drop_constraint("fk_dataset_versions_source_snapshot", "dataset_versions", type_="foreignkey") op.drop_constraint("fk_dataset_versions_source_registry", "dataset_versions", type_="foreignkey") op.drop_constraint("fk_datasets_source_snapshot", "datasets", type_="foreignkey") op.drop_constraint("fk_datasets_source_registry", "datasets", type_="foreignkey") for column_name in ( "lineage_status", "provenance_status", "validation_report_json", "validation_status", "data_contract_version", "data_contract_key", "ingest_key", "source_snapshot_id", "source_registry_id", ): op.drop_column("dataset_versions", column_name) for column_name in ( "quarantine_status", "lineage_status", "provenance_status", "validation_report_json", "validation_status", "data_contract_version", "data_contract_key", "ingest_key", "source_snapshot_id", "source_registry_id", ): op.drop_column("datasets", column_name) op.drop_index("ix_source_snapshots_checksum", table_name="source_snapshots") op.drop_index("ix_source_snapshots_registry_fetched", table_name="source_snapshots") op.drop_table("source_snapshots") op.drop_table("source_registry")