Jens
faeb58ef6d
GeoIntel release gates / Compile, test, contracts and builds (push) Successful in 1m49s
GeoIntel release gates / Python and npm vulnerability policy (push) Successful in 21s
GeoIntel release gates / Production AI image, SBOM and container scan (push) Successful in 5m39s
GeoIntel release gates / Deploy exact gated revision to Unraid (push) Failing after 58m43s
1309 lines
74 KiB
Python
1309 lines
74 KiB
Python
"""Add server-owned source registry, source snapshots and dataset provenance gates.
|
|
|
|
Legacy datasets are retained untouched. They receive an identity link when one
|
|
is available, but remain explicitly incomplete/not_validated until a governed
|
|
ingest writes an immutable source snapshot and contract result.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import uuid
|
|
import json
|
|
|
|
from alembic import op
|
|
import sqlalchemy as sa
|
|
|
|
|
|
revision = "202608010001"
|
|
down_revision = "202607260001"
|
|
branch_labels = None
|
|
depends_on = None
|
|
|
|
|
|
_SOURCE_NAMESPACE = uuid.UUID("6e5f9c11-bc7f-4f5f-ae23-f3eddb4d0149")
|
|
_DEFAULT_LICENSE = "Provider terms must be verified for each immutable snapshot."
|
|
_DEFAULT_LIMITATION = "No authority, ground-truth, freshness or training claim is allowed without a governed snapshot and passed contract."
|
|
_DEFAULT_USAGE_POLICY = {
|
|
"automatic_ground_truth": False,
|
|
"ground_truth_allowed": False,
|
|
"training_allowed": False,
|
|
"allowed_tasks": [],
|
|
"validation_authority": {},
|
|
}
|
|
|
|
|
|
def _source_seed(
|
|
source_key: str,
|
|
display_name: str,
|
|
classification: str,
|
|
authority_name: str,
|
|
*,
|
|
authority_scope: dict | None = None,
|
|
provider_adapter_key: str | None = None,
|
|
source_url: str | None = None,
|
|
license_name: str = _DEFAULT_LICENSE,
|
|
usage_restrictions: str = "Use only according to the source-specific snapshot terms and attribution.",
|
|
default_crs: str = "unknown",
|
|
default_units: str = "unknown",
|
|
spatial_resolution: dict | None = None,
|
|
temporal_coverage: dict | None = None,
|
|
geographic_coverage: dict | None = None,
|
|
expected_geometry_types: list[str] | None = None,
|
|
expected_attributes: dict | None = None,
|
|
usage_policy: dict | None = None,
|
|
freshness_status: str = "unknown",
|
|
ingest_status: str = "registered",
|
|
known_limitations: list[str] | None = None,
|
|
) -> dict:
|
|
return {
|
|
"id": uuid.uuid5(_SOURCE_NAMESPACE, source_key),
|
|
"source_key": source_key,
|
|
"display_name": display_name,
|
|
"classification": classification,
|
|
"authority_name": authority_name,
|
|
"authority_scope_json": authority_scope or {"status": "unknown"},
|
|
"provider_adapter_key": provider_adapter_key,
|
|
"source_url": source_url,
|
|
"license_name": license_name,
|
|
"license_url": None,
|
|
"usage_restrictions": usage_restrictions,
|
|
"default_crs": default_crs,
|
|
"default_units": default_units,
|
|
"spatial_resolution_json": spatial_resolution or {"status": "unknown"},
|
|
"temporal_coverage_json": temporal_coverage or {"status": "unknown"},
|
|
"geographic_coverage_json": geographic_coverage or {"status": "unknown"},
|
|
"expected_geometry_types_json": expected_geometry_types or [],
|
|
"expected_attributes_json": expected_attributes or {"status": "unknown"},
|
|
"usage_policy_json": usage_policy or dict(_DEFAULT_USAGE_POLICY),
|
|
"freshness_status": freshness_status,
|
|
"ingest_status": ingest_status,
|
|
"known_limitations_json": known_limitations or [_DEFAULT_LIMITATION],
|
|
"registry_metadata_json": {"registry_owner": "server", "seeded_by": revision},
|
|
}
|
|
|
|
|
|
def _seed_rows() -> list[dict]:
|
|
governed_vector_policy = {
|
|
"automatic_ground_truth": False,
|
|
"ground_truth_allowed": False,
|
|
"training_allowed": False,
|
|
"allowed_tasks": ["reference_context"],
|
|
"validation_authority": {},
|
|
}
|
|
governed_imagery_policy = {
|
|
"automatic_ground_truth": False,
|
|
"ground_truth_allowed": False,
|
|
"training_allowed": True,
|
|
"allowed_tasks": ["imagery", "training_input", "visual_context"],
|
|
"validation_authority": {},
|
|
}
|
|
regional_building_policy = {
|
|
"automatic_ground_truth": False,
|
|
"ground_truth_allowed": True,
|
|
"training_allowed": True,
|
|
"allowed_tasks": ["building_validation", "building_labels"],
|
|
"validation_authority": {"building_validation": "regional_primary_pending_contract"},
|
|
}
|
|
|
|
return [
|
|
_source_seed(
|
|
"grb",
|
|
"Grootschalig Referentie Bestand",
|
|
"authoritative",
|
|
"Digitaal Vlaanderen",
|
|
authority_scope={"zone": "Flanders", "themes": ["buildings", "roads", "water", "parcels"]},
|
|
provider_adapter_key="grb",
|
|
source_url="https://www.vlaanderen.be/datavindplaats/catalogus/basiskaart-vlaanderen-grb",
|
|
usage_restrictions="Use governed GRB acquisition evidence and retain Digitaal Vlaanderen attribution.",
|
|
default_crs="EPSG:31370",
|
|
default_units="metres",
|
|
geographic_coverage={"zone": "Flanders"},
|
|
expected_geometry_types=["Polygon", "MultiPolygon", "LineString", "MultiLineString"],
|
|
expected_attributes={"required": ["id"], "layers": ["GBG", "Wegsegment", "WTZ", "WLAS", "WGR", "ADP"]},
|
|
usage_policy={
|
|
"automatic_ground_truth": False,
|
|
"ground_truth_allowed": True,
|
|
"training_allowed": True,
|
|
"allowed_tasks": ["building_validation", "building_labels", "reference_context"],
|
|
"validation_authority": {"building_validation": "primary"},
|
|
},
|
|
ingest_status="configured",
|
|
known_limitations=[
|
|
"GRB building geometry is authoritative only for a governed, versioned snapshot within Flanders.",
|
|
"GRB does not establish imagery-time alignment or national model validation by itself.",
|
|
],
|
|
),
|
|
_source_seed(
|
|
"digitaal_vlaanderen",
|
|
"Digitaal Vlaanderen (bronportaal)",
|
|
"authoritative",
|
|
"Digitaal Vlaanderen",
|
|
authority_scope={"zone": "Flanders", "role": "umbrella_catalogue_and_adapter_authority"},
|
|
provider_adapter_key="digitaal_vlaanderen",
|
|
source_url="https://www.vlaanderen.be/datavindplaats",
|
|
default_crs="product_specific",
|
|
default_units="product_specific",
|
|
geographic_coverage={"zone": "Flanders"},
|
|
usage_policy={
|
|
"automatic_ground_truth": False,
|
|
"ground_truth_allowed": False,
|
|
"training_allowed": False,
|
|
"allowed_tasks": ["source_catalogue", "reference_context"],
|
|
"validation_authority": {},
|
|
},
|
|
ingest_status="configured",
|
|
known_limitations=[
|
|
"This umbrella authority is not a product-level ground-truth source.",
|
|
"A governed import must use a product-specific source key whenever one is available.",
|
|
],
|
|
),
|
|
_source_seed(
|
|
"digitaal_vlaanderen_buildings_addresses_register",
|
|
"Gebouwen- en adressenregister",
|
|
"authoritative",
|
|
"Digitaal Vlaanderen",
|
|
authority_scope={"zone": "Flanders", "theme": "buildings_addresses"},
|
|
provider_adapter_key="buildings_addresses_register",
|
|
default_crs="EPSG:31370",
|
|
default_units="metres",
|
|
geographic_coverage={"zone": "Flanders"},
|
|
expected_geometry_types=["Point", "Polygon", "MultiPolygon"],
|
|
expected_attributes={"required": ["id"], "role": "administrative_corroboration"},
|
|
usage_policy={
|
|
"automatic_ground_truth": False,
|
|
"ground_truth_allowed": False,
|
|
"training_allowed": False,
|
|
"allowed_tasks": ["building_validation", "address_corroboration", "building_register_validation"],
|
|
"validation_authority": {
|
|
"building_validation": "corroborative",
|
|
"building_register_validation": "primary",
|
|
},
|
|
},
|
|
known_limitations=["Administrative building/address records do not replace a governed reference-footprint contract."],
|
|
),
|
|
_source_seed(
|
|
"sentinel_2",
|
|
"Sentinel-2",
|
|
"contextual",
|
|
"Copernicus Programme",
|
|
authority_scope={"scope": "Belgium and Belgian North Sea", "role": "multispectral_context"},
|
|
provider_adapter_key="sentinel_2",
|
|
default_crs="product_specific",
|
|
default_units="reflectance",
|
|
spatial_resolution={"metres": [10, 20, 60]},
|
|
temporal_coverage={"cadence_days": 5, "status": "product_specific"},
|
|
geographic_coverage={"scope": "Belgium and Belgian North Sea"},
|
|
expected_geometry_types=[],
|
|
expected_attributes={"required": ["product_id", "sensing_time"]},
|
|
usage_policy={
|
|
"automatic_ground_truth": False,
|
|
"ground_truth_allowed": False,
|
|
"training_allowed": True,
|
|
"allowed_tasks": ["imagery_context", "change_context"],
|
|
"validation_authority": {},
|
|
},
|
|
ingest_status="not_configured",
|
|
known_limitations=["Sentinel-2 is contextual imagery, not automatic ground truth for building labels."],
|
|
),
|
|
_source_seed(
|
|
"digitaal_vlaanderen_dhmv",
|
|
"Digitaal Hoogtemodel Vlaanderen",
|
|
"authoritative",
|
|
"Digitaal Vlaanderen",
|
|
authority_scope={"zone": "Flanders", "role": "terrain_height_corroboration"},
|
|
provider_adapter_key="dhmv",
|
|
default_crs="EPSG:31370",
|
|
default_units="m TAW",
|
|
spatial_resolution={"metres": 1},
|
|
temporal_coverage={"period": "2013-2015", "status": "product_specific"},
|
|
geographic_coverage={"zone": "Flanders"},
|
|
expected_attributes={"bands": 1, "nodata_required": True},
|
|
usage_policy={
|
|
"automatic_ground_truth": False,
|
|
"ground_truth_allowed": False,
|
|
"training_allowed": True,
|
|
"allowed_tasks": ["terrain_context", "height_corroboration", "elevation_validation"],
|
|
"validation_authority": {
|
|
"building_validation": "corroborative",
|
|
"elevation_validation": "primary",
|
|
},
|
|
},
|
|
ingest_status="configured",
|
|
known_limitations=["DHMV is height context and cannot independently establish building labels."],
|
|
),
|
|
_source_seed(
|
|
"osm",
|
|
"OpenStreetMap",
|
|
"contextual",
|
|
"OpenStreetMap contributors",
|
|
authority_scope={"scope": "community-maintained", "role": "contextual"},
|
|
provider_adapter_key="osm",
|
|
source_url="https://www.openstreetmap.org",
|
|
license_name="ODbL",
|
|
usage_restrictions="OpenStreetMap attribution and ODbL obligations apply.",
|
|
default_crs="EPSG:4326",
|
|
default_units="mixed",
|
|
geographic_coverage={"scope": "global"},
|
|
expected_geometry_types=["Point", "LineString", "Polygon", "MultiPolygon"],
|
|
expected_attributes={"status": "community_tags"},
|
|
usage_policy={
|
|
"automatic_ground_truth": False,
|
|
"ground_truth_allowed": False,
|
|
"training_allowed": False,
|
|
"allowed_tasks": ["context", "candidate_discovery"],
|
|
"validation_authority": {},
|
|
},
|
|
ingest_status="not_configured",
|
|
known_limitations=["OSM is never automatic ground truth for GeoIntel validation or labels."],
|
|
),
|
|
_source_seed(
|
|
"manual",
|
|
"Handmatige upload",
|
|
"experimental",
|
|
"Operator supplied",
|
|
authority_scope={"scope": "operator supplied", "trust": "unverified"},
|
|
default_crs="unknown",
|
|
default_units="unknown",
|
|
usage_policy=dict(_DEFAULT_USAGE_POLICY),
|
|
ingest_status="configured",
|
|
known_limitations=["Manual uploads remain untrusted until a passed contract and governed provenance are attached."],
|
|
),
|
|
_source_seed(
|
|
"fixture",
|
|
"Test- en demo fixture",
|
|
"experimental",
|
|
"GeoIntel test fixture",
|
|
authority_scope={"scope": "test_only"},
|
|
default_crs="fixture_specific",
|
|
default_units="fixture_specific",
|
|
usage_policy=dict(_DEFAULT_USAGE_POLICY),
|
|
ingest_status="configured",
|
|
known_limitations=["Fixtures must never be presented as official data or used for production training/promotion."],
|
|
),
|
|
_source_seed(
|
|
"map_selection",
|
|
"Afgeleide kaartselectie",
|
|
"derived",
|
|
"GeoIntel derived operation",
|
|
authority_scope={"scope": "derived_from_registered_input"},
|
|
default_crs="EPSG:4326",
|
|
default_units="source_dependent",
|
|
usage_policy=dict(_DEFAULT_USAGE_POLICY),
|
|
known_limitations=["Derived selections inherit no authority beyond their complete lineage edge and source snapshot."],
|
|
),
|
|
_source_seed(
|
|
"derived",
|
|
"Afgeleide dataset",
|
|
"derived",
|
|
"GeoIntel derived operation",
|
|
authority_scope={"scope": "derived_from_registered_input"},
|
|
usage_policy=dict(_DEFAULT_USAGE_POLICY),
|
|
),
|
|
_source_seed(
|
|
"training_label",
|
|
"Afgeleide trainingslabels",
|
|
"derived",
|
|
"GeoIntel reviewed label pipeline",
|
|
authority_scope={"scope": "derived_from_reviewed_source_snapshots"},
|
|
usage_policy=dict(_DEFAULT_USAGE_POLICY),
|
|
known_limitations=["Training labels require complete source lineage and human-review evidence; they inherit no automatic authority."],
|
|
),
|
|
_source_seed(
|
|
"model",
|
|
"Model artifact",
|
|
"experimental",
|
|
"GeoIntel model pipeline",
|
|
authority_scope={"scope": "internal_model_artifact"},
|
|
usage_policy=dict(_DEFAULT_USAGE_POLICY),
|
|
known_limitations=["A model artifact is not a validated capability or promotion decision without its model card and evaluation evidence."],
|
|
),
|
|
_source_seed(
|
|
"experimental",
|
|
"Experimentele bron",
|
|
"experimental",
|
|
"Unverified",
|
|
authority_scope={"scope": "unverified"},
|
|
usage_policy=dict(_DEFAULT_USAGE_POLICY),
|
|
),
|
|
_source_seed(
|
|
"legacy_unknown",
|
|
"Niet-geclassificeerde historische bron",
|
|
"experimental",
|
|
"Legacy import — unverified",
|
|
authority_scope={"scope": "legacy", "trust": "unverified"},
|
|
usage_policy=dict(_DEFAULT_USAGE_POLICY),
|
|
ingest_status="legacy_unverified",
|
|
known_limitations=["Historical source identity is retained descriptively but has no governed authority until re-ingested."],
|
|
),
|
|
_source_seed("ngi_adminvector", "NGI AdminVector", "authoritative", "Nationaal Geografisch Instituut", authority_scope={"scope": "Belgium"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"scope": "Belgium"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy),
|
|
_source_seed("rbins_marine_reporting_units", "RBINS mariene rapportage-eenheden", "authoritative", "RBINS", authority_scope={"zone": "Belgian North Sea"}, default_crs="EPSG:4326", default_units="degrees", geographic_coverage={"zone": "Belgian North Sea"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy),
|
|
_source_seed("rbins_msp_2026", "Belgisch Marien Ruimtelijk Plan 2026-2034", "authoritative", "RBINS", authority_scope={"zone": "Belgian North Sea", "edition": "2026-2034"}, default_crs="EPSG:4326", default_units="degrees", geographic_coverage={"zone": "Belgian North Sea"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy),
|
|
_source_seed("vrbg", "Vlaams Wegenregister", "authoritative", "Digitaal Vlaanderen", authority_scope={"zone": "Flanders", "theme": "roads"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=["LineString", "MultiLineString"], usage_policy=governed_vector_policy),
|
|
_source_seed("digitaal_vlaanderen_orthophoto", "Orthofoto Vlaanderen", "contextual", "Digitaal Vlaanderen", authority_scope={"zone": "Flanders", "role": "imagery"}, default_crs="EPSG:31370", default_units="pixel", spatial_resolution={"metres": 0.25}, geographic_coverage={"zone": "Flanders"}, usage_policy=governed_imagery_policy, ingest_status="configured"),
|
|
_source_seed("spw_orthophoto", "Orthofoto Wallonië", "contextual", "Service public de Wallonie", authority_scope={"zone": "Wallonia", "role": "imagery"}, license_name="CC BY 4.0", default_crs="EPSG:31370", default_units="pixel", spatial_resolution={"metres": 0.25}, geographic_coverage={"zone": "Wallonia"}, usage_policy=governed_imagery_policy, ingest_status="configured"),
|
|
_source_seed("urbis_orthophoto", "Orthofoto Brussel", "contextual", "UrbIS / Brussels Region", authority_scope={"zone": "Brussels-Capital Region", "role": "imagery"}, license_name="CC0", default_crs="EPSG:31370", default_units="pixel", spatial_resolution={"metres": 0.25}, geographic_coverage={"zone": "Brussels-Capital Region"}, usage_policy=governed_imagery_policy, ingest_status="configured"),
|
|
_source_seed("agentschap_landbouw_zeevisserij_agricultural_parcels", "Landbouwgebruikspercelen", "authoritative", "Agentschap Landbouw en Zeevisserij", authority_scope={"zone": "Flanders", "theme": "agricultural_parcels"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy),
|
|
_source_seed("department_omgeving_land_use", "Landgebruik Vlaanderen", "authoritative", "Departement Omgeving", authority_scope={"zone": "Flanders", "theme": "land_use"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy),
|
|
_source_seed("inbo_bwk_natura2000", "BWK en Natura 2000", "authoritative", "INBO", authority_scope={"zone": "Flanders", "theme": "nature"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy),
|
|
_source_seed("statbel", "Statbel bevolking", "authoritative", "Statbel", authority_scope={"scope": "Belgium", "theme": "population"}, default_crs="EPSG:31370", default_units="persons", geographic_coverage={"scope": "Belgium"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy),
|
|
_source_seed("waterinfo", "Waterinfo", "authoritative", "Waterinfo Vlaanderen", authority_scope={"zone": "Flanders", "theme": "water"}, default_crs="EPSG:31370", default_units="source_specific", geographic_coverage={"zone": "Flanders"}, usage_policy=governed_vector_policy),
|
|
_source_seed("department_omgeving_thematic_raster", "Omgeving thematische rasters", "authoritative", "Departement Omgeving", authority_scope={"zone": "Flanders", "theme": "thematic_raster"}, default_crs="EPSG:31370", default_units="source_specific", geographic_coverage={"zone": "Flanders"}, usage_policy=governed_vector_policy),
|
|
_source_seed("dov_soil_map", "DOV bodemkaart", "authoritative", "Databank Ondergrond Vlaanderen", authority_scope={"zone": "Flanders", "theme": "soil"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy),
|
|
_source_seed("vmm_flood_hazard", "VMM overstromingskaarten", "authoritative", "Vlaamse Milieumaatschappij", authority_scope={"zone": "Flanders", "theme": "flood_hazard"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, usage_policy=governed_vector_policy),
|
|
_source_seed("vmm_vha_bathymetry_profiles", "VHA bathymetrieprofielen", "authoritative", "Vlaamse Milieumaatschappij", authority_scope={"zone": "Flanders", "theme": "bathymetry_profiles"}, default_crs="EPSG:31370", default_units="m TAW", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=["Point"], usage_policy=governed_vector_policy),
|
|
_source_seed("historical_landuse", "Historisch landgebruik", "corroborative", "Historical archive provider", authority_scope={"scope": "Belgium", "theme": "historical_land_use"}, default_crs="source_specific", default_units="source_specific", usage_policy=governed_vector_policy),
|
|
_source_seed(
|
|
"spw_geoportail",
|
|
"SPW Geoportail (bronportaal)",
|
|
"authoritative",
|
|
"Service public de Wallonie",
|
|
authority_scope={"zone": "Wallonia", "role": "umbrella_catalogue_and_adapter_authority"},
|
|
provider_adapter_key="spw_geoportail",
|
|
source_url="https://geoportail.wallonie.be/catalogue",
|
|
default_crs="product_specific",
|
|
default_units="product_specific",
|
|
geographic_coverage={"zone": "Wallonia"},
|
|
usage_policy={
|
|
"automatic_ground_truth": False,
|
|
"ground_truth_allowed": False,
|
|
"training_allowed": False,
|
|
"allowed_tasks": ["source_catalogue", "reference_context"],
|
|
"validation_authority": {},
|
|
},
|
|
ingest_status="configured",
|
|
known_limitations=[
|
|
"This umbrella authority is not a product-level ground-truth source.",
|
|
"A governed import must use PICC, WALOUS, terrain, flood, orthophoto or another product-specific key when available.",
|
|
],
|
|
),
|
|
_source_seed("spw_picc", "PICC", "authoritative", "Service public de Wallonie", authority_scope={"zone": "Wallonia", "theme": "topography_buildings"}, license_name="CC BY 4.0", default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Wallonia"}, expected_geometry_types=["Polygon", "MultiPolygon", "LineString", "MultiLineString"], usage_policy=regional_building_policy),
|
|
_source_seed("urbis", "UrbIS", "authoritative", "Brussels Region", authority_scope={"zone": "Brussels-Capital Region", "theme": "topography_buildings"}, license_name="CC0", default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Brussels-Capital Region"}, expected_geometry_types=["Polygon", "MultiPolygon", "LineString", "MultiLineString"], usage_policy=regional_building_policy),
|
|
_source_seed("spw_walous_land_cover", "WALOUS landbedekking", "authoritative", "Service public de Wallonie", authority_scope={"zone": "Wallonia", "theme": "land_cover"}, license_name="CC BY 4.0", default_crs="EPSG:3812", default_units="class_code", spatial_resolution={"metres": 1}, geographic_coverage={"zone": "Wallonia"}, usage_policy=governed_vector_policy),
|
|
_source_seed("spw_bathymetry", "SPW bathymetrie", "authoritative", "Service public de Wallonie", authority_scope={"zone": "Wallonia", "theme": "bathymetry"}, default_crs="EPSG:3812", default_units="mDNG", geographic_coverage={"zone": "Wallonia"}, usage_policy=governed_vector_policy),
|
|
_source_seed("spw_terrain", "SPW terreinmodel", "corroborative", "Service public de Wallonie", authority_scope={"zone": "Wallonia", "theme": "terrain"}, default_crs="EPSG:3812", default_units="metres", spatial_resolution={"metres": 1}, geographic_coverage={"zone": "Wallonia"}, usage_policy=governed_vector_policy),
|
|
_source_seed("spw_flood_hazard", "SPW overstromingsgevaar", "authoritative", "Service public de Wallonie", authority_scope={"zone": "Wallonia", "theme": "flood_hazard"}, default_crs="EPSG:3812", default_units="metres", geographic_coverage={"zone": "Wallonia"}, usage_policy=governed_vector_policy),
|
|
_source_seed("mdk_bathymetry", "MDK bathymetrie", "authoritative", "Maritieme Dienstverlening en Kust", authority_scope={"zone": "Belgian North Sea", "theme": "bathymetry"}, default_crs="EPSG:3812", default_units="metres", geographic_coverage={"zone": "Belgian North Sea"}, usage_policy=governed_vector_policy),
|
|
_source_seed(
|
|
"mdk_bcp_bathymetry",
|
|
"MDK BCP bathymetrie-probe en verwerving",
|
|
"authoritative",
|
|
"Maritieme Dienstverlening en Kust",
|
|
authority_scope={
|
|
"zone": "Belgian North Sea",
|
|
"theme": "bathymetry",
|
|
"role": "coverage_probe_and_governed_acquisition",
|
|
},
|
|
provider_adapter_key="mdk_bcp_bathymetry",
|
|
source_url="https://www.vlaanderen.be/datavindplaats",
|
|
default_crs="EPSG:3812",
|
|
default_units="metres",
|
|
geographic_coverage={"zone": "Belgian North Sea"},
|
|
usage_policy=governed_vector_policy,
|
|
ingest_status="not_configured",
|
|
known_limitations=[
|
|
"A BCP coverage probe is discovery evidence, not a usable bathymetry dataset.",
|
|
"Only a successful governed acquisition with an immutable response checksum may create a source snapshot.",
|
|
],
|
|
),
|
|
]
|
|
|
|
|
|
def _source_registry_table() -> sa.Table:
|
|
return sa.table(
|
|
"source_registry",
|
|
sa.column("id", sa.UUID()),
|
|
sa.column("source_key", sa.String()),
|
|
sa.column("display_name", sa.String()),
|
|
sa.column("classification", sa.String()),
|
|
sa.column("authority_name", sa.String()),
|
|
sa.column("authority_scope_json", sa.JSON()),
|
|
sa.column("provider_adapter_key", sa.String()),
|
|
sa.column("source_url", sa.Text()),
|
|
sa.column("license_name", sa.String()),
|
|
sa.column("license_url", sa.Text()),
|
|
sa.column("usage_restrictions", sa.Text()),
|
|
sa.column("default_crs", sa.String()),
|
|
sa.column("default_units", sa.String()),
|
|
sa.column("spatial_resolution_json", sa.JSON()),
|
|
sa.column("temporal_coverage_json", sa.JSON()),
|
|
sa.column("geographic_coverage_json", sa.JSON()),
|
|
sa.column("expected_geometry_types_json", sa.JSON()),
|
|
sa.column("expected_attributes_json", sa.JSON()),
|
|
sa.column("usage_policy_json", sa.JSON()),
|
|
sa.column("freshness_status", sa.String()),
|
|
sa.column("ingest_status", sa.String()),
|
|
sa.column("known_limitations_json", sa.JSON()),
|
|
sa.column("registry_metadata_json", sa.JSON()),
|
|
)
|
|
|
|
|
|
def _offline_safe_seed_rows() -> list[dict]:
|
|
json_columns = {
|
|
"authority_scope_json",
|
|
"spatial_resolution_json",
|
|
"temporal_coverage_json",
|
|
"geographic_coverage_json",
|
|
"expected_geometry_types_json",
|
|
"expected_attributes_json",
|
|
"usage_policy_json",
|
|
"known_limitations_json",
|
|
"registry_metadata_json",
|
|
}
|
|
rows: list[dict] = []
|
|
for seed in _seed_rows():
|
|
row = dict(seed)
|
|
row["id"] = op.inline_literal(str(seed["id"]), type_=sa.String())
|
|
for column_name in json_columns:
|
|
row[column_name] = op.inline_literal(
|
|
json.dumps(seed[column_name], sort_keys=True, separators=(",", ":")),
|
|
type_=sa.String(),
|
|
)
|
|
rows.append(row)
|
|
return rows
|
|
|
|
|
|
def upgrade() -> None:
|
|
op.create_table(
|
|
"source_registry",
|
|
sa.Column("id", sa.UUID(), primary_key=True),
|
|
sa.Column("source_key", sa.String(length=120), nullable=False),
|
|
sa.Column("display_name", sa.String(length=255), nullable=False),
|
|
sa.Column("classification", sa.String(length=32), nullable=False),
|
|
sa.Column("authority_name", sa.String(length=255), nullable=False, server_default="unknown"),
|
|
sa.Column("authority_scope_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")),
|
|
sa.Column("provider_adapter_key", sa.String(length=120), nullable=True),
|
|
sa.Column("source_url", sa.Text(), nullable=True),
|
|
sa.Column("license_name", sa.String(length=255), nullable=False, server_default="unknown"),
|
|
sa.Column("license_url", sa.Text(), nullable=True),
|
|
sa.Column("usage_restrictions", sa.Text(), nullable=False, server_default="unknown"),
|
|
sa.Column("default_crs", sa.String(length=64), nullable=False, server_default="unknown"),
|
|
sa.Column("default_units", sa.String(length=120), nullable=False, server_default="unknown"),
|
|
sa.Column("spatial_resolution_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")),
|
|
sa.Column("temporal_coverage_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")),
|
|
sa.Column("geographic_coverage_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")),
|
|
sa.Column("expected_geometry_types_json", sa.JSON(), nullable=False, server_default=sa.text("'[]'::json")),
|
|
sa.Column("expected_attributes_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")),
|
|
sa.Column("usage_policy_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")),
|
|
sa.Column("freshness_status", sa.String(length=32), nullable=False, server_default="unknown"),
|
|
sa.Column("ingest_status", sa.String(length=32), nullable=False, server_default="registered"),
|
|
sa.Column("known_limitations_json", sa.JSON(), nullable=False, server_default=sa.text("'[]'::json")),
|
|
sa.Column("registry_metadata_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")),
|
|
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
|
sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
|
sa.UniqueConstraint("source_key", name="uq_source_registry_source_key"),
|
|
sa.CheckConstraint(
|
|
"classification IN ('authoritative', 'corroborative', 'contextual', 'derived', 'experimental')",
|
|
name="ck_source_registry_classification",
|
|
),
|
|
sa.CheckConstraint(
|
|
"freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')",
|
|
name="ck_source_registry_freshness_status",
|
|
),
|
|
sa.CheckConstraint(
|
|
"ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', "
|
|
"'failed', 'quarantined', 'legacy_unverified')",
|
|
name="ck_source_registry_ingest_status",
|
|
),
|
|
)
|
|
op.create_table(
|
|
"source_snapshots",
|
|
sa.Column("id", sa.UUID(), primary_key=True),
|
|
sa.Column("source_registry_id", sa.UUID(), sa.ForeignKey("source_registry.id", ondelete="CASCADE"), nullable=False),
|
|
sa.Column("snapshot_key", sa.String(length=255), nullable=False),
|
|
sa.Column("source_version", sa.String(length=120), nullable=True),
|
|
sa.Column("snapshot_at", sa.DateTime(timezone=True), nullable=True),
|
|
sa.Column("fetched_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
|
|
sa.Column("source_url", sa.Text(), nullable=True),
|
|
sa.Column("checksum_sha256", sa.String(length=64), nullable=False),
|
|
sa.Column("crs", sa.String(length=64), nullable=True),
|
|
sa.Column("units", sa.String(length=120), nullable=True),
|
|
sa.Column("spatial_resolution_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")),
|
|
sa.Column("temporal_coverage_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")),
|
|
sa.Column("geographic_coverage_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")),
|
|
sa.Column("observed_schema_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")),
|
|
sa.Column("freshness_status", sa.String(length=32), nullable=False, server_default="unknown"),
|
|
sa.Column("ingest_status", sa.String(length=32), nullable=False, server_default="registered"),
|
|
sa.Column("known_limitations_json", sa.JSON(), nullable=False, server_default=sa.text("'[]'::json")),
|
|
sa.Column("snapshot_metadata_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")),
|
|
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
|
sa.UniqueConstraint("source_registry_id", "snapshot_key", name="uq_source_snapshots_registry_key"),
|
|
sa.CheckConstraint(
|
|
"freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')",
|
|
name="ck_source_snapshots_freshness_status",
|
|
),
|
|
sa.CheckConstraint(
|
|
"ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', "
|
|
"'failed', 'quarantined', 'legacy_unverified')",
|
|
name="ck_source_snapshots_ingest_status",
|
|
),
|
|
sa.CheckConstraint(
|
|
"checksum_sha256 = lower(checksum_sha256) AND checksum_sha256 ~ '^[0-9a-f]{64}$'",
|
|
name="ck_source_snapshots_checksum_sha256",
|
|
),
|
|
)
|
|
op.create_index("ix_source_snapshots_registry_fetched", "source_snapshots", ["source_registry_id", "fetched_at"])
|
|
op.create_index("ix_source_snapshots_checksum", "source_snapshots", ["checksum_sha256"])
|
|
|
|
op.add_column("datasets", sa.Column("source_registry_id", sa.UUID(), nullable=True))
|
|
op.add_column("datasets", sa.Column("source_snapshot_id", sa.UUID(), nullable=True))
|
|
op.add_column("datasets", sa.Column("ingest_key", sa.String(length=255), nullable=True))
|
|
op.add_column("datasets", sa.Column("data_contract_key", sa.String(length=120), nullable=True))
|
|
op.add_column("datasets", sa.Column("data_contract_version", sa.String(length=64), nullable=True))
|
|
op.add_column(
|
|
"datasets",
|
|
sa.Column(
|
|
"validation_status",
|
|
sa.String(length=32),
|
|
nullable=False,
|
|
server_default="not_validated",
|
|
comment="not_validated | passed | failed",
|
|
),
|
|
)
|
|
op.add_column("datasets", sa.Column("validation_report_json", sa.JSON(), nullable=True))
|
|
op.add_column(
|
|
"datasets",
|
|
sa.Column(
|
|
"provenance_status",
|
|
sa.String(length=32),
|
|
nullable=False,
|
|
server_default="incomplete",
|
|
comment="complete | incomplete | not_applicable",
|
|
),
|
|
)
|
|
op.add_column(
|
|
"datasets",
|
|
sa.Column(
|
|
"lineage_status",
|
|
sa.String(length=32),
|
|
nullable=False,
|
|
server_default="incomplete",
|
|
comment="complete | incomplete | not_applicable",
|
|
),
|
|
)
|
|
op.add_column(
|
|
"datasets",
|
|
sa.Column(
|
|
"quarantine_status",
|
|
sa.String(length=32),
|
|
nullable=False,
|
|
server_default="not_quarantined",
|
|
comment="not_quarantined | quarantined",
|
|
),
|
|
)
|
|
op.add_column("dataset_versions", sa.Column("source_registry_id", sa.UUID(), nullable=True))
|
|
op.add_column("dataset_versions", sa.Column("source_snapshot_id", sa.UUID(), nullable=True))
|
|
op.add_column("dataset_versions", sa.Column("ingest_key", sa.String(length=255), nullable=True))
|
|
op.add_column("dataset_versions", sa.Column("data_contract_key", sa.String(length=120), nullable=True))
|
|
op.add_column("dataset_versions", sa.Column("data_contract_version", sa.String(length=64), nullable=True))
|
|
op.add_column(
|
|
"dataset_versions",
|
|
sa.Column(
|
|
"validation_status",
|
|
sa.String(length=32),
|
|
nullable=False,
|
|
server_default="not_validated",
|
|
comment="not_validated | passed | failed",
|
|
),
|
|
)
|
|
op.add_column("dataset_versions", sa.Column("validation_report_json", sa.JSON(), nullable=True))
|
|
op.add_column(
|
|
"dataset_versions",
|
|
sa.Column(
|
|
"provenance_status",
|
|
sa.String(length=32),
|
|
nullable=False,
|
|
server_default="incomplete",
|
|
comment="complete | incomplete | not_applicable",
|
|
),
|
|
)
|
|
op.add_column(
|
|
"dataset_versions",
|
|
sa.Column(
|
|
"lineage_status",
|
|
sa.String(length=32),
|
|
nullable=False,
|
|
server_default="incomplete",
|
|
comment="complete | incomplete | not_applicable",
|
|
),
|
|
)
|
|
|
|
op.create_foreign_key("fk_datasets_source_registry", "datasets", "source_registry", ["source_registry_id"], ["id"], ondelete="SET NULL")
|
|
op.create_foreign_key("fk_datasets_source_snapshot", "datasets", "source_snapshots", ["source_snapshot_id"], ["id"], ondelete="SET NULL")
|
|
op.create_foreign_key("fk_dataset_versions_source_registry", "dataset_versions", "source_registry", ["source_registry_id"], ["id"], ondelete="SET NULL")
|
|
op.create_foreign_key("fk_dataset_versions_source_snapshot", "dataset_versions", "source_snapshots", ["source_snapshot_id"], ["id"], ondelete="SET NULL")
|
|
op.create_index("ix_datasets_source_registry_snapshot", "datasets", ["source_registry_id", "source_snapshot_id"])
|
|
op.create_index("ix_dataset_versions_source_registry_snapshot", "dataset_versions", ["source_registry_id", "source_snapshot_id"])
|
|
op.create_unique_constraint("uq_datasets_project_ingest_key", "datasets", ["project_id", "ingest_key"])
|
|
op.create_unique_constraint("uq_dataset_versions_dataset_ingest_key", "dataset_versions", ["dataset_id", "ingest_key"])
|
|
op.create_check_constraint("ck_datasets_validation_status", "datasets", "validation_status IN ('not_validated', 'passed', 'failed')")
|
|
op.create_check_constraint("ck_datasets_provenance_status", "datasets", "provenance_status IN ('complete', 'incomplete', 'not_applicable')")
|
|
op.create_check_constraint("ck_datasets_lineage_status", "datasets", "lineage_status IN ('complete', 'incomplete', 'not_applicable')")
|
|
op.create_check_constraint("ck_datasets_quarantine_status", "datasets", "quarantine_status IN ('not_quarantined', 'quarantined')")
|
|
op.create_check_constraint("ck_datasets_ingest_key_not_blank", "datasets", "ingest_key IS NULL OR btrim(ingest_key) <> ''")
|
|
op.create_check_constraint("ck_dataset_versions_validation_status", "dataset_versions", "validation_status IN ('not_validated', 'passed', 'failed')")
|
|
op.create_check_constraint("ck_dataset_versions_provenance_status", "dataset_versions", "provenance_status IN ('complete', 'incomplete', 'not_applicable')")
|
|
op.create_check_constraint("ck_dataset_versions_lineage_status", "dataset_versions", "lineage_status IN ('complete', 'incomplete', 'not_applicable')")
|
|
op.create_check_constraint("ck_dataset_versions_ingest_key_not_blank", "dataset_versions", "ingest_key IS NULL OR btrim(ingest_key) <> ''")
|
|
|
|
op.create_table(
|
|
"dataset_lineage_edges",
|
|
sa.Column("id", sa.UUID(), primary_key=True),
|
|
sa.Column("parent_dataset_id", sa.UUID(), sa.ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False),
|
|
sa.Column("child_dataset_id", sa.UUID(), sa.ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False),
|
|
sa.Column("parent_dataset_version_id", sa.UUID(), sa.ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True),
|
|
sa.Column("child_dataset_version_id", sa.UUID(), sa.ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True),
|
|
sa.Column("relation_type", sa.String(length=64), nullable=False),
|
|
sa.Column("transformation_name", sa.String(length=255), nullable=False),
|
|
sa.Column("transformation_version", sa.String(length=120), nullable=True),
|
|
sa.Column("parameters_json", sa.JSON(), nullable=True),
|
|
sa.Column("input_checksum_sha256", sa.String(length=64), nullable=True),
|
|
sa.Column("output_checksum_sha256", sa.String(length=64), nullable=True),
|
|
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
|
sa.CheckConstraint("parent_dataset_id <> child_dataset_id", name="ck_dataset_lineage_edges_distinct_datasets"),
|
|
sa.UniqueConstraint("parent_dataset_id", "child_dataset_id", "relation_type", "transformation_name", name="uq_dataset_lineage_edges_relation"),
|
|
)
|
|
op.create_index("ix_dataset_lineage_edges_parent", "dataset_lineage_edges", ["parent_dataset_id"])
|
|
op.create_index("ix_dataset_lineage_edges_child", "dataset_lineage_edges", ["child_dataset_id"])
|
|
op.create_table(
|
|
"dataset_quarantines",
|
|
sa.Column("id", sa.UUID(), primary_key=True),
|
|
sa.Column("dataset_id", sa.UUID(), sa.ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True),
|
|
sa.Column("dataset_version_id", sa.UUID(), sa.ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True),
|
|
sa.Column("source_snapshot_id", sa.UUID(), sa.ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True),
|
|
sa.Column("stage", sa.String(length=64), nullable=False),
|
|
sa.Column("reason_code", sa.String(length=120), nullable=False),
|
|
sa.Column("details_json", sa.JSON(), nullable=True),
|
|
sa.Column("artifact_path", sa.Text(), nullable=True),
|
|
sa.Column("artifact_checksum_sha256", sa.String(length=64), nullable=True),
|
|
sa.Column("status", sa.String(length=32), nullable=False, server_default="quarantined"),
|
|
sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()),
|
|
sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True),
|
|
sa.Column("resolved_by", sa.String(length=120), nullable=True),
|
|
sa.CheckConstraint(
|
|
"dataset_id IS NOT NULL OR dataset_version_id IS NOT NULL OR source_snapshot_id IS NOT NULL",
|
|
name="ck_dataset_quarantines_target_present",
|
|
),
|
|
sa.CheckConstraint("status IN ('quarantined', 'released', 'rejected')", name="ck_dataset_quarantines_status"),
|
|
)
|
|
op.create_index("ix_dataset_quarantines_dataset_status", "dataset_quarantines", ["dataset_id", "status"])
|
|
op.create_index("ix_dataset_quarantines_snapshot_status", "dataset_quarantines", ["source_snapshot_id", "status"])
|
|
|
|
op.bulk_insert(_source_registry_table(), _offline_safe_seed_rows(), multiinsert=False)
|
|
op.execute(
|
|
sa.text(
|
|
"""
|
|
UPDATE datasets
|
|
SET validation_status = 'not_validated',
|
|
provenance_status = 'incomplete',
|
|
lineage_status = 'incomplete',
|
|
quarantine_status = 'not_quarantined'
|
|
"""
|
|
)
|
|
)
|
|
op.execute(
|
|
sa.text(
|
|
"""
|
|
UPDATE datasets AS dataset
|
|
SET source_registry_id = registry.id
|
|
FROM source_registry AS registry
|
|
WHERE registry.source_key = COALESCE(
|
|
NULLIF(lower(btrim(dataset.source_name)), ''),
|
|
NULLIF(lower(btrim(dataset.source)), ''),
|
|
'__unregistered_legacy_source__'
|
|
)
|
|
"""
|
|
)
|
|
)
|
|
op.execute(
|
|
sa.text(
|
|
"""
|
|
UPDATE dataset_versions AS version
|
|
SET source_registry_id = dataset.source_registry_id,
|
|
validation_status = 'not_validated',
|
|
provenance_status = 'incomplete',
|
|
lineage_status = 'incomplete'
|
|
FROM datasets AS dataset
|
|
WHERE version.dataset_id = dataset.id
|
|
"""
|
|
)
|
|
)
|
|
# A Dataset has separate foreign keys to its source registry and immutable
|
|
# snapshot. Those independent FKs alone cannot prove the snapshot belongs
|
|
# to the selected registry. Keep legacy rows with no snapshot valid, but
|
|
# reject every new or mutated mismatched pair at the database boundary.
|
|
# This mirrors the fail-closed application consumption gate and also
|
|
# protects maintenance scripts that bypass the ORM.
|
|
op.execute(
|
|
sa.text(
|
|
"""
|
|
CREATE FUNCTION geointel_phase2_snapshot_registry_guard()
|
|
RETURNS trigger AS $$
|
|
DECLARE
|
|
snapshot_registry uuid;
|
|
BEGIN
|
|
IF NEW.source_snapshot_id IS NULL THEN
|
|
RETURN NEW;
|
|
END IF;
|
|
|
|
SELECT source_registry_id
|
|
INTO snapshot_registry
|
|
FROM source_snapshots
|
|
WHERE id = NEW.source_snapshot_id;
|
|
|
|
IF snapshot_registry IS NULL OR NEW.source_registry_id IS NULL
|
|
OR NEW.source_registry_id <> snapshot_registry THEN
|
|
RAISE EXCEPTION
|
|
'source snapshot % does not belong to source registry %',
|
|
NEW.source_snapshot_id,
|
|
NEW.source_registry_id
|
|
USING ERRCODE = '23514';
|
|
END IF;
|
|
RETURN NEW;
|
|
END;
|
|
$$ LANGUAGE plpgsql;
|
|
|
|
CREATE TRIGGER trg_datasets_snapshot_registry_guard
|
|
BEFORE INSERT OR UPDATE OF source_registry_id, source_snapshot_id ON datasets
|
|
FOR EACH ROW EXECUTE FUNCTION geointel_phase2_snapshot_registry_guard();
|
|
|
|
CREATE TRIGGER trg_dataset_versions_snapshot_registry_guard
|
|
BEFORE INSERT OR UPDATE OF source_registry_id, source_snapshot_id ON dataset_versions
|
|
FOR EACH ROW EXECUTE FUNCTION geointel_phase2_snapshot_registry_guard();
|
|
|
|
CREATE FUNCTION geointel_phase2_snapshot_registry_immutable_guard()
|
|
RETURNS trigger AS $$
|
|
BEGIN
|
|
IF NEW.source_registry_id IS DISTINCT FROM OLD.source_registry_id THEN
|
|
RAISE EXCEPTION
|
|
'source snapshot registry identity is immutable'
|
|
USING ERRCODE = '23514';
|
|
END IF;
|
|
RETURN NEW;
|
|
END;
|
|
$$ LANGUAGE plpgsql;
|
|
|
|
CREATE TRIGGER trg_source_snapshots_registry_immutable
|
|
BEFORE UPDATE OF source_registry_id ON source_snapshots
|
|
FOR EACH ROW EXECUTE FUNCTION geointel_phase2_snapshot_registry_immutable_guard();
|
|
|
|
CREATE FUNCTION geointel_phase2_source_registry_write_guard()
|
|
RETURNS trigger AS $$
|
|
BEGIN
|
|
RAISE EXCEPTION 'source registry entries are server-owned and immutable'
|
|
USING ERRCODE = '23514';
|
|
END;
|
|
$$ LANGUAGE plpgsql;
|
|
|
|
CREATE TRIGGER trg_source_registry_write_guard
|
|
BEFORE UPDATE OR DELETE ON source_registry
|
|
FOR EACH ROW EXECUTE FUNCTION geointel_phase2_source_registry_write_guard();
|
|
|
|
CREATE FUNCTION geointel_phase2_snapshot_evidence_immutable_guard()
|
|
RETURNS trigger AS $$
|
|
BEGIN
|
|
IF TG_OP = 'DELETE' THEN
|
|
RAISE EXCEPTION 'source snapshots are immutable evidence and cannot be deleted'
|
|
USING ERRCODE = '23514';
|
|
END IF;
|
|
|
|
IF NEW.source_registry_id IS DISTINCT FROM OLD.source_registry_id
|
|
OR NEW.snapshot_key IS DISTINCT FROM OLD.snapshot_key
|
|
OR NEW.source_version IS DISTINCT FROM OLD.source_version
|
|
OR NEW.snapshot_at IS DISTINCT FROM OLD.snapshot_at
|
|
OR NEW.fetched_at IS DISTINCT FROM OLD.fetched_at
|
|
OR NEW.source_url IS DISTINCT FROM OLD.source_url
|
|
OR NEW.checksum_sha256 IS DISTINCT FROM OLD.checksum_sha256
|
|
OR NEW.crs IS DISTINCT FROM OLD.crs
|
|
OR NEW.units IS DISTINCT FROM OLD.units
|
|
OR NEW.spatial_resolution_json::text IS DISTINCT FROM OLD.spatial_resolution_json::text
|
|
OR NEW.temporal_coverage_json::text IS DISTINCT FROM OLD.temporal_coverage_json::text
|
|
OR NEW.geographic_coverage_json::text IS DISTINCT FROM OLD.geographic_coverage_json::text
|
|
OR NEW.observed_schema_json::text IS DISTINCT FROM OLD.observed_schema_json::text
|
|
OR NEW.freshness_status IS DISTINCT FROM OLD.freshness_status
|
|
OR NEW.known_limitations_json::text IS DISTINCT FROM OLD.known_limitations_json::text
|
|
OR NEW.snapshot_metadata_json::text IS DISTINCT FROM OLD.snapshot_metadata_json::text THEN
|
|
RAISE EXCEPTION 'source snapshot evidence is immutable'
|
|
USING ERRCODE = '23514';
|
|
END IF;
|
|
IF NEW.ingest_status IS DISTINCT FROM OLD.ingest_status
|
|
AND NEW.ingest_status <> 'quarantined' THEN
|
|
RAISE EXCEPTION 'source snapshot lifecycle may only transition to quarantined'
|
|
USING ERRCODE = '23514';
|
|
END IF;
|
|
RETURN NEW;
|
|
END;
|
|
$$ LANGUAGE plpgsql;
|
|
|
|
CREATE TRIGGER trg_source_snapshots_evidence_immutable
|
|
BEFORE UPDATE OR DELETE ON source_snapshots
|
|
FOR EACH ROW EXECUTE FUNCTION geointel_phase2_snapshot_evidence_immutable_guard();
|
|
|
|
CREATE FUNCTION geointel_phase2_contract_report_guard()
|
|
RETURNS trigger AS $$
|
|
DECLARE
|
|
snapshot_checksum text;
|
|
BEGIN
|
|
IF NEW.validation_status = 'passed' THEN
|
|
IF NEW.data_contract_key IS NULL OR btrim(NEW.data_contract_key) = ''
|
|
OR NEW.data_contract_version IS NULL OR btrim(NEW.data_contract_version) = ''
|
|
OR NEW.source_registry_id IS NULL OR NEW.source_snapshot_id IS NULL
|
|
OR NEW.validation_report_json IS NULL
|
|
OR json_typeof(NEW.validation_report_json) <> 'object'
|
|
OR COALESCE(NEW.validation_report_json ->> 'validation_status', '') <> 'passed'
|
|
OR COALESCE(NEW.validation_report_json ->> 'data_contract_key', '') <> NEW.data_contract_key
|
|
OR COALESCE(NEW.validation_report_json ->> 'data_contract_version', '') <> NEW.data_contract_version
|
|
OR COALESCE(NEW.validation_report_json ->> 'provenance_status', '') <> 'complete'
|
|
OR COALESCE(NEW.validation_report_json ->> 'lineage_status', '') NOT IN ('complete', 'not_applicable')
|
|
OR COALESCE(NEW.validation_report_json ->> 'quarantine_status', '') <> 'not_quarantined'
|
|
OR COALESCE(NEW.validation_report_json ->> 'contract_fingerprint_sha256', '') !~ '^[0-9a-f]{64}$'
|
|
OR COALESCE(NEW.validation_report_json ->> 'report_sha256', '') !~ '^[0-9a-f]{64}$'
|
|
OR NEW.provenance_status <> 'complete'
|
|
OR NEW.lineage_status NOT IN ('complete', 'not_applicable') THEN
|
|
RAISE EXCEPTION 'passed dataset contract state requires a matching complete validation report'
|
|
USING ERRCODE = '23514';
|
|
END IF;
|
|
IF NEW.quarantine_status <> 'not_quarantined' THEN
|
|
RAISE EXCEPTION 'passed dataset cannot be quarantined'
|
|
USING ERRCODE = '23514';
|
|
END IF;
|
|
|
|
SELECT checksum_sha256
|
|
INTO snapshot_checksum
|
|
FROM source_snapshots
|
|
WHERE id = NEW.source_snapshot_id;
|
|
|
|
IF NEW.checksum_sha256 IS NULL
|
|
OR NEW.checksum_sha256 <> lower(NEW.checksum_sha256)
|
|
OR NEW.checksum_sha256 !~ '^[0-9a-f]{64}$'
|
|
OR snapshot_checksum IS NULL
|
|
OR NEW.checksum_sha256 <> snapshot_checksum THEN
|
|
RAISE EXCEPTION 'passed dataset checksum must be a canonical SHA-256 bound to its source snapshot'
|
|
USING ERRCODE = '23514';
|
|
END IF;
|
|
END IF;
|
|
|
|
-- A passed record is bound to exact bytes and the source/time
|
|
-- evidence that was validated with those bytes. A caller may
|
|
-- invalidate it, but cannot atomically replace any evidence
|
|
-- while doing so: invalidate first, then governed re-ingest.
|
|
IF TG_OP = 'UPDATE' AND OLD.validation_status = 'passed' AND (
|
|
NEW.source IS DISTINCT FROM OLD.source
|
|
OR NEW.dataset_type IS DISTINCT FROM OLD.dataset_type
|
|
OR NEW.storage_path IS DISTINCT FROM OLD.storage_path
|
|
OR NEW.original_filename IS DISTINCT FROM OLD.original_filename
|
|
OR NEW.stored_filename IS DISTINCT FROM OLD.stored_filename
|
|
OR NEW.content_type IS DISTINCT FROM OLD.content_type
|
|
OR NEW.size_bytes IS DISTINCT FROM OLD.size_bytes
|
|
OR NEW.checksum_sha256 IS DISTINCT FROM OLD.checksum_sha256
|
|
OR NEW.ingest_key IS DISTINCT FROM OLD.ingest_key
|
|
OR NEW.derived_from_dataset_id IS DISTINCT FROM OLD.derived_from_dataset_id
|
|
OR NEW.crs IS DISTINCT FROM OLD.crs
|
|
OR NEW.bounds_json::text IS DISTINCT FROM OLD.bounds_json::text
|
|
OR NEW.resolution_json::text IS DISTINCT FROM OLD.resolution_json::text
|
|
OR NEW.bands_json::text IS DISTINCT FROM OLD.bands_json::text
|
|
OR NEW.metadata_json::text IS DISTINCT FROM OLD.metadata_json::text
|
|
OR NEW.dataset_role IS DISTINCT FROM OLD.dataset_role
|
|
OR NEW.source_name IS DISTINCT FROM OLD.source_name
|
|
OR NEW.reference_layer_name IS DISTINCT FROM OLD.reference_layer_name
|
|
OR NEW.source_metadata::text IS DISTINCT FROM OLD.source_metadata::text
|
|
OR NEW.provenance_metadata::text IS DISTINCT FROM OLD.provenance_metadata::text
|
|
OR NEW.source_registry_id IS DISTINCT FROM OLD.source_registry_id
|
|
OR NEW.source_snapshot_id IS DISTINCT FROM OLD.source_snapshot_id
|
|
OR NEW.data_contract_key IS DISTINCT FROM OLD.data_contract_key
|
|
OR NEW.data_contract_version IS DISTINCT FROM OLD.data_contract_version
|
|
OR NEW.validation_report_json::text IS DISTINCT FROM OLD.validation_report_json::text
|
|
OR NEW.temporal_series_key IS DISTINCT FROM OLD.temporal_series_key
|
|
OR NEW.observed_at IS DISTINCT FROM OLD.observed_at
|
|
OR NEW.valid_from IS DISTINCT FROM OLD.valid_from
|
|
OR NEW.valid_to IS DISTINCT FROM OLD.valid_to
|
|
OR NEW.temporal_granularity IS DISTINCT FROM OLD.temporal_granularity
|
|
OR NEW.source_version IS DISTINCT FROM OLD.source_version
|
|
) THEN
|
|
RAISE EXCEPTION 'accepted dataset artifact and provenance evidence is immutable; invalidate it before a governed re-ingest'
|
|
USING ERRCODE = '23514';
|
|
END IF;
|
|
|
|
IF TG_OP = 'UPDATE'
|
|
AND OLD.quarantine_status = 'quarantined'
|
|
AND NEW.quarantine_status <> 'quarantined' THEN
|
|
RAISE EXCEPTION 'quarantined dataset cannot be released in place; a governed re-ingest is required'
|
|
USING ERRCODE = '23514';
|
|
END IF;
|
|
RETURN NEW;
|
|
END;
|
|
$$ LANGUAGE plpgsql;
|
|
|
|
CREATE FUNCTION geointel_phase2_dataset_version_contract_report_guard()
|
|
RETURNS trigger AS $$
|
|
DECLARE
|
|
snapshot_checksum text;
|
|
BEGIN
|
|
IF NEW.validation_status = 'passed' THEN
|
|
IF NEW.data_contract_key IS NULL OR btrim(NEW.data_contract_key) = ''
|
|
OR NEW.data_contract_version IS NULL OR btrim(NEW.data_contract_version) = ''
|
|
OR NEW.source_registry_id IS NULL OR NEW.source_snapshot_id IS NULL
|
|
OR NEW.validation_report_json IS NULL
|
|
OR json_typeof(NEW.validation_report_json) <> 'object'
|
|
OR COALESCE(NEW.validation_report_json ->> 'validation_status', '') <> 'passed'
|
|
OR COALESCE(NEW.validation_report_json ->> 'data_contract_key', '') <> NEW.data_contract_key
|
|
OR COALESCE(NEW.validation_report_json ->> 'data_contract_version', '') <> NEW.data_contract_version
|
|
OR COALESCE(NEW.validation_report_json ->> 'provenance_status', '') <> 'complete'
|
|
OR COALESCE(NEW.validation_report_json ->> 'lineage_status', '') NOT IN ('complete', 'not_applicable')
|
|
OR COALESCE(NEW.validation_report_json ->> 'quarantine_status', '') <> 'not_quarantined'
|
|
OR COALESCE(NEW.validation_report_json ->> 'contract_fingerprint_sha256', '') !~ '^[0-9a-f]{64}$'
|
|
OR COALESCE(NEW.validation_report_json ->> 'report_sha256', '') !~ '^[0-9a-f]{64}$'
|
|
OR NEW.provenance_status <> 'complete'
|
|
OR NEW.lineage_status NOT IN ('complete', 'not_applicable') THEN
|
|
RAISE EXCEPTION 'passed dataset version contract state requires a matching complete validation report'
|
|
USING ERRCODE = '23514';
|
|
END IF;
|
|
|
|
SELECT checksum_sha256
|
|
INTO snapshot_checksum
|
|
FROM source_snapshots
|
|
WHERE id = NEW.source_snapshot_id;
|
|
|
|
IF NEW.checksum_sha256 IS NULL
|
|
OR NEW.checksum_sha256 <> lower(NEW.checksum_sha256)
|
|
OR NEW.checksum_sha256 !~ '^[0-9a-f]{64}$'
|
|
OR snapshot_checksum IS NULL
|
|
OR NEW.checksum_sha256 <> snapshot_checksum THEN
|
|
RAISE EXCEPTION 'passed dataset version checksum must be a canonical SHA-256 bound to its source snapshot'
|
|
USING ERRCODE = '23514';
|
|
END IF;
|
|
END IF;
|
|
|
|
IF TG_OP = 'UPDATE' AND OLD.validation_status = 'passed' AND (
|
|
NEW.dataset_id IS DISTINCT FROM OLD.dataset_id
|
|
OR NEW.version IS DISTINCT FROM OLD.version
|
|
OR NEW.storage_path IS DISTINCT FROM OLD.storage_path
|
|
OR NEW.source_version IS DISTINCT FROM OLD.source_version
|
|
OR NEW.observed_at IS DISTINCT FROM OLD.observed_at
|
|
OR NEW.valid_from IS DISTINCT FROM OLD.valid_from
|
|
OR NEW.valid_to IS DISTINCT FROM OLD.valid_to
|
|
OR NEW.checksum_sha256 IS DISTINCT FROM OLD.checksum_sha256
|
|
OR NEW.ingest_key IS DISTINCT FROM OLD.ingest_key
|
|
OR NEW.source_metadata::text IS DISTINCT FROM OLD.source_metadata::text
|
|
OR NEW.provenance_metadata::text IS DISTINCT FROM OLD.provenance_metadata::text
|
|
OR NEW.source_registry_id IS DISTINCT FROM OLD.source_registry_id
|
|
OR NEW.source_snapshot_id IS DISTINCT FROM OLD.source_snapshot_id
|
|
OR NEW.data_contract_key IS DISTINCT FROM OLD.data_contract_key
|
|
OR NEW.data_contract_version IS DISTINCT FROM OLD.data_contract_version
|
|
OR NEW.validation_report_json::text IS DISTINCT FROM OLD.validation_report_json::text
|
|
) THEN
|
|
RAISE EXCEPTION 'accepted dataset-version artifact and provenance evidence is immutable; invalidate it before a governed re-ingest'
|
|
USING ERRCODE = '23514';
|
|
END IF;
|
|
RETURN NEW;
|
|
END;
|
|
$$ LANGUAGE plpgsql;
|
|
|
|
CREATE TRIGGER trg_datasets_contract_report_guard
|
|
BEFORE INSERT OR UPDATE OF source, dataset_type, storage_path,
|
|
original_filename, stored_filename, content_type, size_bytes,
|
|
checksum_sha256, ingest_key, derived_from_dataset_id, crs,
|
|
bounds_json, resolution_json, bands_json, metadata_json,
|
|
dataset_role, source_name, reference_layer_name, source_metadata,
|
|
provenance_metadata, source_registry_id, source_snapshot_id,
|
|
data_contract_key, data_contract_version, validation_status,
|
|
validation_report_json, provenance_status, lineage_status,
|
|
quarantine_status, temporal_series_key, observed_at, valid_from,
|
|
valid_to, temporal_granularity, source_version ON datasets
|
|
FOR EACH ROW EXECUTE FUNCTION geointel_phase2_contract_report_guard();
|
|
|
|
CREATE TRIGGER trg_dataset_versions_contract_report_guard
|
|
BEFORE INSERT OR UPDATE OF dataset_id, version, storage_path,
|
|
source_version, observed_at, valid_from, valid_to, checksum_sha256,
|
|
ingest_key, source_metadata, provenance_metadata, source_registry_id,
|
|
source_snapshot_id, data_contract_key, data_contract_version,
|
|
validation_status, validation_report_json, provenance_status,
|
|
lineage_status ON dataset_versions
|
|
FOR EACH ROW EXECUTE FUNCTION geointel_phase2_dataset_version_contract_report_guard();
|
|
|
|
CREATE FUNCTION geointel_phase2_lineage_cycle_guard()
|
|
RETURNS trigger AS $$
|
|
BEGIN
|
|
IF NEW.parent_dataset_id = NEW.child_dataset_id THEN
|
|
RAISE EXCEPTION 'a dataset cannot be its own lineage parent'
|
|
USING ERRCODE = '23514';
|
|
END IF;
|
|
|
|
IF EXISTS (
|
|
WITH RECURSIVE descendants(dataset_id) AS (
|
|
SELECT edge.child_dataset_id
|
|
FROM dataset_lineage_edges AS edge
|
|
WHERE edge.parent_dataset_id = NEW.child_dataset_id
|
|
AND (TG_OP <> 'UPDATE' OR edge.id <> NEW.id)
|
|
UNION
|
|
SELECT edge.child_dataset_id
|
|
FROM dataset_lineage_edges AS edge
|
|
JOIN descendants ON edge.parent_dataset_id = descendants.dataset_id
|
|
WHERE TG_OP <> 'UPDATE' OR edge.id <> NEW.id
|
|
)
|
|
SELECT 1 FROM descendants WHERE dataset_id = NEW.parent_dataset_id
|
|
) THEN
|
|
RAISE EXCEPTION
|
|
'lineage edge % -> % would create a cycle',
|
|
NEW.parent_dataset_id,
|
|
NEW.child_dataset_id
|
|
USING ERRCODE = '23514';
|
|
END IF;
|
|
RETURN NEW;
|
|
END;
|
|
$$ LANGUAGE plpgsql;
|
|
|
|
CREATE TRIGGER trg_dataset_lineage_edges_cycle_guard
|
|
BEFORE INSERT ON dataset_lineage_edges
|
|
FOR EACH ROW EXECUTE FUNCTION geointel_phase2_lineage_cycle_guard();
|
|
|
|
CREATE FUNCTION geointel_phase2_lineage_edge_immutable_guard()
|
|
RETURNS trigger AS $$
|
|
BEGIN
|
|
RAISE EXCEPTION 'dataset lineage edges are immutable evidence and cannot be updated or deleted'
|
|
USING ERRCODE = '23514';
|
|
END;
|
|
$$ LANGUAGE plpgsql;
|
|
|
|
CREATE TRIGGER trg_dataset_lineage_edges_immutable
|
|
BEFORE UPDATE OR DELETE ON dataset_lineage_edges
|
|
FOR EACH ROW EXECUTE FUNCTION geointel_phase2_lineage_edge_immutable_guard();
|
|
|
|
CREATE FUNCTION geointel_phase2_quarantine_lineage_descendants(
|
|
root_dataset_id uuid,
|
|
root_snapshot_id uuid
|
|
)
|
|
RETURNS void AS $$
|
|
BEGIN
|
|
-- A shared source snapshot has several direct Dataset roots;
|
|
-- each root and every descendant must be made non-consumable.
|
|
-- UNION (rather than UNION ALL) keeps corrupt historic cycles
|
|
-- finite while the immutable cycle guard prevents new ones.
|
|
WITH RECURSIVE quarantine_roots(dataset_id) AS (
|
|
SELECT root_dataset_id WHERE root_dataset_id IS NOT NULL
|
|
UNION
|
|
SELECT dataset.id
|
|
FROM datasets AS dataset
|
|
WHERE root_snapshot_id IS NOT NULL
|
|
AND dataset.source_snapshot_id = root_snapshot_id
|
|
), descendants(dataset_id) AS (
|
|
SELECT dataset_id FROM quarantine_roots
|
|
UNION
|
|
SELECT edge.child_dataset_id
|
|
FROM dataset_lineage_edges AS edge
|
|
JOIN descendants AS upstream ON edge.parent_dataset_id = upstream.dataset_id
|
|
)
|
|
UPDATE dataset_versions AS version
|
|
SET validation_status = 'failed',
|
|
provenance_status = 'incomplete',
|
|
lineage_status = 'incomplete'
|
|
WHERE version.dataset_id IN (SELECT dataset_id FROM descendants);
|
|
|
|
WITH RECURSIVE quarantine_roots(dataset_id) AS (
|
|
SELECT root_dataset_id WHERE root_dataset_id IS NOT NULL
|
|
UNION
|
|
SELECT dataset.id
|
|
FROM datasets AS dataset
|
|
WHERE root_snapshot_id IS NOT NULL
|
|
AND dataset.source_snapshot_id = root_snapshot_id
|
|
), descendants(dataset_id) AS (
|
|
SELECT dataset_id FROM quarantine_roots
|
|
UNION
|
|
SELECT edge.child_dataset_id
|
|
FROM dataset_lineage_edges AS edge
|
|
JOIN descendants AS upstream ON edge.parent_dataset_id = upstream.dataset_id
|
|
)
|
|
UPDATE datasets AS dataset
|
|
SET status = 'quarantined',
|
|
quarantine_status = 'quarantined',
|
|
validation_status = 'failed',
|
|
provenance_status = 'incomplete',
|
|
lineage_status = 'incomplete'
|
|
WHERE dataset.id IN (SELECT dataset_id FROM descendants);
|
|
END;
|
|
$$ LANGUAGE plpgsql;
|
|
|
|
CREATE FUNCTION geointel_phase2_quarantine_state_guard()
|
|
RETURNS trigger AS $$
|
|
DECLARE
|
|
owner_dataset uuid;
|
|
owner_snapshot uuid;
|
|
dataset_snapshot uuid;
|
|
BEGIN
|
|
IF NEW.status <> 'quarantined' THEN
|
|
RETURN NEW;
|
|
END IF;
|
|
|
|
IF NEW.dataset_version_id IS NOT NULL THEN
|
|
SELECT dataset_id, source_snapshot_id
|
|
INTO owner_dataset, owner_snapshot
|
|
FROM dataset_versions
|
|
WHERE id = NEW.dataset_version_id;
|
|
|
|
IF owner_dataset IS NULL THEN
|
|
RAISE EXCEPTION 'quarantine dataset version % has no owning dataset', NEW.dataset_version_id
|
|
USING ERRCODE = '23514';
|
|
END IF;
|
|
IF NEW.dataset_id IS NOT NULL AND NEW.dataset_id <> owner_dataset THEN
|
|
RAISE EXCEPTION 'quarantine dataset and dataset version do not belong together'
|
|
USING ERRCODE = '23514';
|
|
END IF;
|
|
|
|
UPDATE dataset_versions
|
|
SET validation_status = 'failed',
|
|
provenance_status = 'incomplete',
|
|
lineage_status = 'incomplete'
|
|
WHERE id = NEW.dataset_version_id;
|
|
END IF;
|
|
|
|
owner_dataset := COALESCE(NEW.dataset_id, owner_dataset);
|
|
IF owner_dataset IS NOT NULL THEN
|
|
SELECT source_snapshot_id
|
|
INTO dataset_snapshot
|
|
FROM datasets
|
|
WHERE id = owner_dataset;
|
|
|
|
IF NEW.dataset_version_id IS NULL
|
|
AND NEW.source_snapshot_id IS NOT NULL
|
|
AND dataset_snapshot IS NOT NULL
|
|
AND NEW.source_snapshot_id <> dataset_snapshot THEN
|
|
RAISE EXCEPTION 'quarantine dataset and source snapshot do not belong together'
|
|
USING ERRCODE = '23514';
|
|
END IF;
|
|
IF NEW.dataset_version_id IS NOT NULL
|
|
AND NEW.source_snapshot_id IS NOT NULL
|
|
AND owner_snapshot IS NOT NULL
|
|
AND NEW.source_snapshot_id <> owner_snapshot THEN
|
|
RAISE EXCEPTION 'quarantine dataset version and source snapshot do not belong together'
|
|
USING ERRCODE = '23514';
|
|
END IF;
|
|
|
|
UPDATE datasets
|
|
SET status = 'quarantined',
|
|
quarantine_status = 'quarantined',
|
|
validation_status = 'failed',
|
|
provenance_status = 'incomplete',
|
|
lineage_status = 'incomplete'
|
|
WHERE id = owner_dataset;
|
|
END IF;
|
|
|
|
owner_snapshot := COALESCE(NEW.source_snapshot_id, owner_snapshot, dataset_snapshot);
|
|
IF owner_snapshot IS NOT NULL THEN
|
|
UPDATE source_snapshots
|
|
SET ingest_status = 'quarantined'
|
|
WHERE id = owner_snapshot;
|
|
|
|
-- A concrete snapshot is shared immutable evidence. Once
|
|
-- quarantined, every Dataset/Version bound to it must be
|
|
-- non-consumable as well; a consumption gate must never
|
|
-- depend on callers having included source_snapshot_id in
|
|
-- the quarantine record.
|
|
UPDATE dataset_versions
|
|
SET validation_status = 'failed',
|
|
provenance_status = 'incomplete',
|
|
lineage_status = 'incomplete'
|
|
WHERE source_snapshot_id = owner_snapshot;
|
|
|
|
UPDATE datasets
|
|
SET status = 'quarantined',
|
|
quarantine_status = 'quarantined',
|
|
validation_status = 'failed',
|
|
provenance_status = 'incomplete',
|
|
lineage_status = 'incomplete'
|
|
WHERE source_snapshot_id = owner_snapshot;
|
|
END IF;
|
|
|
|
-- Direct quarantine and shared-snapshot fan-out are not
|
|
-- enough: every downstream derivative inherits the unsafe
|
|
-- lineage and must fail all Dataset consumption boundaries.
|
|
PERFORM geointel_phase2_quarantine_lineage_descendants(owner_dataset, owner_snapshot);
|
|
RETURN NEW;
|
|
END;
|
|
$$ LANGUAGE plpgsql;
|
|
|
|
CREATE TRIGGER trg_dataset_quarantines_state_guard
|
|
AFTER INSERT OR UPDATE OF status, dataset_id, dataset_version_id, source_snapshot_id ON dataset_quarantines
|
|
FOR EACH ROW EXECUTE FUNCTION geointel_phase2_quarantine_state_guard();
|
|
"""
|
|
)
|
|
)
|
|
|
|
|
|
def downgrade() -> None:
|
|
op.execute(
|
|
sa.text(
|
|
"""
|
|
DROP TRIGGER IF EXISTS trg_datasets_snapshot_registry_guard ON datasets;
|
|
DROP TRIGGER IF EXISTS trg_dataset_versions_snapshot_registry_guard ON dataset_versions;
|
|
DROP TRIGGER IF EXISTS trg_source_snapshots_registry_immutable ON source_snapshots;
|
|
DROP TRIGGER IF EXISTS trg_source_registry_write_guard ON source_registry;
|
|
DROP TRIGGER IF EXISTS trg_source_snapshots_evidence_immutable ON source_snapshots;
|
|
DROP TRIGGER IF EXISTS trg_datasets_contract_report_guard ON datasets;
|
|
DROP TRIGGER IF EXISTS trg_dataset_versions_contract_report_guard ON dataset_versions;
|
|
DROP TRIGGER IF EXISTS trg_dataset_lineage_edges_cycle_guard ON dataset_lineage_edges;
|
|
DROP TRIGGER IF EXISTS trg_dataset_lineage_edges_immutable ON dataset_lineage_edges;
|
|
DROP TRIGGER IF EXISTS trg_dataset_quarantines_state_guard ON dataset_quarantines;
|
|
DROP FUNCTION IF EXISTS geointel_phase2_snapshot_registry_guard();
|
|
DROP FUNCTION IF EXISTS geointel_phase2_snapshot_registry_immutable_guard();
|
|
DROP FUNCTION IF EXISTS geointel_phase2_source_registry_write_guard();
|
|
DROP FUNCTION IF EXISTS geointel_phase2_snapshot_evidence_immutable_guard();
|
|
DROP FUNCTION IF EXISTS geointel_phase2_contract_report_guard();
|
|
DROP FUNCTION IF EXISTS geointel_phase2_dataset_version_contract_report_guard();
|
|
DROP FUNCTION IF EXISTS geointel_phase2_lineage_cycle_guard();
|
|
DROP FUNCTION IF EXISTS geointel_phase2_lineage_edge_immutable_guard();
|
|
DROP FUNCTION IF EXISTS geointel_phase2_quarantine_lineage_descendants(uuid, uuid);
|
|
DROP FUNCTION IF EXISTS geointel_phase2_quarantine_state_guard();
|
|
"""
|
|
)
|
|
)
|
|
op.drop_table("dataset_quarantines")
|
|
op.drop_table("dataset_lineage_edges")
|
|
|
|
op.drop_constraint("ck_dataset_versions_ingest_key_not_blank", "dataset_versions", type_="check")
|
|
op.drop_constraint("ck_dataset_versions_lineage_status", "dataset_versions", type_="check")
|
|
op.drop_constraint("ck_dataset_versions_provenance_status", "dataset_versions", type_="check")
|
|
op.drop_constraint("ck_dataset_versions_validation_status", "dataset_versions", type_="check")
|
|
op.drop_constraint("ck_datasets_ingest_key_not_blank", "datasets", type_="check")
|
|
op.drop_constraint("ck_datasets_quarantine_status", "datasets", type_="check")
|
|
op.drop_constraint("ck_datasets_lineage_status", "datasets", type_="check")
|
|
op.drop_constraint("ck_datasets_provenance_status", "datasets", type_="check")
|
|
op.drop_constraint("ck_datasets_validation_status", "datasets", type_="check")
|
|
op.drop_constraint("uq_dataset_versions_dataset_ingest_key", "dataset_versions", type_="unique")
|
|
op.drop_constraint("uq_datasets_project_ingest_key", "datasets", type_="unique")
|
|
op.drop_index("ix_dataset_versions_source_registry_snapshot", table_name="dataset_versions")
|
|
op.drop_index("ix_datasets_source_registry_snapshot", table_name="datasets")
|
|
op.drop_constraint("fk_dataset_versions_source_snapshot", "dataset_versions", type_="foreignkey")
|
|
op.drop_constraint("fk_dataset_versions_source_registry", "dataset_versions", type_="foreignkey")
|
|
op.drop_constraint("fk_datasets_source_snapshot", "datasets", type_="foreignkey")
|
|
op.drop_constraint("fk_datasets_source_registry", "datasets", type_="foreignkey")
|
|
|
|
for column_name in (
|
|
"lineage_status",
|
|
"provenance_status",
|
|
"validation_report_json",
|
|
"validation_status",
|
|
"data_contract_version",
|
|
"data_contract_key",
|
|
"ingest_key",
|
|
"source_snapshot_id",
|
|
"source_registry_id",
|
|
):
|
|
op.drop_column("dataset_versions", column_name)
|
|
for column_name in (
|
|
"quarantine_status",
|
|
"lineage_status",
|
|
"provenance_status",
|
|
"validation_report_json",
|
|
"validation_status",
|
|
"data_contract_version",
|
|
"data_contract_key",
|
|
"ingest_key",
|
|
"source_snapshot_id",
|
|
"source_registry_id",
|
|
):
|
|
op.drop_column("datasets", column_name)
|
|
|
|
op.drop_index("ix_source_snapshots_checksum", table_name="source_snapshots")
|
|
op.drop_index("ix_source_snapshots_registry_fetched", table_name="source_snapshots")
|
|
op.drop_table("source_snapshots")
|
|
op.drop_table("source_registry")
|