diff --git a/.gitignore b/.gitignore index 1e10fc85..fec18197 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,8 @@ build/ /artifacts/evidence/accuracy/* !/artifacts/evidence/accuracy/P1/ !/artifacts/evidence/accuracy/P1/** +!/artifacts/evidence/accuracy/P2/ +!/artifacts/evidence/accuracy/P2/** /.cache/ /datasets/raw/* /datasets/processed/* diff --git a/backend/alembic/versions/202608010001_source_registry_provenance.py b/backend/alembic/versions/202608010001_source_registry_provenance.py new file mode 100644 index 00000000..32c8ac4c --- /dev/null +++ b/backend/alembic/versions/202608010001_source_registry_provenance.py @@ -0,0 +1,1308 @@ +"""Add server-owned source registry, source snapshots and dataset provenance gates. + +Legacy datasets are retained untouched. They receive an identity link when one +is available, but remain explicitly incomplete/not_validated until a governed +ingest writes an immutable source snapshot and contract result. +""" + +from __future__ import annotations + +import uuid +import json + +from alembic import op +import sqlalchemy as sa + + +revision = "202608010001" +down_revision = "202607260001" +branch_labels = None +depends_on = None + + +_SOURCE_NAMESPACE = uuid.UUID("6e5f9c11-bc7f-4f5f-ae23-f3eddb4d0149") +_DEFAULT_LICENSE = "Provider terms must be verified for each immutable snapshot." +_DEFAULT_LIMITATION = "No authority, ground-truth, freshness or training claim is allowed without a governed snapshot and passed contract." +_DEFAULT_USAGE_POLICY = { + "automatic_ground_truth": False, + "ground_truth_allowed": False, + "training_allowed": False, + "allowed_tasks": [], + "validation_authority": {}, +} + + +def _source_seed( + source_key: str, + display_name: str, + classification: str, + authority_name: str, + *, + authority_scope: dict | None = None, + provider_adapter_key: str | None = None, + source_url: str | None = None, + license_name: str = _DEFAULT_LICENSE, + usage_restrictions: str = "Use only according to the source-specific snapshot terms and attribution.", + default_crs: str = "unknown", + default_units: str = "unknown", + spatial_resolution: dict | None = None, + temporal_coverage: dict | None = None, + geographic_coverage: dict | None = None, + expected_geometry_types: list[str] | None = None, + expected_attributes: dict | None = None, + usage_policy: dict | None = None, + freshness_status: str = "unknown", + ingest_status: str = "registered", + known_limitations: list[str] | None = None, +) -> dict: + return { + "id": uuid.uuid5(_SOURCE_NAMESPACE, source_key), + "source_key": source_key, + "display_name": display_name, + "classification": classification, + "authority_name": authority_name, + "authority_scope_json": authority_scope or {"status": "unknown"}, + "provider_adapter_key": provider_adapter_key, + "source_url": source_url, + "license_name": license_name, + "license_url": None, + "usage_restrictions": usage_restrictions, + "default_crs": default_crs, + "default_units": default_units, + "spatial_resolution_json": spatial_resolution or {"status": "unknown"}, + "temporal_coverage_json": temporal_coverage or {"status": "unknown"}, + "geographic_coverage_json": geographic_coverage or {"status": "unknown"}, + "expected_geometry_types_json": expected_geometry_types or [], + "expected_attributes_json": expected_attributes or {"status": "unknown"}, + "usage_policy_json": usage_policy or dict(_DEFAULT_USAGE_POLICY), + "freshness_status": freshness_status, + "ingest_status": ingest_status, + "known_limitations_json": known_limitations or [_DEFAULT_LIMITATION], + "registry_metadata_json": {"registry_owner": "server", "seeded_by": revision}, + } + + +def _seed_rows() -> list[dict]: + governed_vector_policy = { + "automatic_ground_truth": False, + "ground_truth_allowed": False, + "training_allowed": False, + "allowed_tasks": ["reference_context"], + "validation_authority": {}, + } + governed_imagery_policy = { + "automatic_ground_truth": False, + "ground_truth_allowed": False, + "training_allowed": True, + "allowed_tasks": ["imagery", "training_input", "visual_context"], + "validation_authority": {}, + } + regional_building_policy = { + "automatic_ground_truth": False, + "ground_truth_allowed": True, + "training_allowed": True, + "allowed_tasks": ["building_validation", "building_labels"], + "validation_authority": {"building_validation": "regional_primary_pending_contract"}, + } + + return [ + _source_seed( + "grb", + "Grootschalig Referentie Bestand", + "authoritative", + "Digitaal Vlaanderen", + authority_scope={"zone": "Flanders", "themes": ["buildings", "roads", "water", "parcels"]}, + provider_adapter_key="grb", + source_url="https://www.vlaanderen.be/datavindplaats/catalogus/basiskaart-vlaanderen-grb", + usage_restrictions="Use governed GRB acquisition evidence and retain Digitaal Vlaanderen attribution.", + default_crs="EPSG:31370", + default_units="metres", + geographic_coverage={"zone": "Flanders"}, + expected_geometry_types=["Polygon", "MultiPolygon", "LineString", "MultiLineString"], + expected_attributes={"required": ["id"], "layers": ["GBG", "Wegsegment", "WTZ", "WLAS", "WGR", "ADP"]}, + usage_policy={ + "automatic_ground_truth": False, + "ground_truth_allowed": True, + "training_allowed": True, + "allowed_tasks": ["building_validation", "building_labels", "reference_context"], + "validation_authority": {"building_validation": "primary"}, + }, + ingest_status="configured", + known_limitations=[ + "GRB building geometry is authoritative only for a governed, versioned snapshot within Flanders.", + "GRB does not establish imagery-time alignment or national model validation by itself.", + ], + ), + _source_seed( + "digitaal_vlaanderen", + "Digitaal Vlaanderen (bronportaal)", + "authoritative", + "Digitaal Vlaanderen", + authority_scope={"zone": "Flanders", "role": "umbrella_catalogue_and_adapter_authority"}, + provider_adapter_key="digitaal_vlaanderen", + source_url="https://www.vlaanderen.be/datavindplaats", + default_crs="product_specific", + default_units="product_specific", + geographic_coverage={"zone": "Flanders"}, + usage_policy={ + "automatic_ground_truth": False, + "ground_truth_allowed": False, + "training_allowed": False, + "allowed_tasks": ["source_catalogue", "reference_context"], + "validation_authority": {}, + }, + ingest_status="configured", + known_limitations=[ + "This umbrella authority is not a product-level ground-truth source.", + "A governed import must use a product-specific source key whenever one is available.", + ], + ), + _source_seed( + "digitaal_vlaanderen_buildings_addresses_register", + "Gebouwen- en adressenregister", + "authoritative", + "Digitaal Vlaanderen", + authority_scope={"zone": "Flanders", "theme": "buildings_addresses"}, + provider_adapter_key="buildings_addresses_register", + default_crs="EPSG:31370", + default_units="metres", + geographic_coverage={"zone": "Flanders"}, + expected_geometry_types=["Point", "Polygon", "MultiPolygon"], + expected_attributes={"required": ["id"], "role": "administrative_corroboration"}, + usage_policy={ + "automatic_ground_truth": False, + "ground_truth_allowed": False, + "training_allowed": False, + "allowed_tasks": ["building_validation", "address_corroboration", "building_register_validation"], + "validation_authority": { + "building_validation": "corroborative", + "building_register_validation": "primary", + }, + }, + known_limitations=["Administrative building/address records do not replace a governed reference-footprint contract."], + ), + _source_seed( + "sentinel_2", + "Sentinel-2", + "contextual", + "Copernicus Programme", + authority_scope={"scope": "Belgium and Belgian North Sea", "role": "multispectral_context"}, + provider_adapter_key="sentinel_2", + default_crs="product_specific", + default_units="reflectance", + spatial_resolution={"metres": [10, 20, 60]}, + temporal_coverage={"cadence_days": 5, "status": "product_specific"}, + geographic_coverage={"scope": "Belgium and Belgian North Sea"}, + expected_geometry_types=[], + expected_attributes={"required": ["product_id", "sensing_time"]}, + usage_policy={ + "automatic_ground_truth": False, + "ground_truth_allowed": False, + "training_allowed": True, + "allowed_tasks": ["imagery_context", "change_context"], + "validation_authority": {}, + }, + ingest_status="not_configured", + known_limitations=["Sentinel-2 is contextual imagery, not automatic ground truth for building labels."], + ), + _source_seed( + "digitaal_vlaanderen_dhmv", + "Digitaal Hoogtemodel Vlaanderen", + "authoritative", + "Digitaal Vlaanderen", + authority_scope={"zone": "Flanders", "role": "terrain_height_corroboration"}, + provider_adapter_key="dhmv", + default_crs="EPSG:31370", + default_units="m TAW", + spatial_resolution={"metres": 1}, + temporal_coverage={"period": "2013-2015", "status": "product_specific"}, + geographic_coverage={"zone": "Flanders"}, + expected_attributes={"bands": 1, "nodata_required": True}, + usage_policy={ + "automatic_ground_truth": False, + "ground_truth_allowed": False, + "training_allowed": True, + "allowed_tasks": ["terrain_context", "height_corroboration", "elevation_validation"], + "validation_authority": { + "building_validation": "corroborative", + "elevation_validation": "primary", + }, + }, + ingest_status="configured", + known_limitations=["DHMV is height context and cannot independently establish building labels."], + ), + _source_seed( + "osm", + "OpenStreetMap", + "contextual", + "OpenStreetMap contributors", + authority_scope={"scope": "community-maintained", "role": "contextual"}, + provider_adapter_key="osm", + source_url="https://www.openstreetmap.org", + license_name="ODbL", + usage_restrictions="OpenStreetMap attribution and ODbL obligations apply.", + default_crs="EPSG:4326", + default_units="mixed", + geographic_coverage={"scope": "global"}, + expected_geometry_types=["Point", "LineString", "Polygon", "MultiPolygon"], + expected_attributes={"status": "community_tags"}, + usage_policy={ + "automatic_ground_truth": False, + "ground_truth_allowed": False, + "training_allowed": False, + "allowed_tasks": ["context", "candidate_discovery"], + "validation_authority": {}, + }, + ingest_status="not_configured", + known_limitations=["OSM is never automatic ground truth for GeoIntel validation or labels."], + ), + _source_seed( + "manual", + "Handmatige upload", + "experimental", + "Operator supplied", + authority_scope={"scope": "operator supplied", "trust": "unverified"}, + default_crs="unknown", + default_units="unknown", + usage_policy=dict(_DEFAULT_USAGE_POLICY), + ingest_status="configured", + known_limitations=["Manual uploads remain untrusted until a passed contract and governed provenance are attached."], + ), + _source_seed( + "fixture", + "Test- en demo fixture", + "experimental", + "GeoIntel test fixture", + authority_scope={"scope": "test_only"}, + default_crs="fixture_specific", + default_units="fixture_specific", + usage_policy=dict(_DEFAULT_USAGE_POLICY), + ingest_status="configured", + known_limitations=["Fixtures must never be presented as official data or used for production training/promotion."], + ), + _source_seed( + "map_selection", + "Afgeleide kaartselectie", + "derived", + "GeoIntel derived operation", + authority_scope={"scope": "derived_from_registered_input"}, + default_crs="EPSG:4326", + default_units="source_dependent", + usage_policy=dict(_DEFAULT_USAGE_POLICY), + known_limitations=["Derived selections inherit no authority beyond their complete lineage edge and source snapshot."], + ), + _source_seed( + "derived", + "Afgeleide dataset", + "derived", + "GeoIntel derived operation", + authority_scope={"scope": "derived_from_registered_input"}, + usage_policy=dict(_DEFAULT_USAGE_POLICY), + ), + _source_seed( + "training_label", + "Afgeleide trainingslabels", + "derived", + "GeoIntel reviewed label pipeline", + authority_scope={"scope": "derived_from_reviewed_source_snapshots"}, + usage_policy=dict(_DEFAULT_USAGE_POLICY), + known_limitations=["Training labels require complete source lineage and human-review evidence; they inherit no automatic authority."], + ), + _source_seed( + "model", + "Model artifact", + "experimental", + "GeoIntel model pipeline", + authority_scope={"scope": "internal_model_artifact"}, + usage_policy=dict(_DEFAULT_USAGE_POLICY), + known_limitations=["A model artifact is not a validated capability or promotion decision without its model card and evaluation evidence."], + ), + _source_seed( + "experimental", + "Experimentele bron", + "experimental", + "Unverified", + authority_scope={"scope": "unverified"}, + usage_policy=dict(_DEFAULT_USAGE_POLICY), + ), + _source_seed( + "legacy_unknown", + "Niet-geclassificeerde historische bron", + "experimental", + "Legacy import — unverified", + authority_scope={"scope": "legacy", "trust": "unverified"}, + usage_policy=dict(_DEFAULT_USAGE_POLICY), + ingest_status="legacy_unverified", + known_limitations=["Historical source identity is retained descriptively but has no governed authority until re-ingested."], + ), + _source_seed("ngi_adminvector", "NGI AdminVector", "authoritative", "Nationaal Geografisch Instituut", authority_scope={"scope": "Belgium"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"scope": "Belgium"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy), + _source_seed("rbins_marine_reporting_units", "RBINS mariene rapportage-eenheden", "authoritative", "RBINS", authority_scope={"zone": "Belgian North Sea"}, default_crs="EPSG:4326", default_units="degrees", geographic_coverage={"zone": "Belgian North Sea"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy), + _source_seed("rbins_msp_2026", "Belgisch Marien Ruimtelijk Plan 2026-2034", "authoritative", "RBINS", authority_scope={"zone": "Belgian North Sea", "edition": "2026-2034"}, default_crs="EPSG:4326", default_units="degrees", geographic_coverage={"zone": "Belgian North Sea"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy), + _source_seed("vrbg", "Vlaams Wegenregister", "authoritative", "Digitaal Vlaanderen", authority_scope={"zone": "Flanders", "theme": "roads"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=["LineString", "MultiLineString"], usage_policy=governed_vector_policy), + _source_seed("digitaal_vlaanderen_orthophoto", "Orthofoto Vlaanderen", "contextual", "Digitaal Vlaanderen", authority_scope={"zone": "Flanders", "role": "imagery"}, default_crs="EPSG:31370", default_units="pixel", spatial_resolution={"metres": 0.25}, geographic_coverage={"zone": "Flanders"}, usage_policy=governed_imagery_policy, ingest_status="configured"), + _source_seed("spw_orthophoto", "Orthofoto Wallonië", "contextual", "Service public de Wallonie", authority_scope={"zone": "Wallonia", "role": "imagery"}, license_name="CC BY 4.0", default_crs="EPSG:31370", default_units="pixel", spatial_resolution={"metres": 0.25}, geographic_coverage={"zone": "Wallonia"}, usage_policy=governed_imagery_policy, ingest_status="configured"), + _source_seed("urbis_orthophoto", "Orthofoto Brussel", "contextual", "UrbIS / Brussels Region", authority_scope={"zone": "Brussels-Capital Region", "role": "imagery"}, license_name="CC0", default_crs="EPSG:31370", default_units="pixel", spatial_resolution={"metres": 0.25}, geographic_coverage={"zone": "Brussels-Capital Region"}, usage_policy=governed_imagery_policy, ingest_status="configured"), + _source_seed("agentschap_landbouw_zeevisserij_agricultural_parcels", "Landbouwgebruikspercelen", "authoritative", "Agentschap Landbouw en Zeevisserij", authority_scope={"zone": "Flanders", "theme": "agricultural_parcels"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy), + _source_seed("department_omgeving_land_use", "Landgebruik Vlaanderen", "authoritative", "Departement Omgeving", authority_scope={"zone": "Flanders", "theme": "land_use"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy), + _source_seed("inbo_bwk_natura2000", "BWK en Natura 2000", "authoritative", "INBO", authority_scope={"zone": "Flanders", "theme": "nature"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy), + _source_seed("statbel", "Statbel bevolking", "authoritative", "Statbel", authority_scope={"scope": "Belgium", "theme": "population"}, default_crs="EPSG:31370", default_units="persons", geographic_coverage={"scope": "Belgium"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy), + _source_seed("waterinfo", "Waterinfo", "authoritative", "Waterinfo Vlaanderen", authority_scope={"zone": "Flanders", "theme": "water"}, default_crs="EPSG:31370", default_units="source_specific", geographic_coverage={"zone": "Flanders"}, usage_policy=governed_vector_policy), + _source_seed("department_omgeving_thematic_raster", "Omgeving thematische rasters", "authoritative", "Departement Omgeving", authority_scope={"zone": "Flanders", "theme": "thematic_raster"}, default_crs="EPSG:31370", default_units="source_specific", geographic_coverage={"zone": "Flanders"}, usage_policy=governed_vector_policy), + _source_seed("dov_soil_map", "DOV bodemkaart", "authoritative", "Databank Ondergrond Vlaanderen", authority_scope={"zone": "Flanders", "theme": "soil"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=["Polygon", "MultiPolygon"], usage_policy=governed_vector_policy), + _source_seed("vmm_flood_hazard", "VMM overstromingskaarten", "authoritative", "Vlaamse Milieumaatschappij", authority_scope={"zone": "Flanders", "theme": "flood_hazard"}, default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Flanders"}, usage_policy=governed_vector_policy), + _source_seed("vmm_vha_bathymetry_profiles", "VHA bathymetrieprofielen", "authoritative", "Vlaamse Milieumaatschappij", authority_scope={"zone": "Flanders", "theme": "bathymetry_profiles"}, default_crs="EPSG:31370", default_units="m TAW", geographic_coverage={"zone": "Flanders"}, expected_geometry_types=["Point"], usage_policy=governed_vector_policy), + _source_seed("historical_landuse", "Historisch landgebruik", "corroborative", "Historical archive provider", authority_scope={"scope": "Belgium", "theme": "historical_land_use"}, default_crs="source_specific", default_units="source_specific", usage_policy=governed_vector_policy), + _source_seed( + "spw_geoportail", + "SPW Geoportail (bronportaal)", + "authoritative", + "Service public de Wallonie", + authority_scope={"zone": "Wallonia", "role": "umbrella_catalogue_and_adapter_authority"}, + provider_adapter_key="spw_geoportail", + source_url="https://geoportail.wallonie.be/catalogue", + default_crs="product_specific", + default_units="product_specific", + geographic_coverage={"zone": "Wallonia"}, + usage_policy={ + "automatic_ground_truth": False, + "ground_truth_allowed": False, + "training_allowed": False, + "allowed_tasks": ["source_catalogue", "reference_context"], + "validation_authority": {}, + }, + ingest_status="configured", + known_limitations=[ + "This umbrella authority is not a product-level ground-truth source.", + "A governed import must use PICC, WALOUS, terrain, flood, orthophoto or another product-specific key when available.", + ], + ), + _source_seed("spw_picc", "PICC", "authoritative", "Service public de Wallonie", authority_scope={"zone": "Wallonia", "theme": "topography_buildings"}, license_name="CC BY 4.0", default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Wallonia"}, expected_geometry_types=["Polygon", "MultiPolygon", "LineString", "MultiLineString"], usage_policy=regional_building_policy), + _source_seed("urbis", "UrbIS", "authoritative", "Brussels Region", authority_scope={"zone": "Brussels-Capital Region", "theme": "topography_buildings"}, license_name="CC0", default_crs="EPSG:31370", default_units="metres", geographic_coverage={"zone": "Brussels-Capital Region"}, expected_geometry_types=["Polygon", "MultiPolygon", "LineString", "MultiLineString"], usage_policy=regional_building_policy), + _source_seed("spw_walous_land_cover", "WALOUS landbedekking", "authoritative", "Service public de Wallonie", authority_scope={"zone": "Wallonia", "theme": "land_cover"}, license_name="CC BY 4.0", default_crs="EPSG:3812", default_units="class_code", spatial_resolution={"metres": 1}, geographic_coverage={"zone": "Wallonia"}, usage_policy=governed_vector_policy), + _source_seed("spw_bathymetry", "SPW bathymetrie", "authoritative", "Service public de Wallonie", authority_scope={"zone": "Wallonia", "theme": "bathymetry"}, default_crs="EPSG:3812", default_units="mDNG", geographic_coverage={"zone": "Wallonia"}, usage_policy=governed_vector_policy), + _source_seed("spw_terrain", "SPW terreinmodel", "corroborative", "Service public de Wallonie", authority_scope={"zone": "Wallonia", "theme": "terrain"}, default_crs="EPSG:3812", default_units="metres", spatial_resolution={"metres": 1}, geographic_coverage={"zone": "Wallonia"}, usage_policy=governed_vector_policy), + _source_seed("spw_flood_hazard", "SPW overstromingsgevaar", "authoritative", "Service public de Wallonie", authority_scope={"zone": "Wallonia", "theme": "flood_hazard"}, default_crs="EPSG:3812", default_units="metres", geographic_coverage={"zone": "Wallonia"}, usage_policy=governed_vector_policy), + _source_seed("mdk_bathymetry", "MDK bathymetrie", "authoritative", "Maritieme Dienstverlening en Kust", authority_scope={"zone": "Belgian North Sea", "theme": "bathymetry"}, default_crs="EPSG:3812", default_units="metres", geographic_coverage={"zone": "Belgian North Sea"}, usage_policy=governed_vector_policy), + _source_seed( + "mdk_bcp_bathymetry", + "MDK BCP bathymetrie-probe en verwerving", + "authoritative", + "Maritieme Dienstverlening en Kust", + authority_scope={ + "zone": "Belgian North Sea", + "theme": "bathymetry", + "role": "coverage_probe_and_governed_acquisition", + }, + provider_adapter_key="mdk_bcp_bathymetry", + source_url="https://www.vlaanderen.be/datavindplaats", + default_crs="EPSG:3812", + default_units="metres", + geographic_coverage={"zone": "Belgian North Sea"}, + usage_policy=governed_vector_policy, + ingest_status="not_configured", + known_limitations=[ + "A BCP coverage probe is discovery evidence, not a usable bathymetry dataset.", + "Only a successful governed acquisition with an immutable response checksum may create a source snapshot.", + ], + ), + ] + + +def _source_registry_table() -> sa.Table: + return sa.table( + "source_registry", + sa.column("id", sa.UUID()), + sa.column("source_key", sa.String()), + sa.column("display_name", sa.String()), + sa.column("classification", sa.String()), + sa.column("authority_name", sa.String()), + sa.column("authority_scope_json", sa.JSON()), + sa.column("provider_adapter_key", sa.String()), + sa.column("source_url", sa.Text()), + sa.column("license_name", sa.String()), + sa.column("license_url", sa.Text()), + sa.column("usage_restrictions", sa.Text()), + sa.column("default_crs", sa.String()), + sa.column("default_units", sa.String()), + sa.column("spatial_resolution_json", sa.JSON()), + sa.column("temporal_coverage_json", sa.JSON()), + sa.column("geographic_coverage_json", sa.JSON()), + sa.column("expected_geometry_types_json", sa.JSON()), + sa.column("expected_attributes_json", sa.JSON()), + sa.column("usage_policy_json", sa.JSON()), + sa.column("freshness_status", sa.String()), + sa.column("ingest_status", sa.String()), + sa.column("known_limitations_json", sa.JSON()), + sa.column("registry_metadata_json", sa.JSON()), + ) + + +def _offline_safe_seed_rows() -> list[dict]: + json_columns = { + "authority_scope_json", + "spatial_resolution_json", + "temporal_coverage_json", + "geographic_coverage_json", + "expected_geometry_types_json", + "expected_attributes_json", + "usage_policy_json", + "known_limitations_json", + "registry_metadata_json", + } + rows: list[dict] = [] + for seed in _seed_rows(): + row = dict(seed) + row["id"] = op.inline_literal(str(seed["id"]), type_=sa.String()) + for column_name in json_columns: + row[column_name] = op.inline_literal( + json.dumps(seed[column_name], sort_keys=True, separators=(",", ":")), + type_=sa.String(), + ) + rows.append(row) + return rows + + +def upgrade() -> None: + op.create_table( + "source_registry", + sa.Column("id", sa.UUID(), primary_key=True), + sa.Column("source_key", sa.String(length=120), nullable=False), + sa.Column("display_name", sa.String(length=255), nullable=False), + sa.Column("classification", sa.String(length=32), nullable=False), + sa.Column("authority_name", sa.String(length=255), nullable=False, server_default="unknown"), + sa.Column("authority_scope_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), + sa.Column("provider_adapter_key", sa.String(length=120), nullable=True), + sa.Column("source_url", sa.Text(), nullable=True), + sa.Column("license_name", sa.String(length=255), nullable=False, server_default="unknown"), + sa.Column("license_url", sa.Text(), nullable=True), + sa.Column("usage_restrictions", sa.Text(), nullable=False, server_default="unknown"), + sa.Column("default_crs", sa.String(length=64), nullable=False, server_default="unknown"), + sa.Column("default_units", sa.String(length=120), nullable=False, server_default="unknown"), + sa.Column("spatial_resolution_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), + sa.Column("temporal_coverage_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), + sa.Column("geographic_coverage_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), + sa.Column("expected_geometry_types_json", sa.JSON(), nullable=False, server_default=sa.text("'[]'::json")), + sa.Column("expected_attributes_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), + sa.Column("usage_policy_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), + sa.Column("freshness_status", sa.String(length=32), nullable=False, server_default="unknown"), + sa.Column("ingest_status", sa.String(length=32), nullable=False, server_default="registered"), + sa.Column("known_limitations_json", sa.JSON(), nullable=False, server_default=sa.text("'[]'::json")), + sa.Column("registry_metadata_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + sa.UniqueConstraint("source_key", name="uq_source_registry_source_key"), + sa.CheckConstraint( + "classification IN ('authoritative', 'corroborative', 'contextual', 'derived', 'experimental')", + name="ck_source_registry_classification", + ), + sa.CheckConstraint( + "freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')", + name="ck_source_registry_freshness_status", + ), + sa.CheckConstraint( + "ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', " + "'failed', 'quarantined', 'legacy_unverified')", + name="ck_source_registry_ingest_status", + ), + ) + op.create_table( + "source_snapshots", + sa.Column("id", sa.UUID(), primary_key=True), + sa.Column("source_registry_id", sa.UUID(), sa.ForeignKey("source_registry.id", ondelete="CASCADE"), nullable=False), + sa.Column("snapshot_key", sa.String(length=255), nullable=False), + sa.Column("source_version", sa.String(length=120), nullable=True), + sa.Column("snapshot_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("fetched_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("source_url", sa.Text(), nullable=True), + sa.Column("checksum_sha256", sa.String(length=64), nullable=False), + sa.Column("crs", sa.String(length=64), nullable=True), + sa.Column("units", sa.String(length=120), nullable=True), + sa.Column("spatial_resolution_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), + sa.Column("temporal_coverage_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), + sa.Column("geographic_coverage_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), + sa.Column("observed_schema_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), + sa.Column("freshness_status", sa.String(length=32), nullable=False, server_default="unknown"), + sa.Column("ingest_status", sa.String(length=32), nullable=False, server_default="registered"), + sa.Column("known_limitations_json", sa.JSON(), nullable=False, server_default=sa.text("'[]'::json")), + sa.Column("snapshot_metadata_json", sa.JSON(), nullable=False, server_default=sa.text("'{}'::json")), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + sa.UniqueConstraint("source_registry_id", "snapshot_key", name="uq_source_snapshots_registry_key"), + sa.CheckConstraint( + "freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')", + name="ck_source_snapshots_freshness_status", + ), + sa.CheckConstraint( + "ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', " + "'failed', 'quarantined', 'legacy_unverified')", + name="ck_source_snapshots_ingest_status", + ), + sa.CheckConstraint( + "checksum_sha256 = lower(checksum_sha256) AND checksum_sha256 ~ '^[0-9a-f]{64}$'", + name="ck_source_snapshots_checksum_sha256", + ), + ) + op.create_index("ix_source_snapshots_registry_fetched", "source_snapshots", ["source_registry_id", "fetched_at"]) + op.create_index("ix_source_snapshots_checksum", "source_snapshots", ["checksum_sha256"]) + + op.add_column("datasets", sa.Column("source_registry_id", sa.UUID(), nullable=True)) + op.add_column("datasets", sa.Column("source_snapshot_id", sa.UUID(), nullable=True)) + op.add_column("datasets", sa.Column("ingest_key", sa.String(length=255), nullable=True)) + op.add_column("datasets", sa.Column("data_contract_key", sa.String(length=120), nullable=True)) + op.add_column("datasets", sa.Column("data_contract_version", sa.String(length=64), nullable=True)) + op.add_column( + "datasets", + sa.Column( + "validation_status", + sa.String(length=32), + nullable=False, + server_default="not_validated", + comment="not_validated | passed | failed", + ), + ) + op.add_column("datasets", sa.Column("validation_report_json", sa.JSON(), nullable=True)) + op.add_column( + "datasets", + sa.Column( + "provenance_status", + sa.String(length=32), + nullable=False, + server_default="incomplete", + comment="complete | incomplete | not_applicable", + ), + ) + op.add_column( + "datasets", + sa.Column( + "lineage_status", + sa.String(length=32), + nullable=False, + server_default="incomplete", + comment="complete | incomplete | not_applicable", + ), + ) + op.add_column( + "datasets", + sa.Column( + "quarantine_status", + sa.String(length=32), + nullable=False, + server_default="not_quarantined", + comment="not_quarantined | quarantined", + ), + ) + op.add_column("dataset_versions", sa.Column("source_registry_id", sa.UUID(), nullable=True)) + op.add_column("dataset_versions", sa.Column("source_snapshot_id", sa.UUID(), nullable=True)) + op.add_column("dataset_versions", sa.Column("ingest_key", sa.String(length=255), nullable=True)) + op.add_column("dataset_versions", sa.Column("data_contract_key", sa.String(length=120), nullable=True)) + op.add_column("dataset_versions", sa.Column("data_contract_version", sa.String(length=64), nullable=True)) + op.add_column( + "dataset_versions", + sa.Column( + "validation_status", + sa.String(length=32), + nullable=False, + server_default="not_validated", + comment="not_validated | passed | failed", + ), + ) + op.add_column("dataset_versions", sa.Column("validation_report_json", sa.JSON(), nullable=True)) + op.add_column( + "dataset_versions", + sa.Column( + "provenance_status", + sa.String(length=32), + nullable=False, + server_default="incomplete", + comment="complete | incomplete | not_applicable", + ), + ) + op.add_column( + "dataset_versions", + sa.Column( + "lineage_status", + sa.String(length=32), + nullable=False, + server_default="incomplete", + comment="complete | incomplete | not_applicable", + ), + ) + + op.create_foreign_key("fk_datasets_source_registry", "datasets", "source_registry", ["source_registry_id"], ["id"], ondelete="SET NULL") + op.create_foreign_key("fk_datasets_source_snapshot", "datasets", "source_snapshots", ["source_snapshot_id"], ["id"], ondelete="SET NULL") + op.create_foreign_key("fk_dataset_versions_source_registry", "dataset_versions", "source_registry", ["source_registry_id"], ["id"], ondelete="SET NULL") + op.create_foreign_key("fk_dataset_versions_source_snapshot", "dataset_versions", "source_snapshots", ["source_snapshot_id"], ["id"], ondelete="SET NULL") + op.create_index("ix_datasets_source_registry_snapshot", "datasets", ["source_registry_id", "source_snapshot_id"]) + op.create_index("ix_dataset_versions_source_registry_snapshot", "dataset_versions", ["source_registry_id", "source_snapshot_id"]) + op.create_unique_constraint("uq_datasets_project_ingest_key", "datasets", ["project_id", "ingest_key"]) + op.create_unique_constraint("uq_dataset_versions_dataset_ingest_key", "dataset_versions", ["dataset_id", "ingest_key"]) + op.create_check_constraint("ck_datasets_validation_status", "datasets", "validation_status IN ('not_validated', 'passed', 'failed')") + op.create_check_constraint("ck_datasets_provenance_status", "datasets", "provenance_status IN ('complete', 'incomplete', 'not_applicable')") + op.create_check_constraint("ck_datasets_lineage_status", "datasets", "lineage_status IN ('complete', 'incomplete', 'not_applicable')") + op.create_check_constraint("ck_datasets_quarantine_status", "datasets", "quarantine_status IN ('not_quarantined', 'quarantined')") + op.create_check_constraint("ck_datasets_ingest_key_not_blank", "datasets", "ingest_key IS NULL OR btrim(ingest_key) <> ''") + op.create_check_constraint("ck_dataset_versions_validation_status", "dataset_versions", "validation_status IN ('not_validated', 'passed', 'failed')") + op.create_check_constraint("ck_dataset_versions_provenance_status", "dataset_versions", "provenance_status IN ('complete', 'incomplete', 'not_applicable')") + op.create_check_constraint("ck_dataset_versions_lineage_status", "dataset_versions", "lineage_status IN ('complete', 'incomplete', 'not_applicable')") + op.create_check_constraint("ck_dataset_versions_ingest_key_not_blank", "dataset_versions", "ingest_key IS NULL OR btrim(ingest_key) <> ''") + + op.create_table( + "dataset_lineage_edges", + sa.Column("id", sa.UUID(), primary_key=True), + sa.Column("parent_dataset_id", sa.UUID(), sa.ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False), + sa.Column("child_dataset_id", sa.UUID(), sa.ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False), + sa.Column("parent_dataset_version_id", sa.UUID(), sa.ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True), + sa.Column("child_dataset_version_id", sa.UUID(), sa.ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True), + sa.Column("relation_type", sa.String(length=64), nullable=False), + sa.Column("transformation_name", sa.String(length=255), nullable=False), + sa.Column("transformation_version", sa.String(length=120), nullable=True), + sa.Column("parameters_json", sa.JSON(), nullable=True), + sa.Column("input_checksum_sha256", sa.String(length=64), nullable=True), + sa.Column("output_checksum_sha256", sa.String(length=64), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + sa.CheckConstraint("parent_dataset_id <> child_dataset_id", name="ck_dataset_lineage_edges_distinct_datasets"), + sa.UniqueConstraint("parent_dataset_id", "child_dataset_id", "relation_type", "transformation_name", name="uq_dataset_lineage_edges_relation"), + ) + op.create_index("ix_dataset_lineage_edges_parent", "dataset_lineage_edges", ["parent_dataset_id"]) + op.create_index("ix_dataset_lineage_edges_child", "dataset_lineage_edges", ["child_dataset_id"]) + op.create_table( + "dataset_quarantines", + sa.Column("id", sa.UUID(), primary_key=True), + sa.Column("dataset_id", sa.UUID(), sa.ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True), + sa.Column("dataset_version_id", sa.UUID(), sa.ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True), + sa.Column("source_snapshot_id", sa.UUID(), sa.ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True), + sa.Column("stage", sa.String(length=64), nullable=False), + sa.Column("reason_code", sa.String(length=120), nullable=False), + sa.Column("details_json", sa.JSON(), nullable=True), + sa.Column("artifact_path", sa.Text(), nullable=True), + sa.Column("artifact_checksum_sha256", sa.String(length=64), nullable=True), + sa.Column("status", sa.String(length=32), nullable=False, server_default="quarantined"), + sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now()), + sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("resolved_by", sa.String(length=120), nullable=True), + sa.CheckConstraint( + "dataset_id IS NOT NULL OR dataset_version_id IS NOT NULL OR source_snapshot_id IS NOT NULL", + name="ck_dataset_quarantines_target_present", + ), + sa.CheckConstraint("status IN ('quarantined', 'released', 'rejected')", name="ck_dataset_quarantines_status"), + ) + op.create_index("ix_dataset_quarantines_dataset_status", "dataset_quarantines", ["dataset_id", "status"]) + op.create_index("ix_dataset_quarantines_snapshot_status", "dataset_quarantines", ["source_snapshot_id", "status"]) + + op.bulk_insert(_source_registry_table(), _offline_safe_seed_rows(), multiinsert=False) + op.execute( + sa.text( + """ + UPDATE datasets + SET validation_status = 'not_validated', + provenance_status = 'incomplete', + lineage_status = 'incomplete', + quarantine_status = 'not_quarantined' + """ + ) + ) + op.execute( + sa.text( + """ + UPDATE datasets AS dataset + SET source_registry_id = registry.id + FROM source_registry AS registry + WHERE registry.source_key = COALESCE( + NULLIF(lower(btrim(dataset.source_name)), ''), + NULLIF(lower(btrim(dataset.source)), ''), + '__unregistered_legacy_source__' + ) + """ + ) + ) + op.execute( + sa.text( + """ + UPDATE dataset_versions AS version + SET source_registry_id = dataset.source_registry_id, + validation_status = 'not_validated', + provenance_status = 'incomplete', + lineage_status = 'incomplete' + FROM datasets AS dataset + WHERE version.dataset_id = dataset.id + """ + ) + ) + # A Dataset has separate foreign keys to its source registry and immutable + # snapshot. Those independent FKs alone cannot prove the snapshot belongs + # to the selected registry. Keep legacy rows with no snapshot valid, but + # reject every new or mutated mismatched pair at the database boundary. + # This mirrors the fail-closed application consumption gate and also + # protects maintenance scripts that bypass the ORM. + op.execute( + sa.text( + """ + CREATE FUNCTION geointel_phase2_snapshot_registry_guard() + RETURNS trigger AS $$ + DECLARE + snapshot_registry uuid; + BEGIN + IF NEW.source_snapshot_id IS NULL THEN + RETURN NEW; + END IF; + + SELECT source_registry_id + INTO snapshot_registry + FROM source_snapshots + WHERE id = NEW.source_snapshot_id; + + IF snapshot_registry IS NULL OR NEW.source_registry_id IS NULL + OR NEW.source_registry_id <> snapshot_registry THEN + RAISE EXCEPTION + 'source snapshot % does not belong to source registry %', + NEW.source_snapshot_id, + NEW.source_registry_id + USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER trg_datasets_snapshot_registry_guard + BEFORE INSERT OR UPDATE OF source_registry_id, source_snapshot_id ON datasets + FOR EACH ROW EXECUTE FUNCTION geointel_phase2_snapshot_registry_guard(); + + CREATE TRIGGER trg_dataset_versions_snapshot_registry_guard + BEFORE INSERT OR UPDATE OF source_registry_id, source_snapshot_id ON dataset_versions + FOR EACH ROW EXECUTE FUNCTION geointel_phase2_snapshot_registry_guard(); + + CREATE FUNCTION geointel_phase2_snapshot_registry_immutable_guard() + RETURNS trigger AS $$ + BEGIN + IF NEW.source_registry_id IS DISTINCT FROM OLD.source_registry_id THEN + RAISE EXCEPTION + 'source snapshot registry identity is immutable' + USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER trg_source_snapshots_registry_immutable + BEFORE UPDATE OF source_registry_id ON source_snapshots + FOR EACH ROW EXECUTE FUNCTION geointel_phase2_snapshot_registry_immutable_guard(); + + CREATE FUNCTION geointel_phase2_source_registry_write_guard() + RETURNS trigger AS $$ + BEGIN + RAISE EXCEPTION 'source registry entries are server-owned and immutable' + USING ERRCODE = '23514'; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER trg_source_registry_write_guard + BEFORE UPDATE OR DELETE ON source_registry + FOR EACH ROW EXECUTE FUNCTION geointel_phase2_source_registry_write_guard(); + + CREATE FUNCTION geointel_phase2_snapshot_evidence_immutable_guard() + RETURNS trigger AS $$ + BEGIN + IF TG_OP = 'DELETE' THEN + RAISE EXCEPTION 'source snapshots are immutable evidence and cannot be deleted' + USING ERRCODE = '23514'; + END IF; + + IF NEW.source_registry_id IS DISTINCT FROM OLD.source_registry_id + OR NEW.snapshot_key IS DISTINCT FROM OLD.snapshot_key + OR NEW.source_version IS DISTINCT FROM OLD.source_version + OR NEW.snapshot_at IS DISTINCT FROM OLD.snapshot_at + OR NEW.fetched_at IS DISTINCT FROM OLD.fetched_at + OR NEW.source_url IS DISTINCT FROM OLD.source_url + OR NEW.checksum_sha256 IS DISTINCT FROM OLD.checksum_sha256 + OR NEW.crs IS DISTINCT FROM OLD.crs + OR NEW.units IS DISTINCT FROM OLD.units + OR NEW.spatial_resolution_json::text IS DISTINCT FROM OLD.spatial_resolution_json::text + OR NEW.temporal_coverage_json::text IS DISTINCT FROM OLD.temporal_coverage_json::text + OR NEW.geographic_coverage_json::text IS DISTINCT FROM OLD.geographic_coverage_json::text + OR NEW.observed_schema_json::text IS DISTINCT FROM OLD.observed_schema_json::text + OR NEW.freshness_status IS DISTINCT FROM OLD.freshness_status + OR NEW.known_limitations_json::text IS DISTINCT FROM OLD.known_limitations_json::text + OR NEW.snapshot_metadata_json::text IS DISTINCT FROM OLD.snapshot_metadata_json::text THEN + RAISE EXCEPTION 'source snapshot evidence is immutable' + USING ERRCODE = '23514'; + END IF; + IF NEW.ingest_status IS DISTINCT FROM OLD.ingest_status + AND NEW.ingest_status <> 'quarantined' THEN + RAISE EXCEPTION 'source snapshot lifecycle may only transition to quarantined' + USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER trg_source_snapshots_evidence_immutable + BEFORE UPDATE OR DELETE ON source_snapshots + FOR EACH ROW EXECUTE FUNCTION geointel_phase2_snapshot_evidence_immutable_guard(); + + CREATE FUNCTION geointel_phase2_contract_report_guard() + RETURNS trigger AS $$ + DECLARE + snapshot_checksum text; + BEGIN + IF NEW.validation_status = 'passed' THEN + IF NEW.data_contract_key IS NULL OR btrim(NEW.data_contract_key) = '' + OR NEW.data_contract_version IS NULL OR btrim(NEW.data_contract_version) = '' + OR NEW.source_registry_id IS NULL OR NEW.source_snapshot_id IS NULL + OR NEW.validation_report_json IS NULL + OR json_typeof(NEW.validation_report_json) <> 'object' + OR COALESCE(NEW.validation_report_json ->> 'validation_status', '') <> 'passed' + OR COALESCE(NEW.validation_report_json ->> 'data_contract_key', '') <> NEW.data_contract_key + OR COALESCE(NEW.validation_report_json ->> 'data_contract_version', '') <> NEW.data_contract_version + OR COALESCE(NEW.validation_report_json ->> 'provenance_status', '') <> 'complete' + OR COALESCE(NEW.validation_report_json ->> 'lineage_status', '') NOT IN ('complete', 'not_applicable') + OR COALESCE(NEW.validation_report_json ->> 'quarantine_status', '') <> 'not_quarantined' + OR COALESCE(NEW.validation_report_json ->> 'contract_fingerprint_sha256', '') !~ '^[0-9a-f]{64}$' + OR COALESCE(NEW.validation_report_json ->> 'report_sha256', '') !~ '^[0-9a-f]{64}$' + OR NEW.provenance_status <> 'complete' + OR NEW.lineage_status NOT IN ('complete', 'not_applicable') THEN + RAISE EXCEPTION 'passed dataset contract state requires a matching complete validation report' + USING ERRCODE = '23514'; + END IF; + IF NEW.quarantine_status <> 'not_quarantined' THEN + RAISE EXCEPTION 'passed dataset cannot be quarantined' + USING ERRCODE = '23514'; + END IF; + + SELECT checksum_sha256 + INTO snapshot_checksum + FROM source_snapshots + WHERE id = NEW.source_snapshot_id; + + IF NEW.checksum_sha256 IS NULL + OR NEW.checksum_sha256 <> lower(NEW.checksum_sha256) + OR NEW.checksum_sha256 !~ '^[0-9a-f]{64}$' + OR snapshot_checksum IS NULL + OR NEW.checksum_sha256 <> snapshot_checksum THEN + RAISE EXCEPTION 'passed dataset checksum must be a canonical SHA-256 bound to its source snapshot' + USING ERRCODE = '23514'; + END IF; + END IF; + + -- A passed record is bound to exact bytes and the source/time + -- evidence that was validated with those bytes. A caller may + -- invalidate it, but cannot atomically replace any evidence + -- while doing so: invalidate first, then governed re-ingest. + IF TG_OP = 'UPDATE' AND OLD.validation_status = 'passed' AND ( + NEW.source IS DISTINCT FROM OLD.source + OR NEW.dataset_type IS DISTINCT FROM OLD.dataset_type + OR NEW.storage_path IS DISTINCT FROM OLD.storage_path + OR NEW.original_filename IS DISTINCT FROM OLD.original_filename + OR NEW.stored_filename IS DISTINCT FROM OLD.stored_filename + OR NEW.content_type IS DISTINCT FROM OLD.content_type + OR NEW.size_bytes IS DISTINCT FROM OLD.size_bytes + OR NEW.checksum_sha256 IS DISTINCT FROM OLD.checksum_sha256 + OR NEW.ingest_key IS DISTINCT FROM OLD.ingest_key + OR NEW.derived_from_dataset_id IS DISTINCT FROM OLD.derived_from_dataset_id + OR NEW.crs IS DISTINCT FROM OLD.crs + OR NEW.bounds_json::text IS DISTINCT FROM OLD.bounds_json::text + OR NEW.resolution_json::text IS DISTINCT FROM OLD.resolution_json::text + OR NEW.bands_json::text IS DISTINCT FROM OLD.bands_json::text + OR NEW.metadata_json::text IS DISTINCT FROM OLD.metadata_json::text + OR NEW.dataset_role IS DISTINCT FROM OLD.dataset_role + OR NEW.source_name IS DISTINCT FROM OLD.source_name + OR NEW.reference_layer_name IS DISTINCT FROM OLD.reference_layer_name + OR NEW.source_metadata::text IS DISTINCT FROM OLD.source_metadata::text + OR NEW.provenance_metadata::text IS DISTINCT FROM OLD.provenance_metadata::text + OR NEW.source_registry_id IS DISTINCT FROM OLD.source_registry_id + OR NEW.source_snapshot_id IS DISTINCT FROM OLD.source_snapshot_id + OR NEW.data_contract_key IS DISTINCT FROM OLD.data_contract_key + OR NEW.data_contract_version IS DISTINCT FROM OLD.data_contract_version + OR NEW.validation_report_json::text IS DISTINCT FROM OLD.validation_report_json::text + OR NEW.temporal_series_key IS DISTINCT FROM OLD.temporal_series_key + OR NEW.observed_at IS DISTINCT FROM OLD.observed_at + OR NEW.valid_from IS DISTINCT FROM OLD.valid_from + OR NEW.valid_to IS DISTINCT FROM OLD.valid_to + OR NEW.temporal_granularity IS DISTINCT FROM OLD.temporal_granularity + OR NEW.source_version IS DISTINCT FROM OLD.source_version + ) THEN + RAISE EXCEPTION 'accepted dataset artifact and provenance evidence is immutable; invalidate it before a governed re-ingest' + USING ERRCODE = '23514'; + END IF; + + IF TG_OP = 'UPDATE' + AND OLD.quarantine_status = 'quarantined' + AND NEW.quarantine_status <> 'quarantined' THEN + RAISE EXCEPTION 'quarantined dataset cannot be released in place; a governed re-ingest is required' + USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE FUNCTION geointel_phase2_dataset_version_contract_report_guard() + RETURNS trigger AS $$ + DECLARE + snapshot_checksum text; + BEGIN + IF NEW.validation_status = 'passed' THEN + IF NEW.data_contract_key IS NULL OR btrim(NEW.data_contract_key) = '' + OR NEW.data_contract_version IS NULL OR btrim(NEW.data_contract_version) = '' + OR NEW.source_registry_id IS NULL OR NEW.source_snapshot_id IS NULL + OR NEW.validation_report_json IS NULL + OR json_typeof(NEW.validation_report_json) <> 'object' + OR COALESCE(NEW.validation_report_json ->> 'validation_status', '') <> 'passed' + OR COALESCE(NEW.validation_report_json ->> 'data_contract_key', '') <> NEW.data_contract_key + OR COALESCE(NEW.validation_report_json ->> 'data_contract_version', '') <> NEW.data_contract_version + OR COALESCE(NEW.validation_report_json ->> 'provenance_status', '') <> 'complete' + OR COALESCE(NEW.validation_report_json ->> 'lineage_status', '') NOT IN ('complete', 'not_applicable') + OR COALESCE(NEW.validation_report_json ->> 'quarantine_status', '') <> 'not_quarantined' + OR COALESCE(NEW.validation_report_json ->> 'contract_fingerprint_sha256', '') !~ '^[0-9a-f]{64}$' + OR COALESCE(NEW.validation_report_json ->> 'report_sha256', '') !~ '^[0-9a-f]{64}$' + OR NEW.provenance_status <> 'complete' + OR NEW.lineage_status NOT IN ('complete', 'not_applicable') THEN + RAISE EXCEPTION 'passed dataset version contract state requires a matching complete validation report' + USING ERRCODE = '23514'; + END IF; + + SELECT checksum_sha256 + INTO snapshot_checksum + FROM source_snapshots + WHERE id = NEW.source_snapshot_id; + + IF NEW.checksum_sha256 IS NULL + OR NEW.checksum_sha256 <> lower(NEW.checksum_sha256) + OR NEW.checksum_sha256 !~ '^[0-9a-f]{64}$' + OR snapshot_checksum IS NULL + OR NEW.checksum_sha256 <> snapshot_checksum THEN + RAISE EXCEPTION 'passed dataset version checksum must be a canonical SHA-256 bound to its source snapshot' + USING ERRCODE = '23514'; + END IF; + END IF; + + IF TG_OP = 'UPDATE' AND OLD.validation_status = 'passed' AND ( + NEW.dataset_id IS DISTINCT FROM OLD.dataset_id + OR NEW.version IS DISTINCT FROM OLD.version + OR NEW.storage_path IS DISTINCT FROM OLD.storage_path + OR NEW.source_version IS DISTINCT FROM OLD.source_version + OR NEW.observed_at IS DISTINCT FROM OLD.observed_at + OR NEW.valid_from IS DISTINCT FROM OLD.valid_from + OR NEW.valid_to IS DISTINCT FROM OLD.valid_to + OR NEW.checksum_sha256 IS DISTINCT FROM OLD.checksum_sha256 + OR NEW.ingest_key IS DISTINCT FROM OLD.ingest_key + OR NEW.source_metadata::text IS DISTINCT FROM OLD.source_metadata::text + OR NEW.provenance_metadata::text IS DISTINCT FROM OLD.provenance_metadata::text + OR NEW.source_registry_id IS DISTINCT FROM OLD.source_registry_id + OR NEW.source_snapshot_id IS DISTINCT FROM OLD.source_snapshot_id + OR NEW.data_contract_key IS DISTINCT FROM OLD.data_contract_key + OR NEW.data_contract_version IS DISTINCT FROM OLD.data_contract_version + OR NEW.validation_report_json::text IS DISTINCT FROM OLD.validation_report_json::text + ) THEN + RAISE EXCEPTION 'accepted dataset-version artifact and provenance evidence is immutable; invalidate it before a governed re-ingest' + USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER trg_datasets_contract_report_guard + BEFORE INSERT OR UPDATE OF source, dataset_type, storage_path, + original_filename, stored_filename, content_type, size_bytes, + checksum_sha256, ingest_key, derived_from_dataset_id, crs, + bounds_json, resolution_json, bands_json, metadata_json, + dataset_role, source_name, reference_layer_name, source_metadata, + provenance_metadata, source_registry_id, source_snapshot_id, + data_contract_key, data_contract_version, validation_status, + validation_report_json, provenance_status, lineage_status, + quarantine_status, temporal_series_key, observed_at, valid_from, + valid_to, temporal_granularity, source_version ON datasets + FOR EACH ROW EXECUTE FUNCTION geointel_phase2_contract_report_guard(); + + CREATE TRIGGER trg_dataset_versions_contract_report_guard + BEFORE INSERT OR UPDATE OF dataset_id, version, storage_path, + source_version, observed_at, valid_from, valid_to, checksum_sha256, + ingest_key, source_metadata, provenance_metadata, source_registry_id, + source_snapshot_id, data_contract_key, data_contract_version, + validation_status, validation_report_json, provenance_status, + lineage_status ON dataset_versions + FOR EACH ROW EXECUTE FUNCTION geointel_phase2_dataset_version_contract_report_guard(); + + CREATE FUNCTION geointel_phase2_lineage_cycle_guard() + RETURNS trigger AS $$ + BEGIN + IF NEW.parent_dataset_id = NEW.child_dataset_id THEN + RAISE EXCEPTION 'a dataset cannot be its own lineage parent' + USING ERRCODE = '23514'; + END IF; + + IF EXISTS ( + WITH RECURSIVE descendants(dataset_id) AS ( + SELECT edge.child_dataset_id + FROM dataset_lineage_edges AS edge + WHERE edge.parent_dataset_id = NEW.child_dataset_id + AND (TG_OP <> 'UPDATE' OR edge.id <> NEW.id) + UNION + SELECT edge.child_dataset_id + FROM dataset_lineage_edges AS edge + JOIN descendants ON edge.parent_dataset_id = descendants.dataset_id + WHERE TG_OP <> 'UPDATE' OR edge.id <> NEW.id + ) + SELECT 1 FROM descendants WHERE dataset_id = NEW.parent_dataset_id + ) THEN + RAISE EXCEPTION + 'lineage edge % -> % would create a cycle', + NEW.parent_dataset_id, + NEW.child_dataset_id + USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER trg_dataset_lineage_edges_cycle_guard + BEFORE INSERT ON dataset_lineage_edges + FOR EACH ROW EXECUTE FUNCTION geointel_phase2_lineage_cycle_guard(); + + CREATE FUNCTION geointel_phase2_lineage_edge_immutable_guard() + RETURNS trigger AS $$ + BEGIN + RAISE EXCEPTION 'dataset lineage edges are immutable evidence and cannot be updated or deleted' + USING ERRCODE = '23514'; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER trg_dataset_lineage_edges_immutable + BEFORE UPDATE OR DELETE ON dataset_lineage_edges + FOR EACH ROW EXECUTE FUNCTION geointel_phase2_lineage_edge_immutable_guard(); + + CREATE FUNCTION geointel_phase2_quarantine_lineage_descendants( + root_dataset_id uuid, + root_snapshot_id uuid + ) + RETURNS void AS $$ + BEGIN + -- A shared source snapshot has several direct Dataset roots; + -- each root and every descendant must be made non-consumable. + -- UNION (rather than UNION ALL) keeps corrupt historic cycles + -- finite while the immutable cycle guard prevents new ones. + WITH RECURSIVE quarantine_roots(dataset_id) AS ( + SELECT root_dataset_id WHERE root_dataset_id IS NOT NULL + UNION + SELECT dataset.id + FROM datasets AS dataset + WHERE root_snapshot_id IS NOT NULL + AND dataset.source_snapshot_id = root_snapshot_id + ), descendants(dataset_id) AS ( + SELECT dataset_id FROM quarantine_roots + UNION + SELECT edge.child_dataset_id + FROM dataset_lineage_edges AS edge + JOIN descendants AS upstream ON edge.parent_dataset_id = upstream.dataset_id + ) + UPDATE dataset_versions AS version + SET validation_status = 'failed', + provenance_status = 'incomplete', + lineage_status = 'incomplete' + WHERE version.dataset_id IN (SELECT dataset_id FROM descendants); + + WITH RECURSIVE quarantine_roots(dataset_id) AS ( + SELECT root_dataset_id WHERE root_dataset_id IS NOT NULL + UNION + SELECT dataset.id + FROM datasets AS dataset + WHERE root_snapshot_id IS NOT NULL + AND dataset.source_snapshot_id = root_snapshot_id + ), descendants(dataset_id) AS ( + SELECT dataset_id FROM quarantine_roots + UNION + SELECT edge.child_dataset_id + FROM dataset_lineage_edges AS edge + JOIN descendants AS upstream ON edge.parent_dataset_id = upstream.dataset_id + ) + UPDATE datasets AS dataset + SET status = 'quarantined', + quarantine_status = 'quarantined', + validation_status = 'failed', + provenance_status = 'incomplete', + lineage_status = 'incomplete' + WHERE dataset.id IN (SELECT dataset_id FROM descendants); + END; + $$ LANGUAGE plpgsql; + + CREATE FUNCTION geointel_phase2_quarantine_state_guard() + RETURNS trigger AS $$ + DECLARE + owner_dataset uuid; + owner_snapshot uuid; + dataset_snapshot uuid; + BEGIN + IF NEW.status <> 'quarantined' THEN + RETURN NEW; + END IF; + + IF NEW.dataset_version_id IS NOT NULL THEN + SELECT dataset_id, source_snapshot_id + INTO owner_dataset, owner_snapshot + FROM dataset_versions + WHERE id = NEW.dataset_version_id; + + IF owner_dataset IS NULL THEN + RAISE EXCEPTION 'quarantine dataset version % has no owning dataset', NEW.dataset_version_id + USING ERRCODE = '23514'; + END IF; + IF NEW.dataset_id IS NOT NULL AND NEW.dataset_id <> owner_dataset THEN + RAISE EXCEPTION 'quarantine dataset and dataset version do not belong together' + USING ERRCODE = '23514'; + END IF; + + UPDATE dataset_versions + SET validation_status = 'failed', + provenance_status = 'incomplete', + lineage_status = 'incomplete' + WHERE id = NEW.dataset_version_id; + END IF; + + owner_dataset := COALESCE(NEW.dataset_id, owner_dataset); + IF owner_dataset IS NOT NULL THEN + SELECT source_snapshot_id + INTO dataset_snapshot + FROM datasets + WHERE id = owner_dataset; + + IF NEW.dataset_version_id IS NULL + AND NEW.source_snapshot_id IS NOT NULL + AND dataset_snapshot IS NOT NULL + AND NEW.source_snapshot_id <> dataset_snapshot THEN + RAISE EXCEPTION 'quarantine dataset and source snapshot do not belong together' + USING ERRCODE = '23514'; + END IF; + IF NEW.dataset_version_id IS NOT NULL + AND NEW.source_snapshot_id IS NOT NULL + AND owner_snapshot IS NOT NULL + AND NEW.source_snapshot_id <> owner_snapshot THEN + RAISE EXCEPTION 'quarantine dataset version and source snapshot do not belong together' + USING ERRCODE = '23514'; + END IF; + + UPDATE datasets + SET status = 'quarantined', + quarantine_status = 'quarantined', + validation_status = 'failed', + provenance_status = 'incomplete', + lineage_status = 'incomplete' + WHERE id = owner_dataset; + END IF; + + owner_snapshot := COALESCE(NEW.source_snapshot_id, owner_snapshot, dataset_snapshot); + IF owner_snapshot IS NOT NULL THEN + UPDATE source_snapshots + SET ingest_status = 'quarantined' + WHERE id = owner_snapshot; + + -- A concrete snapshot is shared immutable evidence. Once + -- quarantined, every Dataset/Version bound to it must be + -- non-consumable as well; a consumption gate must never + -- depend on callers having included source_snapshot_id in + -- the quarantine record. + UPDATE dataset_versions + SET validation_status = 'failed', + provenance_status = 'incomplete', + lineage_status = 'incomplete' + WHERE source_snapshot_id = owner_snapshot; + + UPDATE datasets + SET status = 'quarantined', + quarantine_status = 'quarantined', + validation_status = 'failed', + provenance_status = 'incomplete', + lineage_status = 'incomplete' + WHERE source_snapshot_id = owner_snapshot; + END IF; + + -- Direct quarantine and shared-snapshot fan-out are not + -- enough: every downstream derivative inherits the unsafe + -- lineage and must fail all Dataset consumption boundaries. + PERFORM geointel_phase2_quarantine_lineage_descendants(owner_dataset, owner_snapshot); + RETURN NEW; + END; + $$ LANGUAGE plpgsql; + + CREATE TRIGGER trg_dataset_quarantines_state_guard + AFTER INSERT OR UPDATE OF status, dataset_id, dataset_version_id, source_snapshot_id ON dataset_quarantines + FOR EACH ROW EXECUTE FUNCTION geointel_phase2_quarantine_state_guard(); + """ + ) + ) + + +def downgrade() -> None: + op.execute( + sa.text( + """ + DROP TRIGGER IF EXISTS trg_datasets_snapshot_registry_guard ON datasets; + DROP TRIGGER IF EXISTS trg_dataset_versions_snapshot_registry_guard ON dataset_versions; + DROP TRIGGER IF EXISTS trg_source_snapshots_registry_immutable ON source_snapshots; + DROP TRIGGER IF EXISTS trg_source_registry_write_guard ON source_registry; + DROP TRIGGER IF EXISTS trg_source_snapshots_evidence_immutable ON source_snapshots; + DROP TRIGGER IF EXISTS trg_datasets_contract_report_guard ON datasets; + DROP TRIGGER IF EXISTS trg_dataset_versions_contract_report_guard ON dataset_versions; + DROP TRIGGER IF EXISTS trg_dataset_lineage_edges_cycle_guard ON dataset_lineage_edges; + DROP TRIGGER IF EXISTS trg_dataset_lineage_edges_immutable ON dataset_lineage_edges; + DROP TRIGGER IF EXISTS trg_dataset_quarantines_state_guard ON dataset_quarantines; + DROP FUNCTION IF EXISTS geointel_phase2_snapshot_registry_guard(); + DROP FUNCTION IF EXISTS geointel_phase2_snapshot_registry_immutable_guard(); + DROP FUNCTION IF EXISTS geointel_phase2_source_registry_write_guard(); + DROP FUNCTION IF EXISTS geointel_phase2_snapshot_evidence_immutable_guard(); + DROP FUNCTION IF EXISTS geointel_phase2_contract_report_guard(); + DROP FUNCTION IF EXISTS geointel_phase2_dataset_version_contract_report_guard(); + DROP FUNCTION IF EXISTS geointel_phase2_lineage_cycle_guard(); + DROP FUNCTION IF EXISTS geointel_phase2_lineage_edge_immutable_guard(); + DROP FUNCTION IF EXISTS geointel_phase2_quarantine_lineage_descendants(uuid, uuid); + DROP FUNCTION IF EXISTS geointel_phase2_quarantine_state_guard(); + """ + ) + ) + op.drop_table("dataset_quarantines") + op.drop_table("dataset_lineage_edges") + + op.drop_constraint("ck_dataset_versions_ingest_key_not_blank", "dataset_versions", type_="check") + op.drop_constraint("ck_dataset_versions_lineage_status", "dataset_versions", type_="check") + op.drop_constraint("ck_dataset_versions_provenance_status", "dataset_versions", type_="check") + op.drop_constraint("ck_dataset_versions_validation_status", "dataset_versions", type_="check") + op.drop_constraint("ck_datasets_ingest_key_not_blank", "datasets", type_="check") + op.drop_constraint("ck_datasets_quarantine_status", "datasets", type_="check") + op.drop_constraint("ck_datasets_lineage_status", "datasets", type_="check") + op.drop_constraint("ck_datasets_provenance_status", "datasets", type_="check") + op.drop_constraint("ck_datasets_validation_status", "datasets", type_="check") + op.drop_constraint("uq_dataset_versions_dataset_ingest_key", "dataset_versions", type_="unique") + op.drop_constraint("uq_datasets_project_ingest_key", "datasets", type_="unique") + op.drop_index("ix_dataset_versions_source_registry_snapshot", table_name="dataset_versions") + op.drop_index("ix_datasets_source_registry_snapshot", table_name="datasets") + op.drop_constraint("fk_dataset_versions_source_snapshot", "dataset_versions", type_="foreignkey") + op.drop_constraint("fk_dataset_versions_source_registry", "dataset_versions", type_="foreignkey") + op.drop_constraint("fk_datasets_source_snapshot", "datasets", type_="foreignkey") + op.drop_constraint("fk_datasets_source_registry", "datasets", type_="foreignkey") + + for column_name in ( + "lineage_status", + "provenance_status", + "validation_report_json", + "validation_status", + "data_contract_version", + "data_contract_key", + "ingest_key", + "source_snapshot_id", + "source_registry_id", + ): + op.drop_column("dataset_versions", column_name) + for column_name in ( + "quarantine_status", + "lineage_status", + "provenance_status", + "validation_report_json", + "validation_status", + "data_contract_version", + "data_contract_key", + "ingest_key", + "source_snapshot_id", + "source_registry_id", + ): + op.drop_column("datasets", column_name) + + op.drop_index("ix_source_snapshots_checksum", table_name="source_snapshots") + op.drop_index("ix_source_snapshots_registry_fetched", table_name="source_snapshots") + op.drop_table("source_snapshots") + op.drop_table("source_registry") diff --git a/backend/app/api/routes/__init__.py b/backend/app/api/routes/__init__.py index 3b08153b..245026fe 100644 --- a/backend/app/api/routes/__init__.py +++ b/backend/app/api/routes/__init__.py @@ -1 +1,15 @@ -__all__ = ["analysis", "areas", "assistant", "auth", "datasets", "health", "projects", "exports", "jobs", "external", "qa", "temporal"] +__all__ = [ + "analysis", + "areas", + "assistant", + "auth", + "datasets", + "exports", + "external", + "health", + "jobs", + "projects", + "qa", + "source_registry", + "temporal", +] diff --git a/backend/app/api/routes/source_registry.py b/backend/app/api/routes/source_registry.py new file mode 100644 index 00000000..bb8e536e --- /dev/null +++ b/backend/app/api/routes/source_registry.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +from uuid import UUID + +from fastapi import APIRouter, Depends +from sqlalchemy import func, or_ +from sqlalchemy.orm import Session + +from app.core.errors import AppError +from app.db.session import get_db +from app.models import Dataset, DatasetLineageEdge, DatasetQuarantine, Project, SourceRegistry, SourceSnapshot +from app.schemas import ( + DatasetLineageEdgeRead, + DatasetProvenanceRead, + DatasetQuarantineRead, + Envelope, + ItemList, + SourceRegistryDetailRead, + SourceRegistryRead, + SourceSnapshotRead, +) +from app.utils.response import envelope + + +router = APIRouter(tags=["source-registry"]) + + +def _source_read(source: SourceRegistry, *, snapshot_count: int = 0) -> SourceRegistryRead: + return SourceRegistryRead.model_validate(source).model_copy(update={"snapshot_count": int(snapshot_count)}) + + +@router.get("/source-registry", response_model=Envelope[ItemList[SourceRegistryRead]]) +def list_source_registry( + classification: str | None = None, + db: Session = Depends(get_db), +) -> dict: + query = ( + db.query(SourceRegistry, func.count(SourceSnapshot.id).label("snapshot_count")) + .outerjoin(SourceSnapshot, SourceSnapshot.source_registry_id == SourceRegistry.id) + ) + if classification: + query = query.filter(SourceRegistry.classification == classification.strip().lower()) + rows = ( + query.group_by(SourceRegistry.id) + .order_by(SourceRegistry.classification.asc(), SourceRegistry.display_name.asc()) + .all() + ) + items = [_source_read(source, snapshot_count=count) for source, count in rows] + return envelope({"items": items, "total": len(items)}) + + +@router.get("/source-registry/{source_key}", response_model=Envelope[SourceRegistryDetailRead]) +def get_source_registry_entry(source_key: str, db: Session = Depends(get_db)) -> dict: + normalized_key = source_key.strip().lower() + source = db.query(SourceRegistry).filter(SourceRegistry.source_key == normalized_key).one_or_none() + if source is None: + raise AppError(code="SOURCE_REGISTRY_ENTRY_NOT_FOUND", message="Source registry entry was not found", status_code=404) + snapshots = ( + db.query(SourceSnapshot) + .filter(SourceSnapshot.source_registry_id == source.id) + .order_by(SourceSnapshot.fetched_at.desc(), SourceSnapshot.created_at.desc()) + .all() + ) + detail = SourceRegistryDetailRead( + source=_source_read(source, snapshot_count=len(snapshots)), + snapshots=[SourceSnapshotRead.model_validate(snapshot) for snapshot in snapshots], + ) + return envelope(detail) + + +@router.get( + "/projects/{project_id}/datasets/{dataset_id}/provenance", + response_model=Envelope[DatasetProvenanceRead], +) +def get_dataset_provenance( + project_id: UUID, + dataset_id: UUID, + db: Session = Depends(get_db), +) -> dict: + if db.get(Project, project_id) is None: + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + dataset = db.get(Dataset, dataset_id) + if dataset is None or dataset.project_id != project_id: + raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404) + + source = db.get(SourceRegistry, dataset.source_registry_id) if dataset.source_registry_id else None + snapshot = db.get(SourceSnapshot, dataset.source_snapshot_id) if dataset.source_snapshot_id else None + lineage = ( + db.query(DatasetLineageEdge) + .filter( + or_( + DatasetLineageEdge.parent_dataset_id == dataset.id, + DatasetLineageEdge.child_dataset_id == dataset.id, + ) + ) + .order_by(DatasetLineageEdge.created_at.asc(), DatasetLineageEdge.id.asc()) + .all() + ) + quarantines = ( + db.query(DatasetQuarantine) + .filter(DatasetQuarantine.dataset_id == dataset.id) + .order_by(DatasetQuarantine.created_at.desc(), DatasetQuarantine.id.desc()) + .all() + ) + result = DatasetProvenanceRead( + dataset_id=dataset.id, + source=_source_read(source) if source else None, + snapshot=SourceSnapshotRead.model_validate(snapshot) if snapshot else None, + data_contract_key=dataset.data_contract_key, + data_contract_version=dataset.data_contract_version, + validation_status=dataset.validation_status, + validation_report_json=dataset.validation_report_json, + provenance_status=dataset.provenance_status, + lineage_status=dataset.lineage_status, + quarantine_status=dataset.quarantine_status, + lineage=[DatasetLineageEdgeRead.model_validate(item) for item in lineage], + quarantines=[DatasetQuarantineRead.model_validate(item) for item in quarantines], + ) + return envelope(result) diff --git a/backend/app/main.py b/backend/app/main.py index 952fbf58..983135a4 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -12,7 +12,7 @@ from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import JSONResponse -from app.api.routes import analysis, aoi_operations, areas, assistant, auth, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation, selection_partitions, temporal +from app.api.routes import analysis, aoi_operations, areas, assistant, auth, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation, selection_partitions, source_registry, temporal from app.core.config import get_settings from app.core.errors import AppError from app.core.logging import configure_logging @@ -103,6 +103,7 @@ def create_app() -> FastAPI: app.include_router(quality_checks.router, prefix=settings.api_prefix) app.include_router(exports.router, prefix=settings.api_prefix) app.include_router(external.router, prefix=settings.api_prefix) + app.include_router(source_registry.router, prefix=settings.api_prefix) app.include_router(demo.router, prefix=settings.api_prefix) app.include_router(qa.router, prefix=settings.api_prefix) app.include_router(detection.router, prefix=settings.api_prefix) diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py index 7d8d5fcd..22649957 100644 --- a/backend/app/models/__init__.py +++ b/backend/app/models/__init__.py @@ -1,4 +1,24 @@ -from .entities import AoiOperation, AoiOperationPartition, AnalysisRun, Area, Dataset, DatasetVersion, Detection, DetectionReview, Export, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature +from .entities import ( + AoiOperation, + AoiOperationPartition, + AnalysisRun, + Area, + Dataset, + DatasetLineageEdge, + DatasetQuarantine, + DatasetVersion, + Detection, + DetectionReview, + Export, + Job, + Metric, + Project, + QualityCheck, + Segmentation, + SourceRegistry, + SourceSnapshot, + VectorFeature, +) __all__ = [ "AnalysisRun", @@ -6,6 +26,8 @@ __all__ = [ "AoiOperationPartition", "Area", "Dataset", + "DatasetLineageEdge", + "DatasetQuarantine", "DatasetVersion", "Detection", "DetectionReview", @@ -15,5 +37,7 @@ __all__ = [ "Project", "QualityCheck", "Segmentation", + "SourceRegistry", + "SourceSnapshot", "VectorFeature", ] diff --git a/backend/app/models/entities.py b/backend/app/models/entities.py index 499e82f0..4c47a813 100644 --- a/backend/app/models/entities.py +++ b/backend/app/models/entities.py @@ -12,6 +12,37 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship from app.db.base import Base +SOURCE_CLASSIFICATIONS = ( + "authoritative", + "corroborative", + "contextual", + "derived", + "experimental", +) +SOURCE_FRESHNESS_STATUSES = ( + "unknown", + "current", + "due", + "stale", + "not_applicable", + "review_required", +) +SOURCE_INGEST_STATUSES = ( + "registered", + "configured", + "not_configured", + "available", + "ingested", + "failed", + "quarantined", + "legacy_unverified", +) +PROVENANCE_STATUSES = ("complete", "incomplete", "not_applicable") +LINEAGE_STATUSES = ("complete", "incomplete", "not_applicable") +VALIDATION_STATUSES = ("not_validated", "passed", "failed") +QUARANTINE_STATUSES = ("not_quarantined", "quarantined") + + class Project(Base): __tablename__ = "projects" @@ -42,6 +73,123 @@ class Area(Base): project: Mapped[Project] = relationship("Project", back_populates="areas") +class SourceRegistry(Base): + """Server-owned source identity and authority contract. + + Dataset metadata remains descriptive until a governed importer binds a + dataset to both this registry entry and an immutable SourceSnapshot. + """ + + __tablename__ = "source_registry" + __table_args__ = ( + UniqueConstraint("source_key", name="uq_source_registry_source_key"), + CheckConstraint( + "classification IN ('authoritative', 'corroborative', 'contextual', 'derived', 'experimental')", + name="ck_source_registry_classification", + ), + CheckConstraint( + "freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')", + name="ck_source_registry_freshness_status", + ), + CheckConstraint( + "ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', " + "'failed', 'quarantined', 'legacy_unverified')", + name="ck_source_registry_ingest_status", + ), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + source_key: Mapped[str] = mapped_column(String(120), nullable=False) + display_name: Mapped[str] = mapped_column(String(255), nullable=False) + classification: Mapped[str] = mapped_column(String(32), nullable=False) + authority_name: Mapped[str] = mapped_column(String(255), nullable=False, default="unknown", server_default="unknown") + authority_scope_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + provider_adapter_key: Mapped[str | None] = mapped_column(String(120), nullable=True) + source_url: Mapped[str | None] = mapped_column(Text, nullable=True) + license_name: Mapped[str] = mapped_column(String(255), nullable=False, default="unknown", server_default="unknown") + license_url: Mapped[str | None] = mapped_column(Text, nullable=True) + usage_restrictions: Mapped[str] = mapped_column(Text, nullable=False, default="unknown", server_default="unknown") + default_crs: Mapped[str] = mapped_column(String(64), nullable=False, default="unknown", server_default="unknown") + default_units: Mapped[str] = mapped_column(String(120), nullable=False, default="unknown", server_default="unknown") + spatial_resolution_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + temporal_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + geographic_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + expected_geometry_types_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list) + expected_attributes_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + usage_policy_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + freshness_status: Mapped[str] = mapped_column( + String(32), nullable=False, default="unknown", server_default="unknown" + ) + ingest_status: Mapped[str] = mapped_column( + String(32), nullable=False, default="registered", server_default="registered" + ) + known_limitations_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list) + registry_metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + snapshots: Mapped[list["SourceSnapshot"]] = relationship( + "SourceSnapshot", back_populates="source_registry", cascade="all, delete-orphan" + ) + datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="source_registry") + dataset_versions: Mapped[list["DatasetVersion"]] = relationship("DatasetVersion", back_populates="source_registry") + + +class SourceSnapshot(Base): + """Immutable source-version evidence recorded by governed ingestion.""" + + __tablename__ = "source_snapshots" + __table_args__ = ( + UniqueConstraint("source_registry_id", "snapshot_key", name="uq_source_snapshots_registry_key"), + CheckConstraint( + "freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')", + name="ck_source_snapshots_freshness_status", + ), + CheckConstraint( + "ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', " + "'failed', 'quarantined', 'legacy_unverified')", + name="ck_source_snapshots_ingest_status", + ), + CheckConstraint( + "checksum_sha256 = lower(checksum_sha256) AND checksum_sha256 ~ '^[0-9a-f]{64}$'", + name="ck_source_snapshots_checksum_sha256", + ), + Index("ix_source_snapshots_registry_fetched", "source_registry_id", "fetched_at"), + Index("ix_source_snapshots_checksum", "checksum_sha256"), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + source_registry_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="CASCADE"), nullable=False + ) + snapshot_key: Mapped[str] = mapped_column(String(255), nullable=False) + source_version: Mapped[str | None] = mapped_column(String(120), nullable=True) + snapshot_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now()) + source_url: Mapped[str | None] = mapped_column(Text, nullable=True) + checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False) + crs: Mapped[str | None] = mapped_column(String(64), nullable=True) + units: Mapped[str | None] = mapped_column(String(120), nullable=True) + spatial_resolution_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + temporal_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + geographic_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + observed_schema_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + freshness_status: Mapped[str] = mapped_column( + String(32), nullable=False, default="unknown", server_default="unknown" + ) + ingest_status: Mapped[str] = mapped_column( + String(32), nullable=False, default="registered", server_default="registered" + ) + known_limitations_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list) + snapshot_metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + + source_registry: Mapped[SourceRegistry] = relationship("SourceRegistry", back_populates="snapshots") + datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="source_snapshot") + dataset_versions: Mapped[list["DatasetVersion"]] = relationship("DatasetVersion", back_populates="source_snapshot") + quarantines: Mapped[list["DatasetQuarantine"]] = relationship("DatasetQuarantine", back_populates="source_snapshot") + + class Dataset(Base): __tablename__ = "datasets" __table_args__ = ( @@ -49,6 +197,27 @@ class Dataset(Base): "valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from", name="ck_datasets_temporal_valid_range", ), + CheckConstraint( + "validation_status IN ('not_validated', 'passed', 'failed')", + name="ck_datasets_validation_status", + ), + CheckConstraint( + "provenance_status IN ('complete', 'incomplete', 'not_applicable')", + name="ck_datasets_provenance_status", + ), + CheckConstraint( + "lineage_status IN ('complete', 'incomplete', 'not_applicable')", + name="ck_datasets_lineage_status", + ), + CheckConstraint( + "quarantine_status IN ('not_quarantined', 'quarantined')", + name="ck_datasets_quarantine_status", + ), + CheckConstraint( + "ingest_key IS NULL OR btrim(ingest_key) <> ''", + name="ck_datasets_ingest_key_not_blank", + ), + UniqueConstraint("project_id", "ingest_key", name="uq_datasets_project_ingest_key"), Index( "ix_datasets_project_temporal_series_observed", "project_id", @@ -69,6 +238,7 @@ class Dataset(Base): content_type: Mapped[str | None] = mapped_column(String(120), nullable=True) size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True) checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True) + ingest_key: Mapped[str | None] = mapped_column(String(255), nullable=True) derived_from_dataset_id: Mapped[uuid.UUID | None] = mapped_column( UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), @@ -84,6 +254,43 @@ class Dataset(Base): reference_layer_name: Mapped[str | None] = mapped_column(String(120), nullable=True) source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True) provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True) + source_registry_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="SET NULL"), nullable=True + ) + source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True + ) + data_contract_key: Mapped[str | None] = mapped_column(String(120), nullable=True) + data_contract_version: Mapped[str | None] = mapped_column(String(64), nullable=True) + validation_status: Mapped[str] = mapped_column( + String(32), + nullable=False, + default="not_validated", + server_default="not_validated", + comment="not_validated | passed | failed", + ) + validation_report_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + provenance_status: Mapped[str] = mapped_column( + String(32), + nullable=False, + default="incomplete", + server_default="incomplete", + comment="complete | incomplete | not_applicable", + ) + lineage_status: Mapped[str] = mapped_column( + String(32), + nullable=False, + default="incomplete", + server_default="incomplete", + comment="complete | incomplete | not_applicable", + ) + quarantine_status: Mapped[str] = mapped_column( + String(32), + nullable=False, + default="not_quarantined", + server_default="not_quarantined", + comment="not_quarantined | quarantined", + ) imported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) temporal_series_key: Mapped[str | None] = mapped_column(String(255), nullable=True) observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) @@ -106,6 +313,23 @@ class Dataset(Base): back_populates="dataset", cascade="all, delete-orphan", ) + source_registry: Mapped[SourceRegistry | None] = relationship("SourceRegistry", back_populates="datasets") + source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="datasets") + parent_lineage_edges: Mapped[list["DatasetLineageEdge"]] = relationship( + "DatasetLineageEdge", + foreign_keys="DatasetLineageEdge.parent_dataset_id", + back_populates="parent_dataset", + cascade="all, delete-orphan", + ) + child_lineage_edges: Mapped[list["DatasetLineageEdge"]] = relationship( + "DatasetLineageEdge", + foreign_keys="DatasetLineageEdge.child_dataset_id", + back_populates="child_dataset", + cascade="all, delete-orphan", + ) + quarantines: Mapped[list["DatasetQuarantine"]] = relationship( + "DatasetQuarantine", back_populates="dataset", cascade="all, delete-orphan" + ) class DatasetVersion(Base): @@ -115,7 +339,24 @@ class DatasetVersion(Base): "valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from", name="ck_dataset_versions_temporal_valid_range", ), + CheckConstraint( + "validation_status IN ('not_validated', 'passed', 'failed')", + name="ck_dataset_versions_validation_status", + ), + CheckConstraint( + "provenance_status IN ('complete', 'incomplete', 'not_applicable')", + name="ck_dataset_versions_provenance_status", + ), + CheckConstraint( + "lineage_status IN ('complete', 'incomplete', 'not_applicable')", + name="ck_dataset_versions_lineage_status", + ), + CheckConstraint( + "ingest_key IS NULL OR btrim(ingest_key) <> ''", + name="ck_dataset_versions_ingest_key_not_blank", + ), Index("ix_dataset_versions_dataset_version", "dataset_id", "version", unique=True), + UniqueConstraint("dataset_id", "ingest_key", name="uq_dataset_versions_dataset_ingest_key"), ) id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) @@ -127,11 +368,135 @@ class DatasetVersion(Base): valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True) + ingest_key: Mapped[str | None] = mapped_column(String(255), nullable=True) source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True) provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True) + source_registry_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="SET NULL"), nullable=True + ) + source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True + ) + data_contract_key: Mapped[str | None] = mapped_column(String(120), nullable=True) + data_contract_version: Mapped[str | None] = mapped_column(String(64), nullable=True) + validation_status: Mapped[str] = mapped_column( + String(32), + nullable=False, + default="not_validated", + server_default="not_validated", + comment="not_validated | passed | failed", + ) + validation_report_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + provenance_status: Mapped[str] = mapped_column( + String(32), + nullable=False, + default="incomplete", + server_default="incomplete", + comment="complete | incomplete | not_applicable", + ) + lineage_status: Mapped[str] = mapped_column( + String(32), + nullable=False, + default="incomplete", + server_default="incomplete", + comment="complete | incomplete | not_applicable", + ) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) dataset: Mapped[Dataset] = relationship("Dataset", back_populates="versions") + source_registry: Mapped[SourceRegistry | None] = relationship("SourceRegistry", back_populates="dataset_versions") + source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="dataset_versions") + quarantines: Mapped[list["DatasetQuarantine"]] = relationship("DatasetQuarantine", back_populates="dataset_version") + + +class DatasetLineageEdge(Base): + """Immutable relationship between input/output datasets and transforms.""" + + __tablename__ = "dataset_lineage_edges" + __table_args__ = ( + CheckConstraint("parent_dataset_id <> child_dataset_id", name="ck_dataset_lineage_edges_distinct_datasets"), + UniqueConstraint( + "parent_dataset_id", + "child_dataset_id", + "relation_type", + "transformation_name", + name="uq_dataset_lineage_edges_relation", + ), + Index("ix_dataset_lineage_edges_parent", "parent_dataset_id"), + Index("ix_dataset_lineage_edges_child", "child_dataset_id"), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + parent_dataset_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False + ) + child_dataset_id: Mapped[uuid.UUID] = mapped_column( + UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False + ) + parent_dataset_version_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True + ) + child_dataset_version_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True + ) + relation_type: Mapped[str] = mapped_column(String(64), nullable=False) + transformation_name: Mapped[str] = mapped_column(String(255), nullable=False) + transformation_version: Mapped[str | None] = mapped_column(String(120), nullable=True) + parameters_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + input_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True) + output_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + + parent_dataset: Mapped[Dataset] = relationship( + "Dataset", foreign_keys=[parent_dataset_id], back_populates="parent_lineage_edges" + ) + child_dataset: Mapped[Dataset] = relationship( + "Dataset", foreign_keys=[child_dataset_id], back_populates="child_lineage_edges" + ) + + +class DatasetQuarantine(Base): + """Durable fail-closed record for rejected or doubtful source artifacts.""" + + __tablename__ = "dataset_quarantines" + __table_args__ = ( + CheckConstraint( + "dataset_id IS NOT NULL OR dataset_version_id IS NOT NULL OR source_snapshot_id IS NOT NULL", + name="ck_dataset_quarantines_target_present", + ), + CheckConstraint( + "status IN ('quarantined', 'released', 'rejected')", + name="ck_dataset_quarantines_status", + ), + Index("ix_dataset_quarantines_dataset_status", "dataset_id", "status"), + Index("ix_dataset_quarantines_snapshot_status", "source_snapshot_id", "status"), + ) + + id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + dataset_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True + ) + dataset_version_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True + ) + source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column( + UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True + ) + stage: Mapped[str] = mapped_column(String(64), nullable=False) + reason_code: Mapped[str] = mapped_column(String(120), nullable=False) + details_json: Mapped[dict | None] = mapped_column(JSON, nullable=True) + artifact_path: Mapped[str | None] = mapped_column(Text, nullable=True) + artifact_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True) + status: Mapped[str] = mapped_column( + String(32), nullable=False, default="quarantined", server_default="quarantined" + ) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now()) + resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + resolved_by: Mapped[str | None] = mapped_column(String(120), nullable=True) + + dataset: Mapped[Dataset | None] = relationship("Dataset", back_populates="quarantines") + dataset_version: Mapped[DatasetVersion | None] = relationship("DatasetVersion", back_populates="quarantines") + source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="quarantines") class VectorFeature(Base): diff --git a/backend/app/schemas/__init__.py b/backend/app/schemas/__init__.py index 60c0870f..d1d18db9 100644 --- a/backend/app/schemas/__init__.py +++ b/backend/app/schemas/__init__.py @@ -32,6 +32,14 @@ from .source_catalog import ( SourceCatalogProbeReport, SourceCatalogProbeSummary, ) +from .source_registry import ( + DatasetProvenanceRead, + DatasetLineageEdgeRead, + DatasetQuarantineRead, + SourceRegistryDetailRead, + SourceRegistryRead, + SourceSnapshotRead, +) from .grb_refresh import GrbRefreshLayerPlan, GrbRefreshPlan, GrbRefreshPlanSummary from .grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead from .official_vector import ( @@ -207,6 +215,12 @@ __all__ = [ "SourceCatalogProbeItem", "SourceCatalogProbeReport", "SourceCatalogProbeSummary", + "SourceRegistryRead", + "SourceRegistryDetailRead", + "SourceSnapshotRead", + "DatasetLineageEdgeRead", + "DatasetQuarantineRead", + "DatasetProvenanceRead", "GrbRefreshLayerPlan", "GrbRefreshPlan", "GrbRefreshPlanSummary", @@ -271,6 +285,10 @@ __all__ = [ "FloodHazardSelectionSummary", "BathymetryProfileAcquireRequest", "BathymetryProfileAcquisitionResult", + "BathymetryRasterMetric", + "BathymetryRasterSelectionRequest", + "BathymetryRasterSelectionResponse", + "BathymetryRasterSelectionSummary", "BathymetryPartitionFinalizeRequest", "BathymetryPartitionFinalizationResult", "BathymetrySourceProbeRead", diff --git a/backend/app/schemas/dataset.py b/backend/app/schemas/dataset.py index 5495380d..221b9f0c 100644 --- a/backend/app/schemas/dataset.py +++ b/backend/app/schemas/dataset.py @@ -35,6 +35,16 @@ class DatasetCreateResponse(BaseModel): reference_layer_name: str | None = None source_metadata: dict | None = None provenance_metadata: dict | None = None + ingest_key: str | None = None + source_registry_id: UUID | None = None + source_snapshot_id: UUID | None = None + data_contract_key: str | None = None + data_contract_version: str | None = None + validation_status: str | None = None + validation_report_json: dict | None = None + provenance_status: str | None = None + lineage_status: str | None = None + quarantine_status: str | None = None imported_at: datetime | None = None temporal_series_key: str | None = None observed_at: datetime | None = None @@ -97,6 +107,15 @@ class DatasetVersionRead(BaseModel): checksum_sha256: str | None = None source_metadata: dict | None = None provenance_metadata: dict | None = None + ingest_key: str | None = None + source_registry_id: UUID | None = None + source_snapshot_id: UUID | None = None + data_contract_key: str | None = None + data_contract_version: str | None = None + validation_status: str | None = None + validation_report_json: dict | None = None + provenance_status: str | None = None + lineage_status: str | None = None created_at: datetime | None = None model_config = {"from_attributes": True} diff --git a/backend/app/schemas/detection.py b/backend/app/schemas/detection.py index e11e5ea8..36f1777c 100644 --- a/backend/app/schemas/detection.py +++ b/backend/app/schemas/detection.py @@ -144,6 +144,8 @@ class YoloPreflightChecks(BaseModel): accelerator_ready: bool | None = None model_path_set: bool | None = None model_file_exists: bool | None = None + model_provenance_manifest_path: str | None = None + model_provenance_valid: bool | None = None model_load_requested: bool model_load_ok: bool | None = None manifest_path_set: bool | None = None diff --git a/backend/app/schemas/source_registry.py b/backend/app/schemas/source_registry.py new file mode 100644 index 00000000..ee4e3fd4 --- /dev/null +++ b/backend/app/schemas/source_registry.py @@ -0,0 +1,120 @@ +from __future__ import annotations + +from datetime import datetime +from uuid import UUID + +from pydantic import BaseModel, Field + + +class SourceRegistryRead(BaseModel): + """Read-only, server-owned source-authority definition.""" + + id: UUID + source_key: str + display_name: str + classification: str + authority_name: str + authority_scope_json: dict + provider_adapter_key: str | None = None + source_url: str | None = None + license_name: str + license_url: str | None = None + usage_restrictions: str + default_crs: str + default_units: str + spatial_resolution_json: dict + temporal_coverage_json: dict + geographic_coverage_json: dict + expected_geometry_types_json: list + expected_attributes_json: dict + usage_policy_json: dict + freshness_status: str + ingest_status: str + known_limitations_json: list + registry_metadata_json: dict + created_at: datetime | None = None + updated_at: datetime | None = None + snapshot_count: int = 0 + + model_config = {"from_attributes": True} + + +class SourceSnapshotRead(BaseModel): + """Immutable version/snapshot evidence attached to an imported dataset.""" + + id: UUID + source_registry_id: UUID + snapshot_key: str + source_version: str | None = None + snapshot_at: datetime | None = None + fetched_at: datetime | None = None + source_url: str | None = None + checksum_sha256: str | None = None + crs: str | None = None + units: str | None = None + spatial_resolution_json: dict + temporal_coverage_json: dict + geographic_coverage_json: dict + observed_schema_json: dict + freshness_status: str + ingest_status: str + known_limitations_json: list + snapshot_metadata_json: dict + created_at: datetime | None = None + + model_config = {"from_attributes": True} + + +class SourceRegistryDetailRead(BaseModel): + source: SourceRegistryRead + snapshots: list[SourceSnapshotRead] + + +class DatasetLineageEdgeRead(BaseModel): + id: UUID + parent_dataset_id: UUID + child_dataset_id: UUID + parent_dataset_version_id: UUID | None = None + child_dataset_version_id: UUID | None = None + relation_type: str + transformation_name: str + transformation_version: str | None = None + parameters_json: dict | None = None + input_checksum_sha256: str | None = None + output_checksum_sha256: str | None = None + created_at: datetime | None = None + + model_config = {"from_attributes": True} + + +class DatasetQuarantineRead(BaseModel): + id: UUID + dataset_id: UUID | None = None + dataset_version_id: UUID | None = None + source_snapshot_id: UUID | None = None + stage: str + reason_code: str + details_json: dict | None = None + artifact_path: str | None = None + artifact_checksum_sha256: str | None = None + status: str + created_at: datetime | None = None + resolved_at: datetime | None = None + resolved_by: str | None = None + + model_config = {"from_attributes": True} + + +class DatasetProvenanceRead(BaseModel): + dataset_id: UUID + source: SourceRegistryRead | None = None + snapshot: SourceSnapshotRead | None = None + data_contract_key: str | None = None + data_contract_version: str | None = None + validation_status: str | None = None + validation_report_json: dict | None = None + provenance_status: str | None = None + lineage_status: str | None = None + quarantine_status: str | None = None + lineage: list[DatasetLineageEdgeRead] = Field(default_factory=list) + quarantines: list[DatasetQuarantineRead] = Field(default_factory=list) diff --git a/backend/app/services/coverage_registry_service.py b/backend/app/services/coverage_registry_service.py index 404c0f4e..4e2ef199 100644 --- a/backend/app/services/coverage_registry_service.py +++ b/backend/app/services/coverage_registry_service.py @@ -12,6 +12,7 @@ from sqlalchemy.orm import Session from app.core.errors import AppError from app.models import Area, Dataset, Project +from app.services.dataset_consumption_gate_service import DatasetConsumptionGate from app.schemas.coverage import ( CoverageBBox, CoverageCatalogResponse, @@ -463,6 +464,10 @@ class CoverageRegistryService: for dataset in datasets: if dataset.status != "ready" or dataset.source_name not in definition.materialized_source_names: continue + # A source-name claim alone must not cause an unsafe artifact to + # appear as operational authoritative coverage. + if not DatasetConsumptionGate.eligible_for_authoritative_coverage(dataset): + continue layer_names = definition.materialized_layer_names if definition.contract.source_name == "digitaal_vlaanderen": theme_sources = FLANDERS_THEME_DATASETS.get(theme, {}) diff --git a/backend/app/services/data_contract_validation.py b/backend/app/services/data_contract_validation.py new file mode 100644 index 00000000..b7365fe5 --- /dev/null +++ b/backend/app/services/data_contract_validation.py @@ -0,0 +1,2081 @@ +"""Versioned, fail-closed validation contracts for GeoIntel data assets. + +This module intentionally has no ORM, route or storage dependency. Import +services build :class:`DataAssetValidationInput` from a staged artifact and +persist the report/decision in their own transaction. Keeping validation pure +makes it safe to run before an artifact is eligible for training, inference or +publication. + +The contracts are deliberately explicit: an unknown contract version, an +unknown CRS, a missing checksum, incomplete lineage or an uncertain temporal +claim is a validation failure rather than a best-effort import. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timedelta, timezone +from enum import StrEnum +from hashlib import sha256 +from math import isfinite +from numbers import Integral, Real +from typing import Any, Iterable, Mapping, Sequence +import json +import re + +from pyproj import CRS +from shapely.geometry import shape +from shapely.geometry.base import BaseGeometry +from shapely.strtree import STRtree + + +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$") + + +class ContractKind(StrEnum): + """The supported top-level artifact families.""" + + RASTER = "raster" + VECTOR = "vector" + LABEL = "label" + MODEL = "model" + + +class ValidationStatus(StrEnum): + """Persisted validation status agreed for Phase 2 provenance fields.""" + + PASSED = "passed" + FAILED = "failed" + + +class ProvenanceStatus(StrEnum): + COMPLETE = "complete" + INCOMPLETE = "incomplete" + NOT_APPLICABLE = "not_applicable" + + +class LineageStatus(StrEnum): + COMPLETE = "complete" + INCOMPLETE = "incomplete" + NOT_APPLICABLE = "not_applicable" + + +class QuarantineStatus(StrEnum): + NOT_QUARANTINED = "not_quarantined" + QUARANTINED = "quarantined" + + +class IssueSeverity(StrEnum): + ERROR = "error" + WARNING = "warning" + + +class RequirementLevel(StrEnum): + REQUIRED = "required" + OPTIONAL = "optional" + NOT_APPLICABLE = "not_applicable" + UNKNOWN_WITH_REASON = "unknown_with_reason" + + +@dataclass(frozen=True) +class BoundingBox: + """A numeric bounding box in the explicitly declared coordinate system.""" + + min_x: float + min_y: float + max_x: float + max_y: float + + @classmethod + def from_value(cls, value: BoundingBox | Mapping[str, Any] | Sequence[float]) -> BoundingBox: + if isinstance(value, BoundingBox): + return value + if isinstance(value, Mapping): + try: + return cls( + min_x=float(value.get("min_x", value.get("minx"))), + min_y=float(value.get("min_y", value.get("miny"))), + max_x=float(value.get("max_x", value.get("maxx"))), + max_y=float(value.get("max_y", value.get("maxy"))), + ) + except (TypeError, ValueError) as exc: + raise ValueError("Bounding box mapping requires min/max x/y values") from exc + if isinstance(value, Sequence) and not isinstance(value, (str, bytes)) and len(value) == 4: + try: + return cls(*(float(item) for item in value)) + except (TypeError, ValueError) as exc: + raise ValueError("Bounding box values must be numeric") from exc + raise ValueError("Bounding box must be a four-value sequence or mapping") + + def is_valid(self) -> bool: + values = (self.min_x, self.min_y, self.max_x, self.max_y) + return all(isfinite(value) for value in values) and self.min_x <= self.max_x and self.min_y <= self.max_y + + def contains(self, other: BoundingBox, *, tolerance: float = 0.0) -> bool: + return ( + self.min_x - tolerance <= other.min_x + and self.min_y - tolerance <= other.min_y + and self.max_x + tolerance >= other.max_x + and self.max_y + tolerance >= other.max_y + ) + + def nearly_equals(self, other: BoundingBox, *, tolerance: float) -> bool: + return all( + abs(left - right) <= tolerance + for left, right in zip( + (self.min_x, self.min_y, self.max_x, self.max_y), + (other.min_x, other.min_y, other.max_x, other.max_y), + strict=True, + ) + ) + + def to_dict(self) -> dict[str, float]: + return { + "min_x": self.min_x, + "min_y": self.min_y, + "max_x": self.max_x, + "max_y": self.max_y, + } + + +@dataclass(frozen=True) +class Resolution: + """Explicit raster ground/sample resolution; no implicit unit conversion.""" + + x: float + y: float + unit: str + + @classmethod + def from_value(cls, value: Resolution | Mapping[str, Any] | Sequence[Any]) -> Resolution: + if isinstance(value, Resolution): + return value + if isinstance(value, Mapping): + try: + return cls(float(value["x"]), float(value["y"]), str(value["unit"]).strip()) + except (KeyError, TypeError, ValueError) as exc: + raise ValueError("Resolution mapping requires x, y and unit") from exc + if isinstance(value, Sequence) and not isinstance(value, (str, bytes)) and len(value) == 3: + try: + return cls(float(value[0]), float(value[1]), str(value[2]).strip()) + except (TypeError, ValueError) as exc: + raise ValueError("Resolution values must contain numeric x/y and a unit") from exc + raise ValueError("Resolution must be a mapping or three-value sequence") + + def is_valid(self) -> bool: + return isfinite(self.x) and isfinite(self.y) and self.x > 0.0 and self.y > 0.0 and bool(self.unit) + + def to_dict(self) -> dict[str, float | str]: + return {"x": self.x, "y": self.y, "unit": self.unit} + + +@dataclass(frozen=True) +class AttributeRule: + """An expected feature attribute and its portable JSON type contract.""" + + name: str + required: bool = True + nullable: bool = False + accepted_types: tuple[str, ...] = ("string",) + allowed_values: frozenset[Any] = frozenset() + + +@dataclass(frozen=True) +class GeometryRules: + allowed_geometry_types: frozenset[str] = frozenset() + attribute_rules: tuple[AttributeRule, ...] = () + unique_attribute_fields: tuple[str, ...] = () + require_features: bool = True + forbid_shared_area: bool = False + topology_max_features: int = 10_000 + + +@dataclass(frozen=True) +class RasterRules: + required_profile_fields: tuple[str, ...] = ("width", "height", "band_count", "dtype") + allowed_band_counts: frozenset[int] = frozenset() + allowed_dtypes: frozenset[str] = frozenset() + + +@dataclass(frozen=True) +class LabelRules: + allowed_class_ids: frozenset[int] = frozenset() + normalized_coordinates: bool = True + required_fields: tuple[str, ...] = ("class_id", "x_center", "y_center", "width", "height") + # Empty YOLO text files are not implicit negatives. A later contract + # version can permit them only when the caller declares and evidences a + # reviewed pure-background sample. + allow_empty_pure_background: bool = False + pure_background_required_metadata_fields: tuple[str, ...] = () + allowed_pure_background_splits: frozenset[str] = frozenset() + + +@dataclass(frozen=True) +class ModelRules: + required_fields: tuple[str, ...] = ("model_format", "framework", "class_mapping") + allowed_formats: frozenset[str] = frozenset() + minimum_class_count: int | None = None + + +@dataclass(frozen=True) +class ResolutionRules: + required: bool = True + allowed_units: frozenset[str] = frozenset({"m"}) + min_x: float | None = None + max_x: float | None = None + min_y: float | None = None + max_y: float | None = None + + +@dataclass(frozen=True) +class FreshnessRules: + observed_at: RequirementLevel = RequirementLevel.OPTIONAL + source_version: RequirementLevel = RequirementLevel.OPTIONAL + imported_at_required: bool = True + max_age: timedelta | None = None + allow_future_observation: bool = False + + +@dataclass(frozen=True) +class LineageRules: + """The evidence required before an artifact can be considered traceable.""" + + require_source_registry: bool = True + require_source_snapshot: bool = True + require_upstream_assets: bool = False + require_transformation_when_crs_changes: bool = True + + +@dataclass(frozen=True) +class TransformationEvidence: + name: str + version: str + checksum_sha256: str + + +@dataclass(frozen=True) +class LineageEvidence: + upstream_asset_ids: tuple[str, ...] = () + upstream_checksums_sha256: tuple[str, ...] = () + transformations: tuple[TransformationEvidence, ...] = () + + +@dataclass(frozen=True) +class GeometryRecord: + """A geometry plus the source attributes needed for vector schema checks.""" + + geometry: BaseGeometry | Mapping[str, Any] + properties: Mapping[str, Any] = field(default_factory=dict) + identifier: str | None = None + + +@dataclass(frozen=True) +class DataContract: + """A schema and evidence contract identified by immutable key/version.""" + + key: str + version: str + kind: ContractKind + accepted_source_crs: frozenset[str] = frozenset() + canonical_storage_crs: str | None = None + require_storage_crs: bool = True + spatial_domain: BoundingBox | None = None + bounds_tolerance: float = 0.0 + require_bounds: bool = False + require_checksum: bool = True + required_metadata_fields: tuple[str, ...] = () + metadata_checksum_fields: tuple[str, ...] = () + expected_units: Mapping[str, frozenset[str]] = field(default_factory=dict) + geometry_rules: GeometryRules | None = None + raster_rules: RasterRules | None = None + label_rules: LabelRules | None = None + model_rules: ModelRules | None = None + resolution_rules: ResolutionRules | None = None + freshness_rules: FreshnessRules = field(default_factory=FreshnessRules) + lineage_rules: LineageRules = field(default_factory=LineageRules) + quarantine_on_warning: bool = True + + def fingerprint(self) -> str: + """Return a deterministic hash of the schema semantics, not a filename.""" + + payload = { + "key": self.key, + "version": self.version, + "kind": self.kind.value, + "accepted_source_crs": sorted(self.accepted_source_crs), + "canonical_storage_crs": self.canonical_storage_crs, + "require_storage_crs": self.require_storage_crs, + "spatial_domain": self.spatial_domain.to_dict() if self.spatial_domain else None, + "bounds_tolerance": self.bounds_tolerance, + "require_bounds": self.require_bounds, + "require_checksum": self.require_checksum, + "required_metadata_fields": list(self.required_metadata_fields), + "metadata_checksum_fields": list(self.metadata_checksum_fields), + "expected_units": {key: sorted(value) for key, value in sorted(self.expected_units.items())}, + "geometry_rules": _geometry_rules_payload(self.geometry_rules), + "raster_rules": _raster_rules_payload(self.raster_rules), + "label_rules": _label_rules_payload(self.label_rules), + "model_rules": _model_rules_payload(self.model_rules), + "resolution_rules": _resolution_rules_payload(self.resolution_rules), + "freshness_rules": _freshness_rules_payload(self.freshness_rules), + "lineage_rules": _lineage_rules_payload(self.lineage_rules), + "quarantine_on_warning": self.quarantine_on_warning, + } + return _stable_sha256(payload) + + +@dataclass(frozen=True) +class DataAssetValidationInput: + """Validated metadata from an already staged artifact. + + ``content`` is optional for large files. In that case the caller must + supply a trusted ``computed_checksum_sha256`` calculated while streaming + the staged bytes; a filename alone never satisfies checksum validation. + Geometry coordinates are expected in ``storage_crs``. + """ + + asset_id: str + data_contract_key: str + data_contract_version: str + kind: ContractKind + source_crs: str | None = None + storage_crs: str | None = None + bounds: BoundingBox | Mapping[str, Any] | Sequence[float] | None = None + checksum_sha256: str | None = None + computed_checksum_sha256: str | None = None + content: bytes | None = None + metadata: Mapping[str, Any] = field(default_factory=dict) + units: Mapping[str, str] = field(default_factory=dict) + resolution: Resolution | Mapping[str, Any] | Sequence[Any] | None = None + # Vector checks make a bounds pass and a schema pass. Callers with a + # large partitioned source must therefore supply a *re-iterable* + # collection, not a one-shot generator. This permits bounded-memory + # validation without weakening any geometry or attribute checks. + geometry_records: Iterable[GeometryRecord] = () + raster_profile: Mapping[str, Any] = field(default_factory=dict) + label_records: tuple[Mapping[str, Any], ...] = () + label_mode: str = "objects" + model_metadata: Mapping[str, Any] = field(default_factory=dict) + source_registry_id: str | None = None + source_snapshot_id: str | None = None + lineage: LineageEvidence = field(default_factory=LineageEvidence) + imported_at: datetime | None = None + observed_at: datetime | None = None + valid_from: datetime | None = None + valid_to: datetime | None = None + temporal_unknown_reason: str | None = None + source_version: str | None = None + source_version_unknown_reason: str | None = None + + +@dataclass(frozen=True) +class ValidationIssue: + code: str + category: str + message: str + severity: IssueSeverity = IssueSeverity.ERROR + field: str | None = None + expected: Any = None + observed: Any = None + + def to_dict(self) -> dict[str, Any]: + return { + "code": self.code, + "category": self.category, + "message": self.message, + "severity": self.severity.value, + "field": self.field, + "expected": _json_safe(self.expected), + "observed": _json_safe(self.observed), + } + + +@dataclass(frozen=True) +class ValidationReport: + asset_id: str + data_contract_key: str + data_contract_version: str + contract_fingerprint_sha256: str | None + validation_status: ValidationStatus + provenance_status: ProvenanceStatus + lineage_status: LineageStatus + quarantine_status: QuarantineStatus + validation_scope: tuple[str, ...] + checked_at: datetime + issues: tuple[ValidationIssue, ...] = () + + @property + def report_sha256(self) -> str: + return _stable_sha256(self.to_dict(include_hash=False)) + + @property + def failed(self) -> bool: + return self.validation_status == ValidationStatus.FAILED + + def to_dict(self, *, include_hash: bool = True) -> dict[str, Any]: + payload: dict[str, Any] = { + "asset_id": self.asset_id, + "data_contract_key": self.data_contract_key, + "data_contract_version": self.data_contract_version, + "contract_fingerprint_sha256": self.contract_fingerprint_sha256, + "validation_status": self.validation_status.value, + "provenance_status": self.provenance_status.value, + "lineage_status": self.lineage_status.value, + "quarantine_status": self.quarantine_status.value, + "validation_scope": list(self.validation_scope), + "checked_at": _datetime_payload(self.checked_at), + "issues": [issue.to_dict() for issue in self.issues], + } + if include_hash: + payload["report_sha256"] = self.report_sha256 + return payload + + def persistence_fields(self) -> dict[str, Any]: + """Fields that map directly to the additive Phase 2 Dataset columns.""" + + return { + "data_contract_key": self.data_contract_key, + "data_contract_version": self.data_contract_version, + "validation_status": self.validation_status.value, + "validation_report_json": self.to_dict(), + "provenance_status": self.provenance_status.value, + "lineage_status": self.lineage_status.value, + "quarantine_status": self.quarantine_status.value, + } + + +class DataContractRegistry: + """An in-memory exact-version registry; no implicit latest-version lookup.""" + + def __init__(self, contracts: Iterable[DataContract] = ()) -> None: + self._contracts: dict[tuple[str, str], DataContract] = {} + for contract in contracts: + self.register(contract) + + def register(self, contract: DataContract) -> None: + key = _contract_identity(contract.key, contract.version) + if key in self._contracts: + raise ValueError(f"Data contract {contract.key}@{contract.version} is already registered") + self._contracts[key] = contract + + def resolve(self, key: str, version: str) -> DataContract | None: + try: + identity = _contract_identity(key, version) + except (AttributeError, ValueError): + return None + return self._contracts.get(identity) + + def registered_contracts(self) -> tuple[DataContract, ...]: + """Return every exact contract identity in deterministic order. + + Audit/evidence tooling may enumerate contracts, but callers still have + to resolve a concrete key/version to validate an asset. There is no + implicit "latest" policy. + """ + + return tuple( + contract + for _identity, contract in sorted(self._contracts.items(), key=lambda item: item[0]) + ) + + def validate(self, asset: DataAssetValidationInput, *, now: datetime | None = None) -> ValidationReport: + contract = self.resolve(asset.data_contract_key, asset.data_contract_version) + if contract is None: + issue = ValidationIssue( + code="DATA_CONTRACT_UNKNOWN", + category="contract", + field="data_contract_key", + message="No exact data contract version is registered for this asset.", + expected="registered key and version", + observed=f"{asset.data_contract_key}@{asset.data_contract_version}", + ) + return _failed_unknown_contract_report(asset, issue, now=now) + return DataContractValidator.validate(contract, asset, now=now) + + +class DataContractValidator: + """Pure validation engine used by staged imports and derived-artifact jobs.""" + + @classmethod + def validate( + cls, + contract: DataContract, + asset: DataAssetValidationInput, + *, + now: datetime | None = None, + ) -> ValidationReport: + checked_at = _as_utc(now) or datetime.now(timezone.utc) + issues: list[ValidationIssue] = [] + cls._check_identity(contract, asset, issues) + cls._check_checksum(contract, asset, issues) + cls._check_metadata(contract, asset, issues) + cls._check_temporal(contract, asset, checked_at, issues) + cls._check_provenance_and_lineage(contract, asset, issues) + cls._check_crs_and_bounds(contract, asset, issues) + cls._check_units(contract, asset, issues) + cls._check_resolution(contract, asset, issues) + + if contract.kind == ContractKind.VECTOR: + cls._check_vector(contract, asset, issues) + elif contract.kind == ContractKind.RASTER: + cls._check_raster(contract, asset, issues) + elif contract.kind == ContractKind.LABEL: + cls._check_labels(contract, asset, issues) + elif contract.kind == ContractKind.MODEL: + cls._check_model(contract, asset, issues) + + has_error = any(issue.severity == IssueSeverity.ERROR for issue in issues) + has_warning = any(issue.severity == IssueSeverity.WARNING for issue in issues) + quarantined = has_error or (has_warning and contract.quarantine_on_warning) + validation_status = ValidationStatus.FAILED if quarantined else ValidationStatus.PASSED + provenance_status = _provenance_status(contract, issues) + lineage_status = _lineage_status(contract, issues) + return ValidationReport( + asset_id=asset.asset_id, + data_contract_key=contract.key, + data_contract_version=contract.version, + contract_fingerprint_sha256=contract.fingerprint(), + validation_status=validation_status, + provenance_status=provenance_status, + lineage_status=lineage_status, + quarantine_status=(QuarantineStatus.QUARANTINED if quarantined else QuarantineStatus.NOT_QUARANTINED), + validation_scope=_validation_scope(contract), + checked_at=checked_at, + issues=tuple(issues), + ) + + @staticmethod + def _check_identity( + contract: DataContract, + asset: DataAssetValidationInput, + issues: list[ValidationIssue], + ) -> None: + if not asset.asset_id.strip(): + _issue(issues, "ASSET_ID_REQUIRED", "identity", "asset_id", "A non-empty asset id is required.") + if asset.data_contract_key != contract.key or asset.data_contract_version != contract.version: + _issue( + issues, + "DATA_CONTRACT_IDENTITY_MISMATCH", + "contract", + "data_contract_key", + "Asset contract identity does not match the supplied validator contract.", + expected=f"{contract.key}@{contract.version}", + observed=f"{asset.data_contract_key}@{asset.data_contract_version}", + ) + if asset.kind != contract.kind: + _issue( + issues, + "DATA_KIND_MISMATCH", + "contract", + "kind", + "Asset kind does not match the selected data contract.", + expected=contract.kind.value, + observed=asset.kind.value, + ) + + @staticmethod + def _check_checksum( + contract: DataContract, + asset: DataAssetValidationInput, + issues: list[ValidationIssue], + ) -> None: + declared = _normalise_checksum(asset.checksum_sha256) + computed = _normalise_checksum(asset.computed_checksum_sha256) + if asset.checksum_sha256 and declared is None: + _issue(issues, "CHECKSUM_FORMAT_INVALID", "checksum", "checksum_sha256", "Checksum must be a lowercase SHA-256 hex digest.") + if asset.computed_checksum_sha256 and computed is None: + _issue( + issues, + "COMPUTED_CHECKSUM_FORMAT_INVALID", + "checksum", + "computed_checksum_sha256", + "Computed checksum must be a lowercase SHA-256 hex digest.", + ) + if asset.content is not None: + actual = sha256(asset.content).hexdigest() + if computed is not None and computed != actual: + _issue( + issues, + "COMPUTED_CHECKSUM_MISMATCH", + "checksum", + "computed_checksum_sha256", + "Provided computed checksum does not match staged bytes.", + expected=actual, + observed=computed, + ) + computed = actual + if contract.require_checksum and (declared is None or computed is None): + _issue( + issues, + "CHECKSUM_EVIDENCE_REQUIRED", + "checksum", + "checksum_sha256", + "Both declared and computed checksum evidence are required before use.", + ) + if declared is not None and computed is not None and declared != computed: + _issue( + issues, + "CHECKSUM_MISMATCH", + "checksum", + "checksum_sha256", + "Declared checksum does not match the staged artifact checksum.", + expected=computed, + observed=declared, + ) + + @staticmethod + def _check_metadata( + contract: DataContract, + asset: DataAssetValidationInput, + issues: list[ValidationIssue], + ) -> None: + for field_name in contract.required_metadata_fields: + value = asset.metadata.get(field_name) + if value is None or (isinstance(value, str) and not value.strip()): + _issue( + issues, + "METADATA_FIELD_REQUIRED", + "metadata", + f"metadata.{field_name}", + "Required metadata field is missing or empty.", + expected=field_name, + observed=value, + ) + for field_name in contract.metadata_checksum_fields: + value = asset.metadata.get(field_name) + if _normalise_checksum(value) is None: + _issue( + issues, + "METADATA_CHECKSUM_INVALID", + "checksum", + f"metadata.{field_name}", + "Metadata checksum must be a lowercase SHA-256 digest.", + observed=value, + ) + + @classmethod + def _check_temporal( + cls, + contract: DataContract, + asset: DataAssetValidationInput, + checked_at: datetime, + issues: list[ValidationIssue], + ) -> None: + rules = contract.freshness_rules + observed_at = _as_utc(asset.observed_at) + imported_at = _as_utc(asset.imported_at) + valid_from = _as_utc(asset.valid_from) + valid_to = _as_utc(asset.valid_to) + + if rules.imported_at_required and imported_at is None: + _issue(issues, "IMPORT_TIMESTAMP_REQUIRED", "temporal", "imported_at", "Import timestamp is required.") + if asset.imported_at is not None and imported_at is None: + _issue(issues, "IMPORT_TIMESTAMP_INVALID", "temporal", "imported_at", "Import timestamp must be timezone-aware.") + if asset.observed_at is not None and observed_at is None: + _issue(issues, "OBSERVATION_TIMESTAMP_INVALID", "temporal", "observed_at", "Observation timestamp must be timezone-aware.") + if rules.observed_at == RequirementLevel.REQUIRED and observed_at is None: + _issue(issues, "OBSERVATION_TIMESTAMP_REQUIRED", "temporal", "observed_at", "Observation timestamp is required by this contract.") + if rules.observed_at == RequirementLevel.UNKNOWN_WITH_REASON and observed_at is None and not _nonempty(asset.temporal_unknown_reason): + _issue( + issues, + "OBSERVATION_UNKNOWN_REASON_REQUIRED", + "temporal", + "temporal_unknown_reason", + "A documented reason is required when observation time is unknown.", + ) + if rules.observed_at == RequirementLevel.NOT_APPLICABLE and observed_at is not None: + _issue( + issues, + "OBSERVATION_TIMESTAMP_NOT_APPLICABLE", + "temporal", + "observed_at", + "This contract does not permit an observation timestamp claim.", + ) + if rules.source_version == RequirementLevel.REQUIRED and not _nonempty(asset.source_version): + _issue(issues, "SOURCE_VERSION_REQUIRED", "temporal", "source_version", "Source version is required by this contract.") + if rules.source_version == RequirementLevel.UNKNOWN_WITH_REASON and not _nonempty(asset.source_version) and not _nonempty(asset.source_version_unknown_reason): + _issue( + issues, + "SOURCE_VERSION_UNKNOWN_REASON_REQUIRED", + "temporal", + "source_version_unknown_reason", + "A documented reason is required when source version is unknown.", + ) + if rules.source_version == RequirementLevel.NOT_APPLICABLE and _nonempty(asset.source_version): + _issue( + issues, + "SOURCE_VERSION_NOT_APPLICABLE", + "temporal", + "source_version", + "This contract does not permit a source version claim.", + ) + if valid_from is not None and valid_to is not None and valid_to < valid_from: + _issue( + issues, + "VALIDITY_RANGE_INVALID", + "temporal", + "valid_to", + "valid_to must be on or after valid_from.", + expected=_datetime_payload(valid_from), + observed=_datetime_payload(valid_to), + ) + if observed_at is not None and not rules.allow_future_observation and observed_at > checked_at: + _issue( + issues, + "OBSERVATION_IN_FUTURE", + "freshness", + "observed_at", + "Observation time cannot be in the future for this contract.", + expected=f"<= {_datetime_payload(checked_at)}", + observed=_datetime_payload(observed_at), + ) + if rules.max_age is not None and observed_at is not None and checked_at - observed_at > rules.max_age: + _issue( + issues, + "FRESHNESS_EXCEEDED", + "freshness", + "observed_at", + "Observation evidence exceeds this contract's maximum age.", + expected=f"at most {rules.max_age.total_seconds()} seconds old", + observed=f"{(checked_at - observed_at).total_seconds()} seconds old", + ) + + @staticmethod + def _check_provenance_and_lineage( + contract: DataContract, + asset: DataAssetValidationInput, + issues: list[ValidationIssue], + ) -> None: + rules = contract.lineage_rules + if rules.require_source_registry and not _nonempty(asset.source_registry_id): + _issue( + issues, + "SOURCE_REGISTRY_REQUIRED", + "provenance", + "source_registry_id", + "A server-attested source registry id is required.", + ) + if rules.require_source_snapshot and not _nonempty(asset.source_snapshot_id): + _issue( + issues, + "SOURCE_SNAPSHOT_REQUIRED", + "provenance", + "source_snapshot_id", + "An immutable source snapshot id is required.", + ) + if rules.require_upstream_assets: + if not asset.lineage.upstream_asset_ids or not asset.lineage.upstream_checksums_sha256: + _issue( + issues, + "UPSTREAM_LINEAGE_REQUIRED", + "lineage", + "lineage.upstream_asset_ids", + "Derived artifacts require upstream asset ids and checksums.", + ) + elif len(asset.lineage.upstream_asset_ids) != len(asset.lineage.upstream_checksums_sha256): + _issue( + issues, + "UPSTREAM_LINEAGE_CARDINALITY_INVALID", + "lineage", + "lineage", + "Each upstream asset id must have one corresponding checksum.", + expected=len(asset.lineage.upstream_asset_ids), + observed=len(asset.lineage.upstream_checksums_sha256), + ) + for checksum in asset.lineage.upstream_checksums_sha256: + if _normalise_checksum(checksum) is None: + _issue( + issues, + "UPSTREAM_CHECKSUM_FORMAT_INVALID", + "lineage", + "lineage.upstream_checksums_sha256", + "Each upstream checksum must be a lowercase SHA-256 digest.", + observed=checksum, + ) + for transformation in asset.lineage.transformations: + if not _nonempty(transformation.name) or not _nonempty(transformation.version) or _normalise_checksum(transformation.checksum_sha256) is None: + _issue( + issues, + "TRANSFORMATION_EVIDENCE_INVALID", + "lineage", + "lineage.transformations", + "Every transformation requires name, version and checksum.", + observed={ + "name": transformation.name, + "version": transformation.version, + "checksum_sha256": transformation.checksum_sha256, + }, + ) + + @classmethod + def _check_crs_and_bounds( + cls, + contract: DataContract, + asset: DataAssetValidationInput, + issues: list[ValidationIssue], + ) -> None: + source_crs = _normalise_crs(asset.source_crs) + storage_crs = _normalise_crs(asset.storage_crs) + if asset.source_crs and source_crs is None: + _issue(issues, "SOURCE_CRS_INVALID", "crs", "source_crs", "Source CRS is not parseable.", observed=asset.source_crs) + if asset.storage_crs and storage_crs is None: + _issue(issues, "STORAGE_CRS_INVALID", "crs", "storage_crs", "Storage CRS is not parseable.", observed=asset.storage_crs) + if contract.require_storage_crs and storage_crs is None: + _issue( + issues, + "STORAGE_CRS_REQUIRED", + "crs", + "storage_crs", + "An explicit storage CRS is required by this contract.", + ) + if contract.accepted_source_crs: + expected = {_normalise_crs(value) for value in contract.accepted_source_crs} + if source_crs is None or source_crs not in expected: + _issue( + issues, + "SOURCE_CRS_NOT_ALLOWED", + "crs", + "source_crs", + "Source CRS is not allowed by this contract.", + expected=sorted(value for value in expected if value), + observed=source_crs or asset.source_crs, + ) + if contract.canonical_storage_crs: + expected_storage_crs = _normalise_crs(contract.canonical_storage_crs) + if storage_crs != expected_storage_crs: + _issue( + issues, + "CANONICAL_STORAGE_CRS_REQUIRED", + "crs", + "storage_crs", + "Stored coordinates must use the contract's canonical CRS.", + expected=expected_storage_crs, + observed=storage_crs or asset.storage_crs, + ) + if source_crs is not None and storage_crs is not None and source_crs != storage_crs and contract.lineage_rules.require_transformation_when_crs_changes: + if not asset.lineage.transformations: + _issue( + issues, + "CRS_TRANSFORMATION_LINEAGE_REQUIRED", + "lineage", + "lineage.transformations", + "A CRS change requires an explicit transformation record.", + expected=f"{source_crs} -> {storage_crs}", + ) + + bounds = _coerce_bounds(asset.bounds, issues) + observed_bounds = _geometry_bounds(asset.geometry_records) + effective_bounds = observed_bounds or bounds + if contract.require_bounds and effective_bounds is None: + _issue(issues, "BOUNDS_REQUIRED", "bounds", "bounds", "Spatial bounds are required by this contract.") + if bounds is not None and not bounds.is_valid(): + _issue(issues, "BOUNDS_INVALID", "bounds", "bounds", "Bounds must be finite and ordered.", observed=bounds.to_dict()) + if bounds is not None and observed_bounds is not None and not bounds.nearly_equals(observed_bounds, tolerance=contract.bounds_tolerance): + _issue( + issues, + "BOUNDS_GEOMETRY_MISMATCH", + "bounds", + "bounds", + "Declared bounds do not match the geometry envelope.", + expected=observed_bounds.to_dict(), + observed=bounds.to_dict(), + ) + if contract.spatial_domain is not None and effective_bounds is not None and effective_bounds.is_valid(): + if not contract.spatial_domain.contains(effective_bounds, tolerance=contract.bounds_tolerance): + _issue( + issues, + "CRS_COORDINATE_DOMAIN_VIOLATION", + "crs", + "bounds", + "Coordinates fall outside the contract's declared storage CRS domain.", + expected=contract.spatial_domain.to_dict(), + observed=effective_bounds.to_dict(), + ) + + @staticmethod + def _check_units(contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue]) -> None: + for field_name, allowed_units in contract.expected_units.items(): + observed = asset.units.get(field_name) + normalised_allowed = {unit.strip().lower() for unit in allowed_units} + if not _nonempty(observed): + _issue( + issues, + "UNIT_REQUIRED", + "units", + f"units.{field_name}", + "A declared unit is required for this field.", + expected=sorted(normalised_allowed), + ) + elif str(observed).strip().lower() not in normalised_allowed: + _issue( + issues, + "UNIT_NOT_ALLOWED", + "units", + f"units.{field_name}", + "Unit is not allowed by this contract; implicit conversion is forbidden.", + expected=sorted(normalised_allowed), + observed=observed, + ) + + @staticmethod + def _check_resolution(contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue]) -> None: + rules = contract.resolution_rules + if rules is None: + return + resolution = _coerce_resolution(asset.resolution, issues) + if resolution is None: + if rules.required: + _issue(issues, "RESOLUTION_REQUIRED", "resolution", "resolution", "Resolution is required by this contract.") + return + if not resolution.is_valid(): + _issue(issues, "RESOLUTION_INVALID", "resolution", "resolution", "Resolution must be finite, positive and unit-labelled.", observed=resolution.to_dict()) + return + allowed_units = {unit.strip().lower() for unit in rules.allowed_units} + if allowed_units and resolution.unit.strip().lower() not in allowed_units: + _issue( + issues, + "RESOLUTION_UNIT_NOT_ALLOWED", + "resolution", + "resolution.unit", + "Resolution unit is not allowed; no implicit conversion is applied.", + expected=sorted(allowed_units), + observed=resolution.unit, + ) + for field_name, value, minimum, maximum in ( + ("x", resolution.x, rules.min_x, rules.max_x), + ("y", resolution.y, rules.min_y, rules.max_y), + ): + if minimum is not None and value < minimum or maximum is not None and value > maximum: + _issue( + issues, + "RESOLUTION_OUT_OF_RANGE", + "resolution", + f"resolution.{field_name}", + "Resolution is outside the contract's permitted range.", + expected={"min": minimum, "max": maximum}, + observed=value, + ) + + @classmethod + def _check_vector(cls, contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue]) -> None: + rules = contract.geometry_rules + if rules is None: + _issue(issues, "VECTOR_RULES_REQUIRED", "schema", "geometry_rules", "Vector contracts require geometry rules.") + return + records = asset.geometry_records + has_records = False + # Topology checks necessarily need the complete geometry set. Default + # GeoJSON contracts do not prohibit overlapping source features, so + # keep large regional import validation streaming unless a stricter + # source-specific contract explicitly asks for that topology rule. + parsed: list[BaseGeometry] | None = [] if rules.forbid_shared_area else None + unique_values: dict[str, dict[Any, int]] = { + field_name: {} for field_name in rules.unique_attribute_fields + } + allowed_types = {value.lower() for value in rules.allowed_geometry_types} + for index, record in enumerate(records): + has_records = True + geometry = _coerce_geometry(record.geometry, index, issues) + if geometry is None: + continue + if geometry.is_empty: + _issue(issues, "GEOMETRY_EMPTY", "geometry", f"geometry_records[{index}]", "Geometry must not be empty.") + continue + if not geometry.is_valid: + _issue( + issues, + "GEOMETRY_INVALID", + "geometry", + f"geometry_records[{index}]", + "Geometry is invalid; this validator never silently repairs geometry.", + ) + continue + if allowed_types and geometry.geom_type.lower() not in allowed_types: + _issue( + issues, + "GEOMETRY_TYPE_NOT_ALLOWED", + "geometry", + f"geometry_records[{index}]", + "Geometry type is not allowed by this contract.", + expected=sorted(rules.allowed_geometry_types), + observed=geometry.geom_type, + ) + cls._check_attributes(rules.attribute_rules, record.properties, index, issues) + cls._check_unique_attribute_values( + rules.unique_attribute_fields, + record.properties, + index, + unique_values, + issues, + ) + if parsed is not None: + parsed.append(geometry) + if rules.require_features and not has_records: + _issue(issues, "VECTOR_FEATURES_REQUIRED", "geometry", "geometry_records", "At least one vector feature is required.") + return + if parsed is not None: + cls._check_shared_area(parsed, rules, issues) + + @staticmethod + def _check_attributes( + rules: tuple[AttributeRule, ...], + properties: Mapping[str, Any], + index: int, + issues: list[ValidationIssue], + ) -> None: + for rule in rules: + present = rule.name in properties + value = properties.get(rule.name) + location = f"geometry_records[{index}].properties.{rule.name}" + if not present and rule.required: + _issue(issues, "ATTRIBUTE_REQUIRED", "attributes", location, "Required attribute is missing.", expected=rule.name) + continue + if not present: + continue + if value is None: + if not rule.nullable: + _issue(issues, "ATTRIBUTE_NULL_NOT_ALLOWED", "attributes", location, "Null is not allowed for this attribute.") + continue + observed_type = _json_value_type(value) + if rule.accepted_types and observed_type not in rule.accepted_types: + _issue( + issues, + "ATTRIBUTE_TYPE_INVALID", + "attributes", + location, + "Attribute type does not match the contract.", + expected=list(rule.accepted_types), + observed=observed_type, + ) + if rule.allowed_values and value not in rule.allowed_values: + _issue( + issues, + "ATTRIBUTE_VALUE_NOT_ALLOWED", + "attributes", + location, + "Attribute value is not in the contract allowlist.", + expected=_sorted_json_values(rule.allowed_values), + observed=value, + ) + + @staticmethod + def _check_shared_area( + geometries: list[BaseGeometry], rules: GeometryRules, issues: list[ValidationIssue]) -> None: + if len(geometries) > rules.topology_max_features: + _issue( + issues, + "TOPOLOGY_CHECK_LIMIT_EXCEEDED", + "topology", + "geometry_records", + "Topology check was not run because the batch exceeds its declared safe limit.", + expected=f"<= {rules.topology_max_features} features", + observed=len(geometries), + ) + return + tree = STRtree(geometries) + for index, geometry in enumerate(geometries): + for candidate_index in tree.query(geometry): + if not isinstance(candidate_index, Integral): + continue + if candidate_index <= index: + continue + candidate = geometries[int(candidate_index)] + if geometry.intersection(candidate).area > 0.0: + _issue( + issues, + "TOPOLOGY_SHARED_AREA", + "topology", + "geometry_records", + "Features share non-zero polygon area where this contract forbids overlap.", + observed={"left_index": index, "right_index": int(candidate_index)}, + ) + return + + @staticmethod + def _check_unique_attribute_values( + field_names: tuple[str, ...], + properties: Mapping[str, Any], + index: int, + values_by_field: dict[str, dict[Any, int]], + issues: list[ValidationIssue], + ) -> None: + for field_name in field_names: + value = properties.get(field_name) + if value is None: + continue + values = values_by_field[field_name] + try: + previous_index = values.get(value) + except TypeError: + _issue( + issues, + "ATTRIBUTE_UNIQUENESS_VALUE_UNHASHABLE", + "attributes", + f"geometry_records[{index}].properties.{field_name}", + "A unique attribute must have a scalar, hashable value.", + observed=value, + ) + continue + if previous_index is not None: + _issue( + issues, + "ATTRIBUTE_UNIQUENESS_VIOLATION", + "attributes", + f"geometry_records[{index}].properties.{field_name}", + "A field declared unique has a duplicate value.", + observed={"value": value, "first_index": previous_index, "duplicate_index": index}, + ) + continue + values[value] = index + + @staticmethod + def _check_raster(contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue]) -> None: + rules = contract.raster_rules + if rules is None: + _issue(issues, "RASTER_RULES_REQUIRED", "schema", "raster_rules", "Raster contracts require raster profile rules.") + return + profile = asset.raster_profile + for field_name in rules.required_profile_fields: + if profile.get(field_name) is None: + _issue( + issues, + "RASTER_PROFILE_FIELD_REQUIRED", + "raster", + f"raster_profile.{field_name}", + "Raster profile field is required.", + ) + for field_name in ("width", "height", "band_count"): + value = profile.get(field_name) + if value is not None and (not isinstance(value, Integral) or isinstance(value, bool) or value <= 0): + _issue( + issues, + "RASTER_PROFILE_VALUE_INVALID", + "raster", + f"raster_profile.{field_name}", + "Raster dimensions and band count must be positive integers.", + observed=value, + ) + band_count = profile.get("band_count") + if rules.allowed_band_counts and isinstance(band_count, Integral) and band_count not in rules.allowed_band_counts: + _issue( + issues, + "RASTER_BAND_COUNT_NOT_ALLOWED", + "raster", + "raster_profile.band_count", + "Raster band count is not allowed by this contract.", + expected=sorted(rules.allowed_band_counts), + observed=band_count, + ) + dtype_values = profile.get("dtype") + dtypes = dtype_values if isinstance(dtype_values, (list, tuple, set)) else [dtype_values] + if rules.allowed_dtypes and any(dtype not in rules.allowed_dtypes for dtype in dtypes if dtype is not None): + _issue( + issues, + "RASTER_DTYPE_NOT_ALLOWED", + "raster", + "raster_profile.dtype", + "Raster dtype is not allowed by this contract.", + expected=sorted(rules.allowed_dtypes), + observed=list(dtypes), + ) + + @staticmethod + def _check_labels(contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue]) -> None: + rules = contract.label_rules + if rules is None: + _issue(issues, "LABEL_RULES_REQUIRED", "schema", "label_rules", "Label contracts require label rules.") + return + label_mode = str(asset.label_mode or "").strip().lower() + declared_mode = str(asset.metadata.get("label_mode") or "").strip().lower() + if declared_mode and declared_mode != label_mode: + _issue( + issues, + "LABEL_MODE_MISMATCH", + "labels", + "metadata.label_mode", + "The persisted label mode must match the validation input.", + expected=label_mode, + observed=declared_mode, + ) + if not asset.label_records: + if not rules.allow_empty_pure_background: + _issue(issues, "LABEL_RECORDS_REQUIRED", "labels", "label_records", "At least one label record is required.") + return + if label_mode != "pure_background" or declared_mode != "pure_background": + _issue( + issues, + "PURE_BACKGROUND_MODE_REQUIRED", + "labels", + "metadata.label_mode", + "An empty YOLO label is valid only as explicitly declared pure_background evidence.", + expected="pure_background", + observed=declared_mode or label_mode or None, + ) + return + DataContractValidator._check_pure_background_label(rules, asset, issues) + return + if label_mode != "objects": + _issue( + issues, + "LABEL_MODE_WITH_OBJECTS_INVALID", + "labels", + "label_mode", + "Non-empty YOLO labels must use the objects label mode.", + expected="objects", + observed=label_mode or None, + ) + for index, record in enumerate(asset.label_records): + for field_name in rules.required_fields: + if field_name not in record: + _issue( + issues, + "LABEL_FIELD_REQUIRED", + "labels", + f"label_records[{index}].{field_name}", + "Label record field is required.", + ) + class_id = record.get("class_id") + if not isinstance(class_id, Integral) or isinstance(class_id, bool): + _issue( + issues, + "LABEL_CLASS_ID_INVALID", + "labels", + f"label_records[{index}].class_id", + "Label class_id must be an integer.", + observed=class_id, + ) + elif rules.allowed_class_ids and class_id not in rules.allowed_class_ids: + _issue( + issues, + "LABEL_CLASS_ID_NOT_ALLOWED", + "labels", + f"label_records[{index}].class_id", + "Label class_id is not present in the contract ontology.", + expected=sorted(rules.allowed_class_ids), + observed=class_id, + ) + values: dict[str, float] = {} + for field_name in ("x_center", "y_center", "width", "height"): + value = record.get(field_name) + if not isinstance(value, Real) or isinstance(value, bool) or not isfinite(float(value)): + _issue( + issues, + "LABEL_COORDINATE_INVALID", + "labels", + f"label_records[{index}].{field_name}", + "Label coordinates must be finite numeric values.", + observed=value, + ) + else: + values[field_name] = float(value) + if len(values) == 4 and rules.normalized_coordinates: + x_center, y_center, width, height = (values[field] for field in ("x_center", "y_center", "width", "height")) + if width <= 0.0 or height <= 0.0 or width > 1.0 or height > 1.0 or not 0.0 <= x_center <= 1.0 or not 0.0 <= y_center <= 1.0: + _issue( + issues, + "LABEL_NORMALIZED_COORDINATE_INVALID", + "labels", + f"label_records[{index}]", + "Normalized labels must have positive dimensions and coordinates within [0, 1].", + observed=values, + ) + elif x_center - width / 2 < 0.0 or x_center + width / 2 > 1.0 or y_center - height / 2 < 0.0 or y_center + height / 2 > 1.0: + _issue( + issues, + "LABEL_BOX_OUTSIDE_IMAGE", + "labels", + f"label_records[{index}]", + "Label bounding box exceeds normalized image bounds.", + observed=values, + ) + + @staticmethod + def _check_pure_background_label( + rules: LabelRules, + asset: DataAssetValidationInput, + issues: list[ValidationIssue], + ) -> None: + """Require explicit provenance and human-review evidence for a zero-object label.""" + + metadata = asset.metadata + for field_name in rules.pure_background_required_metadata_fields: + value = metadata.get(field_name) + if value is None or (isinstance(value, str) and not value.strip()): + _issue( + issues, + "PURE_BACKGROUND_EVIDENCE_REQUIRED", + "labels", + f"metadata.{field_name}", + "Pure-background labels require explicit source, split and review evidence.", + expected=field_name, + observed=value, + ) + + split = str(metadata.get("split") or "").strip().lower() + if rules.allowed_pure_background_splits and split not in rules.allowed_pure_background_splits: + _issue( + issues, + "PURE_BACKGROUND_SPLIT_INVALID", + "labels", + "metadata.split", + "Pure-background label split is not allowed by this contract.", + expected=sorted(rules.allowed_pure_background_splits), + observed=split or None, + ) + if metadata.get("review_decision") != "accepted": + _issue( + issues, + "PURE_BACKGROUND_REVIEW_NOT_ACCEPTED", + "labels", + "metadata.review_decision", + "A zero-object label must have an accepted human review decision.", + expected="accepted", + observed=metadata.get("review_decision"), + ) + if _normalise_checksum(metadata.get("review_artifact_sha256")) is None: + _issue( + issues, + "PURE_BACKGROUND_REVIEW_ARTIFACT_CHECKSUM_INVALID", + "labels", + "metadata.review_artifact_sha256", + "A zero-object label must bind the reviewed artifact checksum.", + observed=metadata.get("review_artifact_sha256"), + ) + reviewed_at = metadata.get("reviewed_at") + if not isinstance(reviewed_at, str) or not reviewed_at.strip(): + _issue( + issues, + "PURE_BACKGROUND_REVIEW_TIMESTAMP_INVALID", + "labels", + "metadata.reviewed_at", + "A zero-object label must record a timezone-aware review timestamp.", + observed=reviewed_at, + ) + else: + try: + timestamp = datetime.fromisoformat(reviewed_at.strip().replace("Z", "+00:00")) + except ValueError: + timestamp = None + if timestamp is None or timestamp.tzinfo is None: + _issue( + issues, + "PURE_BACKGROUND_REVIEW_TIMESTAMP_INVALID", + "labels", + "metadata.reviewed_at", + "A zero-object label must record a timezone-aware review timestamp.", + observed=reviewed_at, + ) + + @staticmethod + def _check_model(contract: DataContract, asset: DataAssetValidationInput, issues: list[ValidationIssue]) -> None: + rules = contract.model_rules + if rules is None: + _issue(issues, "MODEL_RULES_REQUIRED", "schema", "model_rules", "Model contracts require model metadata rules.") + return + metadata = asset.model_metadata + for field_name in rules.required_fields: + value = metadata.get(field_name) + if value is None or (isinstance(value, str) and not value.strip()): + _issue( + issues, + "MODEL_METADATA_FIELD_REQUIRED", + "model", + f"model_metadata.{field_name}", + "Model metadata field is required.", + ) + model_format = metadata.get("model_format") + if rules.allowed_formats and model_format not in rules.allowed_formats: + _issue( + issues, + "MODEL_FORMAT_NOT_ALLOWED", + "model", + "model_metadata.model_format", + "Model format is not allowed by this contract.", + expected=sorted(rules.allowed_formats), + observed=model_format, + ) + class_mapping = metadata.get("class_mapping") + if rules.minimum_class_count is not None: + class_count = len(class_mapping) if isinstance(class_mapping, (Mapping, list, tuple)) else 0 + if class_count < rules.minimum_class_count: + _issue( + issues, + "MODEL_CLASS_MAPPING_INCOMPLETE", + "model", + "model_metadata.class_mapping", + "Model class mapping does not meet the minimum ontology size.", + expected=rules.minimum_class_count, + observed=class_count, + ) + + +def _failed_unknown_contract_report( + asset: DataAssetValidationInput, + issue: ValidationIssue, + *, + now: datetime | None, +) -> ValidationReport: + checked_at = _as_utc(now) or datetime.now(timezone.utc) + return ValidationReport( + asset_id=asset.asset_id, + data_contract_key=asset.data_contract_key, + data_contract_version=asset.data_contract_version, + contract_fingerprint_sha256=None, + validation_status=ValidationStatus.FAILED, + provenance_status=ProvenanceStatus.INCOMPLETE, + lineage_status=LineageStatus.INCOMPLETE, + quarantine_status=QuarantineStatus.QUARANTINED, + validation_scope=("contract",), + checked_at=checked_at, + issues=(issue,), + ) + + +def _issue( + issues: list[ValidationIssue], + code: str, + category: str, + field: str | None, + message: str, + *, + expected: Any = None, + observed: Any = None, + severity: IssueSeverity = IssueSeverity.ERROR, +) -> None: + issues.append( + ValidationIssue( + code=code, + category=category, + field=field, + message=message, + expected=expected, + observed=observed, + severity=severity, + ) + ) + + +def _normalise_crs(value: str | None) -> str | None: + if not _nonempty(value): + return None + try: + crs = CRS.from_user_input(value) + except Exception: + return None + authority = crs.to_authority() + if authority: + return f"{authority[0].upper()}:{authority[1]}" + return crs.to_string() + + +def _coerce_bounds(value: Any, issues: list[ValidationIssue]) -> BoundingBox | None: + if value is None: + return None + try: + return BoundingBox.from_value(value) + except ValueError as exc: + _issue(issues, "BOUNDS_FORMAT_INVALID", "bounds", "bounds", str(exc), observed=value) + return None + + +def _coerce_resolution(value: Any, issues: list[ValidationIssue]) -> Resolution | None: + if value is None: + return None + try: + return Resolution.from_value(value) + except ValueError as exc: + _issue(issues, "RESOLUTION_FORMAT_INVALID", "resolution", "resolution", str(exc), observed=value) + return None + + +def _coerce_geometry(value: BaseGeometry | Mapping[str, Any], index: int, issues: list[ValidationIssue]) -> BaseGeometry | None: + if isinstance(value, BaseGeometry): + return value + try: + return shape(value) + except Exception: + _issue( + issues, + "GEOMETRY_PARSE_FAILED", + "geometry", + f"geometry_records[{index}]", + "Geometry cannot be parsed as GeoJSON/Shapely geometry.", + ) + return None + + +def _geometry_bounds(records: Iterable[GeometryRecord]) -> BoundingBox | None: + min_x = min_y = max_x = max_y = None + for record in records: + if isinstance(record.geometry, BaseGeometry): + geometry = record.geometry + else: + try: + geometry = shape(record.geometry) + except Exception: + continue + if not geometry.is_empty: + record_min_x, record_min_y, record_max_x, record_max_y = ( + float(value) for value in geometry.bounds + ) + min_x = record_min_x if min_x is None else min(min_x, record_min_x) + min_y = record_min_y if min_y is None else min(min_y, record_min_y) + max_x = record_max_x if max_x is None else max(max_x, record_max_x) + max_y = record_max_y if max_y is None else max(max_y, record_max_y) + if min_x is None or min_y is None or max_x is None or max_y is None: + return None + return BoundingBox(min_x, min_y, max_x, max_y) + + +def _normalise_checksum(value: str | None) -> str | None: + if not _nonempty(value): + return None + normalised = str(value).strip().lower() + return normalised if _SHA256_RE.fullmatch(normalised) else None + + +def _as_utc(value: datetime | None) -> datetime | None: + if value is None or value.tzinfo is None: + return None + return value.astimezone(timezone.utc) + + +def _nonempty(value: Any) -> bool: + return value is not None and (not isinstance(value, str) or bool(value.strip())) + + +def _json_value_type(value: Any) -> str: + if value is None: + return "null" + if isinstance(value, bool): + return "boolean" + if isinstance(value, Integral): + return "integer" + if isinstance(value, Real): + return "number" + if isinstance(value, str): + return "string" + if isinstance(value, Mapping): + return "object" + if isinstance(value, (list, tuple)): + return "array" + return type(value).__name__ + + +def _contract_identity(key: str, version: str) -> tuple[str, str]: + normalized_key = key.strip() + normalized_version = version.strip() + if not normalized_key or not normalized_version: + raise ValueError("Data contract key and version must be non-empty") + return normalized_key, normalized_version + + +def _stable_sha256(payload: Mapping[str, Any]) -> str: + encoded = json.dumps(_json_safe(payload), sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8") + return sha256(encoded).hexdigest() + + +def _json_safe(value: Any) -> Any: + if isinstance(value, StrEnum): + return value.value + if isinstance(value, datetime): + return _datetime_payload(value) + if isinstance(value, BoundingBox): + return value.to_dict() + if isinstance(value, Resolution): + return value.to_dict() + if isinstance(value, Mapping): + return {str(key): _json_safe(item) for key, item in value.items()} + if isinstance(value, (list, tuple, set, frozenset)): + return [_json_safe(item) for item in value] + return value + + +def _datetime_payload(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat() + + +def _geometry_rules_payload(value: GeometryRules | None) -> dict[str, Any] | None: + if value is None: + return None + return { + "allowed_geometry_types": sorted(value.allowed_geometry_types), + "attribute_rules": [ + { + "name": rule.name, + "required": rule.required, + "nullable": rule.nullable, + "accepted_types": list(rule.accepted_types), + "allowed_values": _sorted_json_values(rule.allowed_values), + } + for rule in value.attribute_rules + ], + "unique_attribute_fields": list(value.unique_attribute_fields), + "require_features": value.require_features, + "forbid_shared_area": value.forbid_shared_area, + "topology_max_features": value.topology_max_features, + } + + +def _raster_rules_payload(value: RasterRules | None) -> dict[str, Any] | None: + if value is None: + return None + return { + "required_profile_fields": list(value.required_profile_fields), + "allowed_band_counts": sorted(value.allowed_band_counts), + "allowed_dtypes": sorted(value.allowed_dtypes), + } + + +def _label_rules_payload(value: LabelRules | None) -> dict[str, Any] | None: + if value is None: + return None + payload: dict[str, Any] = { + "allowed_class_ids": sorted(value.allowed_class_ids), + "normalized_coordinates": value.normalized_coordinates, + "required_fields": list(value.required_fields), + } + # Keep the historical v1.0.0 fingerprint stable. Pure-background support + # is introduced by a new exact contract version rather than silently + # widening the meaning of an already frozen label contract. + if value.allow_empty_pure_background: + payload.update( + { + "allow_empty_pure_background": True, + "pure_background_required_metadata_fields": list(value.pure_background_required_metadata_fields), + "allowed_pure_background_splits": sorted(value.allowed_pure_background_splits), + } + ) + return payload + + +def _model_rules_payload(value: ModelRules | None) -> dict[str, Any] | None: + if value is None: + return None + return { + "required_fields": list(value.required_fields), + "allowed_formats": sorted(value.allowed_formats), + "minimum_class_count": value.minimum_class_count, + } + + +def _resolution_rules_payload(value: ResolutionRules | None) -> dict[str, Any] | None: + if value is None: + return None + return { + "required": value.required, + "allowed_units": sorted(value.allowed_units), + "min_x": value.min_x, + "max_x": value.max_x, + "min_y": value.min_y, + "max_y": value.max_y, + } + + +def _freshness_rules_payload(value: FreshnessRules) -> dict[str, Any]: + return { + "observed_at": value.observed_at.value, + "source_version": value.source_version.value, + "imported_at_required": value.imported_at_required, + "max_age_seconds": value.max_age.total_seconds() if value.max_age else None, + "allow_future_observation": value.allow_future_observation, + } + + +def _lineage_rules_payload(value: LineageRules) -> dict[str, Any]: + return { + "require_source_registry": value.require_source_registry, + "require_source_snapshot": value.require_source_snapshot, + "require_upstream_assets": value.require_upstream_assets, + "require_transformation_when_crs_changes": value.require_transformation_when_crs_changes, + } + + +def _sorted_json_values(values: Iterable[Any]) -> list[Any]: + serialised = [_json_safe(value) for value in values] + return sorted(serialised, key=lambda value: json.dumps(value, sort_keys=True, ensure_ascii=True)) + + +def _provenance_status(contract: DataContract, issues: list[ValidationIssue]) -> ProvenanceStatus: + categories = {"provenance", "checksum", "temporal", "freshness", "crs", "bounds", "units", "resolution", "metadata"} + if any(issue.category in categories for issue in issues): + return ProvenanceStatus.INCOMPLETE + if not contract.lineage_rules.require_source_registry and not contract.lineage_rules.require_source_snapshot: + return ProvenanceStatus.NOT_APPLICABLE + return ProvenanceStatus.COMPLETE + + +def _lineage_status(contract: DataContract, issues: list[ValidationIssue]) -> LineageStatus: + if any(issue.category == "lineage" for issue in issues): + return LineageStatus.INCOMPLETE + if not contract.lineage_rules.require_upstream_assets and not contract.lineage_rules.require_transformation_when_crs_changes: + return LineageStatus.NOT_APPLICABLE + return LineageStatus.COMPLETE + + +def _validation_scope(contract: DataContract) -> tuple[str, ...]: + checks = ["contract", "checksum", "metadata", "temporal", "provenance", "lineage", "crs", "bounds", "units"] + if contract.resolution_rules is not None: + checks.append("resolution") + checks.append(contract.kind.value) + return tuple(checks) + + +# The generic contracts below are deliberately narrow in evidence requirements +# but broad in legitimate Belgian source CRSs. A source-specific registry may +# register an additional, stricter version; ingestion must always choose an +# explicit key/version and may never silently choose a "latest" contract. +VECTOR_GEOJSON_CONTRACT_KEY = "geointel.vector.geojson" +VECTOR_GEOJSON_CONTRACT_VERSION = "1.0.0" +RASTER_GEOTIFF_CONTRACT_KEY = "geointel.raster.geotiff" +RASTER_GEOTIFF_CONTRACT_VERSION = "1.0.0" +YOLO_LABEL_CONTRACT_KEY = "geointel.label.yolo" +YOLO_LABEL_LEGACY_CONTRACT_VERSION = "1.0.0" +YOLO_LABEL_CONTRACT_VERSION = "1.1.0" +PYTORCH_MODEL_CONTRACT_KEY = "geointel.model.pytorch" +PYTORCH_MODEL_CONTRACT_VERSION = "1.0.0" + +_BELGIUM_AND_NORTH_SEA_WGS84_DOMAIN = BoundingBox(min_x=1.5, min_y=48.5, max_x=7.5, max_y=52.5) +_BELGIAN_SOURCE_CRS = frozenset({"EPSG:4326", "EPSG:31370", "EPSG:3812"}) + + +def build_default_data_contract_registry() -> DataContractRegistry: + """Build the concrete exact-version registry used by generic ingestion. + + The defaults are not a trust registry. They validate a safely staged + artifact only after the caller supplies server-attested source registry and + snapshot identities. GRB/PICC/UrbIS and source-specific semantic rules are + intentionally supplied by stricter source-registry contracts. + """ + + vector_contract = DataContract( + key=VECTOR_GEOJSON_CONTRACT_KEY, + version=VECTOR_GEOJSON_CONTRACT_VERSION, + kind=ContractKind.VECTOR, + accepted_source_crs=_BELGIAN_SOURCE_CRS, + canonical_storage_crs="EPSG:4326", + spatial_domain=_BELGIUM_AND_NORTH_SEA_WGS84_DOMAIN, + require_bounds=True, + required_metadata_fields=("license",), + geometry_rules=GeometryRules(require_features=True), + freshness_rules=FreshnessRules( + observed_at=RequirementLevel.UNKNOWN_WITH_REASON, + source_version=RequirementLevel.UNKNOWN_WITH_REASON, + ), + ) + raster_contract = DataContract( + key=RASTER_GEOTIFF_CONTRACT_KEY, + version=RASTER_GEOTIFF_CONTRACT_VERSION, + kind=ContractKind.RASTER, + accepted_source_crs=_BELGIAN_SOURCE_CRS, + require_bounds=True, + required_metadata_fields=("license",), + raster_rules=RasterRules(), + resolution_rules=ResolutionRules( + allowed_units=frozenset({"m", "degree"}), + min_x=0.000001, + max_x=10_000.0, + min_y=0.000001, + max_y=10_000.0, + ), + freshness_rules=FreshnessRules( + observed_at=RequirementLevel.UNKNOWN_WITH_REASON, + source_version=RequirementLevel.UNKNOWN_WITH_REASON, + ), + lineage_rules=LineageRules(require_transformation_when_crs_changes=False), + ) + legacy_label_contract = DataContract( + key=YOLO_LABEL_CONTRACT_KEY, + version=YOLO_LABEL_LEGACY_CONTRACT_VERSION, + kind=ContractKind.LABEL, + require_storage_crs=False, + required_metadata_fields=("image_checksum_sha256", "class_ontology_version", "tile_manifest_sha256"), + metadata_checksum_fields=("image_checksum_sha256", "tile_manifest_sha256"), + label_rules=LabelRules(allowed_class_ids=frozenset({0})), + freshness_rules=FreshnessRules( + observed_at=RequirementLevel.UNKNOWN_WITH_REASON, + source_version=RequirementLevel.UNKNOWN_WITH_REASON, + ), + lineage_rules=LineageRules( + require_source_registry=True, + require_source_snapshot=True, + require_upstream_assets=True, + require_transformation_when_crs_changes=False, + ), + ) + label_contract = DataContract( + key=YOLO_LABEL_CONTRACT_KEY, + version=YOLO_LABEL_CONTRACT_VERSION, + kind=ContractKind.LABEL, + require_storage_crs=False, + required_metadata_fields=( + "image_checksum_sha256", + "class_ontology_version", + "source_corpus_manifest_sha256", + "label_mode", + ), + metadata_checksum_fields=("image_checksum_sha256", "source_corpus_manifest_sha256"), + label_rules=LabelRules( + allowed_class_ids=frozenset({0}), + allow_empty_pure_background=True, + pure_background_required_metadata_fields=( + "sample_slug", + "split", + "raster_dataset_id", + "reference_dataset_id", + "review_decision", + "reviewer_id", + "reviewed_at", + "review_artifact_sha256", + ), + allowed_pure_background_splits=frozenset({"train", "val"}), + ), + freshness_rules=FreshnessRules( + observed_at=RequirementLevel.UNKNOWN_WITH_REASON, + source_version=RequirementLevel.UNKNOWN_WITH_REASON, + ), + lineage_rules=LineageRules( + require_source_registry=True, + require_source_snapshot=True, + require_upstream_assets=True, + require_transformation_when_crs_changes=False, + ), + ) + model_contract = DataContract( + key=PYTORCH_MODEL_CONTRACT_KEY, + version=PYTORCH_MODEL_CONTRACT_VERSION, + kind=ContractKind.MODEL, + require_storage_crs=False, + required_metadata_fields=("training_manifest_sha256", "runtime_manifest_sha256"), + metadata_checksum_fields=("training_manifest_sha256", "runtime_manifest_sha256"), + model_rules=ModelRules(allowed_formats=frozenset({"pytorch", "ultralytics"}), minimum_class_count=1), + freshness_rules=FreshnessRules( + observed_at=RequirementLevel.NOT_APPLICABLE, + source_version=RequirementLevel.REQUIRED, + ), + lineage_rules=LineageRules( + require_source_registry=True, + require_source_snapshot=True, + require_upstream_assets=True, + require_transformation_when_crs_changes=False, + ), + ) + return DataContractRegistry((vector_contract, raster_contract, legacy_label_contract, label_contract, model_contract)) + + +def validate_registered_asset( + asset: DataAssetValidationInput, + *, + registry: DataContractRegistry | None = None, + now: datetime | None = None, +) -> ValidationReport: + """Validate an explicitly versioned asset against a supplied/default registry.""" + + active_registry = registry or build_default_data_contract_registry() + return active_registry.validate(asset, now=now) + + +def build_vector_ingest_input( + *, + asset_id: str, + source_crs: str | None, + storage_crs: str | None, + feature_collection: Mapping[str, Any], + checksum_sha256: str | None, + computed_checksum_sha256: str | None, + source_registry_id: str | None, + source_snapshot_id: str | None, + imported_at: datetime | None, + metadata: Mapping[str, Any] | None = None, + content: bytes | None = None, + units: Mapping[str, str] | None = None, + observed_at: datetime | None = None, + valid_from: datetime | None = None, + valid_to: datetime | None = None, + temporal_unknown_reason: str | None = None, + source_version: str | None = None, + source_version_unknown_reason: str | None = None, + lineage: LineageEvidence | None = None, + data_contract_key: str = VECTOR_GEOJSON_CONTRACT_KEY, + data_contract_version: str = VECTOR_GEOJSON_CONTRACT_VERSION, +) -> DataAssetValidationInput: + """Adapt a GeoJSON FeatureCollection to the generic vector contract input.""" + + raw_features = feature_collection.get("features") + records = () + if isinstance(raw_features, list): + records = tuple( + GeometryRecord( + geometry=feature.get("geometry", {}), + properties=feature.get("properties") if isinstance(feature.get("properties"), Mapping) else {}, + identifier=str(feature.get("id")) if feature.get("id") is not None else None, + ) + for feature in raw_features + if isinstance(feature, Mapping) + ) + merged_metadata = dict(metadata or {}) + bounds = merged_metadata.get("bounds_json", merged_metadata.get("bounds")) + return DataAssetValidationInput( + asset_id=asset_id, + data_contract_key=data_contract_key, + data_contract_version=data_contract_version, + kind=ContractKind.VECTOR, + source_crs=source_crs, + storage_crs=storage_crs, + bounds=bounds, + checksum_sha256=checksum_sha256, + computed_checksum_sha256=computed_checksum_sha256, + content=content, + metadata=merged_metadata, + units=units or {}, + geometry_records=records, + source_registry_id=source_registry_id, + source_snapshot_id=source_snapshot_id, + lineage=lineage or LineageEvidence(), + imported_at=imported_at, + observed_at=observed_at, + valid_from=valid_from, + valid_to=valid_to, + temporal_unknown_reason=temporal_unknown_reason, + source_version=source_version, + source_version_unknown_reason=source_version_unknown_reason, + ) + + +def build_raster_ingest_input( + *, + asset_id: str, + source_crs: str | None, + storage_crs: str | None, + raster_profile: Mapping[str, Any], + bounds: BoundingBox | Mapping[str, Any] | Sequence[float] | None, + resolution: Resolution | Mapping[str, Any] | Sequence[Any] | None, + checksum_sha256: str | None, + computed_checksum_sha256: str | None, + source_registry_id: str | None, + source_snapshot_id: str | None, + imported_at: datetime | None, + metadata: Mapping[str, Any] | None = None, + content: bytes | None = None, + units: Mapping[str, str] | None = None, + observed_at: datetime | None = None, + valid_from: datetime | None = None, + valid_to: datetime | None = None, + temporal_unknown_reason: str | None = None, + source_version: str | None = None, + source_version_unknown_reason: str | None = None, + lineage: LineageEvidence | None = None, + data_contract_key: str = RASTER_GEOTIFF_CONTRACT_KEY, + data_contract_version: str = RASTER_GEOTIFF_CONTRACT_VERSION, +) -> DataAssetValidationInput: + """Adapt extracted GeoTIFF metadata to the generic raster contract input.""" + + return DataAssetValidationInput( + asset_id=asset_id, + data_contract_key=data_contract_key, + data_contract_version=data_contract_version, + kind=ContractKind.RASTER, + source_crs=source_crs, + storage_crs=storage_crs, + bounds=bounds, + checksum_sha256=checksum_sha256, + computed_checksum_sha256=computed_checksum_sha256, + content=content, + metadata=dict(metadata or {}), + units=units or {}, + resolution=resolution, + raster_profile=dict(raster_profile), + source_registry_id=source_registry_id, + source_snapshot_id=source_snapshot_id, + lineage=lineage or LineageEvidence(), + imported_at=imported_at, + observed_at=observed_at, + valid_from=valid_from, + valid_to=valid_to, + temporal_unknown_reason=temporal_unknown_reason, + source_version=source_version, + source_version_unknown_reason=source_version_unknown_reason, + ) + + +def build_label_validation_input( + *, + asset_id: str, + label_records: Sequence[Mapping[str, Any]], + checksum_sha256: str | None, + computed_checksum_sha256: str | None, + source_registry_id: str | None, + source_snapshot_id: str | None, + imported_at: datetime | None, + metadata: Mapping[str, Any] | None = None, + content: bytes | None = None, + label_mode: str = "objects", + observed_at: datetime | None = None, + temporal_unknown_reason: str | None = None, + source_version: str | None = None, + source_version_unknown_reason: str | None = None, + lineage: LineageEvidence | None = None, + data_contract_key: str = YOLO_LABEL_CONTRACT_KEY, + data_contract_version: str = YOLO_LABEL_CONTRACT_VERSION, +) -> DataAssetValidationInput: + """Build a strict YOLO-label validation input with explicit upstream lineage.""" + + normalized_metadata = dict(metadata or {}) + normalized_metadata.setdefault("label_mode", label_mode) + + return DataAssetValidationInput( + asset_id=asset_id, + data_contract_key=data_contract_key, + data_contract_version=data_contract_version, + kind=ContractKind.LABEL, + checksum_sha256=checksum_sha256, + computed_checksum_sha256=computed_checksum_sha256, + content=content, + metadata=normalized_metadata, + label_records=tuple(label_records), + label_mode=label_mode, + source_registry_id=source_registry_id, + source_snapshot_id=source_snapshot_id, + lineage=lineage or LineageEvidence(), + imported_at=imported_at, + observed_at=observed_at, + temporal_unknown_reason=temporal_unknown_reason, + source_version=source_version, + source_version_unknown_reason=source_version_unknown_reason, + ) + + +def build_model_validation_input( + *, + asset_id: str, + model_metadata: Mapping[str, Any], + checksum_sha256: str | None, + computed_checksum_sha256: str | None, + source_registry_id: str | None, + source_snapshot_id: str | None, + imported_at: datetime | None, + metadata: Mapping[str, Any] | None = None, + content: bytes | None = None, + source_version: str | None = None, + lineage: LineageEvidence | None = None, + data_contract_key: str = PYTORCH_MODEL_CONTRACT_KEY, + data_contract_version: str = PYTORCH_MODEL_CONTRACT_VERSION, +) -> DataAssetValidationInput: + """Build a model-asset validation input; model output is never inferred.""" + + return DataAssetValidationInput( + asset_id=asset_id, + data_contract_key=data_contract_key, + data_contract_version=data_contract_version, + kind=ContractKind.MODEL, + checksum_sha256=checksum_sha256, + computed_checksum_sha256=computed_checksum_sha256, + content=content, + metadata=dict(metadata or {}), + model_metadata=dict(model_metadata), + source_registry_id=source_registry_id, + source_snapshot_id=source_snapshot_id, + lineage=lineage or LineageEvidence(), + imported_at=imported_at, + source_version=source_version, + ) diff --git a/backend/app/services/data_quarantine_service.py b/backend/app/services/data_quarantine_service.py new file mode 100644 index 00000000..24cd4600 --- /dev/null +++ b/backend/app/services/data_quarantine_service.py @@ -0,0 +1,160 @@ +"""Fail-closed quarantine decisions for validated data assets. + +Persistence is intentionally delegated to the caller's transaction. This +module derives stable decisions from immutable validation reports and blocks a +quarantined or failed asset from training, production inference and derived +processing until an explicit, separately persisted release action exists. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum +from hashlib import sha256 +from typing import Any +import json + +from app.core.errors import AppError +from app.services.data_contract_validation import QuarantineStatus, ValidationReport, ValidationStatus + + +class AssetUse(StrEnum): + TRAINING = "training" + PRODUCTION_INFERENCE = "production_inference" + DERIVED_PROCESSING = "derived_processing" + EXPORT = "export" + + +@dataclass(frozen=True) +class QuarantineDecision: + """A deterministic, auditable quarantine decision for one report.""" + + asset_id: str + data_contract_key: str + data_contract_version: str + validation_report_sha256: str + validation_status: ValidationStatus + quarantine_status: QuarantineStatus + reason_codes: tuple[str, ...] + idempotency_key: str + requires_explicit_release: bool = False + + @property + def eligible_for_use(self) -> bool: + return self.validation_status == ValidationStatus.PASSED and self.quarantine_status == QuarantineStatus.NOT_QUARANTINED + + def to_dict(self) -> dict[str, Any]: + return { + "asset_id": self.asset_id, + "data_contract_key": self.data_contract_key, + "data_contract_version": self.data_contract_version, + "validation_report_sha256": self.validation_report_sha256, + "validation_status": self.validation_status.value, + "quarantine_status": self.quarantine_status.value, + "reason_codes": list(self.reason_codes), + "idempotency_key": self.idempotency_key, + "requires_explicit_release": self.requires_explicit_release, + "eligible_for_use": self.eligible_for_use, + } + + +class DataQuarantineService: + """Derive and enforce safe use decisions from validation results.""" + + @staticmethod + def decide( + report: ValidationReport, + *, + previous: QuarantineDecision | None = None, + ) -> QuarantineDecision: + """Create a stable decision without silently releasing old quarantines. + + A fresh passing validation report can be persisted as a new validated + version by the import transaction. It cannot automatically release an + existing quarantined record: the caller must explicitly record that + reviewed state transition against the new report/version. + """ + + is_quarantined = report.validation_status == ValidationStatus.FAILED or report.quarantine_status == QuarantineStatus.QUARANTINED + failure_codes = tuple( + sorted( + { + issue.code + for issue in report.issues + if issue.severity.value == "error" or is_quarantined + } + ) + ) + requires_explicit_release = False + reason_codes = failure_codes + status = QuarantineStatus.QUARANTINED if is_quarantined else QuarantineStatus.NOT_QUARANTINED + + if previous is not None and previous.quarantine_status == QuarantineStatus.QUARANTINED and not is_quarantined: + status = QuarantineStatus.QUARANTINED + requires_explicit_release = True + reason_codes = ("QUARANTINE_RELEASE_REQUIRES_EXPLICIT_PERSISTENCE",) + + idempotency_key = _decision_key( + asset_id=report.asset_id, + contract_key=report.data_contract_key, + contract_version=report.data_contract_version, + report_sha256=report.report_sha256, + quarantine_status=status, + reason_codes=reason_codes, + requires_explicit_release=requires_explicit_release, + ) + return QuarantineDecision( + asset_id=report.asset_id, + data_contract_key=report.data_contract_key, + data_contract_version=report.data_contract_version, + validation_report_sha256=report.report_sha256, + validation_status=report.validation_status, + quarantine_status=status, + reason_codes=reason_codes, + idempotency_key=idempotency_key, + requires_explicit_release=requires_explicit_release, + ) + + @staticmethod + def require_eligible(decision: QuarantineDecision, *, use: AssetUse) -> None: + """Raise a typed error before a non-eligible artifact reaches a pipeline.""" + + if decision.eligible_for_use: + return + raise AppError( + code="DATASET_QUARANTINED", + message="Dataset is quarantined or failed validation and cannot enter this pipeline.", + status_code=409, + details={ + "asset_id": decision.asset_id, + "use": use.value, + "quarantine_status": decision.quarantine_status.value, + "validation_status": decision.validation_status.value, + "reason_codes": list(decision.reason_codes), + "validation_report_sha256": decision.validation_report_sha256, + "idempotency_key": decision.idempotency_key, + "requires_explicit_release": decision.requires_explicit_release, + }, + ) + + +def _decision_key( + *, + asset_id: str, + contract_key: str, + contract_version: str, + report_sha256: str, + quarantine_status: QuarantineStatus, + reason_codes: tuple[str, ...], + requires_explicit_release: bool, +) -> str: + payload = { + "asset_id": asset_id, + "contract_key": contract_key, + "contract_version": contract_version, + "report_sha256": report_sha256, + "quarantine_status": quarantine_status.value, + "reason_codes": list(reason_codes), + "requires_explicit_release": requires_explicit_release, + } + return sha256(json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")).hexdigest() diff --git a/backend/app/services/dataset_consumption_gate_service.py b/backend/app/services/dataset_consumption_gate_service.py new file mode 100644 index 00000000..6d94333e --- /dev/null +++ b/backend/app/services/dataset_consumption_gate_service.py @@ -0,0 +1,440 @@ +"""Fail-closed provenance gates at data-consumption boundaries. + +Import validation protects newly staged assets, but a persisted record can +subsequently become quarantined or have its provenance marked incomplete. The +callers of this service therefore re-check the durable Dataset state directly +before production inference, QA, derived processing, export, or authoritative +coverage reporting. + +The only legacy relaxation is deliberately narrow: an *explicitly tagged* +fixture with no Phase-2 state can be used for fixture QA. A caller-provided +``fixture_mode`` flag alone never creates that trust claim. Fixture data can +never become a production inference, derived-processing, export or +authoritative-coverage input, and it never relaxes a recorded failed, +incomplete, or quarantined state. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +import re +from typing import Any, Literal + +from sqlalchemy import inspect as sa_inspect + +from app.core.errors import AppError +from app.models import Dataset + + +DatasetConsumptionPurpose = Literal[ + "production_inference", + "quality_assessment", + "reference_validation", + "derived_processing", + "authoritative_coverage", + "export", +] + +_FIXTURE_SOURCE_KEYS = {"fixture", "test", "test_fixture", "test-fixture", "unit-test-fixture"} +_UNTRUSTED_SOURCE_KEYS = {"manual", "fixture", "experimental", "legacy_unknown"} +_CHECKSUM_SHA256 = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE) +_CONSUMABLE_SNAPSHOT_FRESHNESS = {"current", "not_applicable"} +_VALID_PURPOSES = { + "production_inference", + "quality_assessment", + "reference_validation", + "derived_processing", + "authoritative_coverage", + "export", +} + + +@dataclass(frozen=True) +class DatasetConsumptionDecision: + """Auditable decision returned by the consumption gate.""" + + eligible: bool + purpose: DatasetConsumptionPurpose + fixture_legacy_exception: bool + reasons: tuple[str, ...] + evidence: dict[str, Any] + + +class DatasetConsumptionGate: + """Evaluate durable provenance before a Dataset is consumed downstream.""" + + @staticmethod + def _value(dataset: Any, field: str, default: Any = None) -> Any: + if isinstance(dataset, Mapping): + return dataset.get(field, default) + return getattr(dataset, field, default) + + @staticmethod + def _mapping(value: Any) -> dict[str, Any]: + return dict(value) if isinstance(value, Mapping) else {} + + @classmethod + def _normalise(cls, value: Any) -> str: + return str(value or "").strip().lower() + + @classmethod + def is_explicit_fixture(cls, dataset: Any) -> bool: + source = cls._normalise(cls._value(dataset, "source")) + source_name = cls._normalise(cls._value(dataset, "source_name")) + metadata = cls._mapping(cls._value(dataset, "metadata_json")) + source_metadata = cls._mapping(cls._value(dataset, "source_metadata")) + provenance = cls._mapping(cls._value(dataset, "provenance_metadata")) + return bool( + source in _FIXTURE_SOURCE_KEYS + or source_name in _FIXTURE_SOURCE_KEYS + or metadata.get("fixture") is True + or metadata.get("fixture_mode") is True + or source_metadata.get("fixture") is True + or source_metadata.get("fixture_mode") is True + or provenance.get("fixture") is True + or provenance.get("fixture_mode") is True + ) + + @classmethod + def _phase2_state_is_absent(cls, dataset: Any) -> bool: + fields = ( + "data_contract_key", + "data_contract_version", + "validation_status", + "provenance_status", + "lineage_status", + "quarantine_status", + "source_registry_id", + "source_snapshot_id", + ) + return all(cls._value(dataset, field) in {None, ""} for field in fields) + + @staticmethod + def _is_transient_orm_dataset(dataset: Any) -> bool: + """Recognize only unpersisted ORM fixtures, never database rows.""" + + if not isinstance(dataset, Dataset): + return False + try: + return bool(sa_inspect(dataset).transient) + except Exception: # pragma: no cover - defensive for unusual test doubles + return False + + @classmethod + def _evidence( + cls, + dataset: Any, + purpose: DatasetConsumptionPurpose, + reference_task: str | None = None, + ) -> dict[str, Any]: + source_registry = cls._value(dataset, "source_registry") + source_snapshot = cls._value(dataset, "source_snapshot") + source_policy = cls._mapping(cls._value(source_registry, "usage_policy_json")) + validation_authority = cls._mapping(source_policy.get("validation_authority")) + reference_approvals = cls._mapping(source_policy.get("reference_validation_approvals")) + return { + "dataset_id": str(cls._value(dataset, "id") or ""), + "purpose": purpose, + "dataset_status": cls._normalise(cls._value(dataset, "status")), + "source": cls._normalise(cls._value(dataset, "source")), + "source_name": cls._normalise(cls._value(dataset, "source_name")), + "data_contract_key": cls._value(dataset, "data_contract_key"), + "data_contract_version": cls._value(dataset, "data_contract_version"), + "validation_status": cls._normalise(cls._value(dataset, "validation_status")), + "provenance_status": cls._normalise(cls._value(dataset, "provenance_status")), + "lineage_status": cls._normalise(cls._value(dataset, "lineage_status")), + "quarantine_status": cls._normalise(cls._value(dataset, "quarantine_status")), + "checksum_sha256": cls._value(dataset, "checksum_sha256"), + "source_registry_id": str(cls._value(dataset, "source_registry_id") or ""), + "source_snapshot_id": str(cls._value(dataset, "source_snapshot_id") or ""), + "source_classification": cls._normalise(cls._value(source_registry, "classification")), + "source_key": cls._normalise(cls._value(source_registry, "source_key")), + "source_ground_truth_allowed": source_policy.get("ground_truth_allowed") is True, + "source_validation_authority": validation_authority, + "source_reference_validation_approvals": reference_approvals, + "source_authority_scope": cls._mapping(cls._value(source_registry, "authority_scope_json")), + "reference_task": cls._normalise(reference_task), + "snapshot_source_registry_id": str(cls._value(source_snapshot, "source_registry_id") or ""), + "snapshot_ingest_status": cls._normalise(cls._value(source_snapshot, "ingest_status")), + "snapshot_freshness_status": cls._normalise(cls._value(source_snapshot, "freshness_status")), + "snapshot_checksum_sha256": cls._value(source_snapshot, "checksum_sha256"), + } + + @classmethod + def evaluate( + cls, + dataset: Any, + *, + purpose: DatasetConsumptionPurpose, + fixture_mode: bool = False, + reference_task: str | None = None, + ) -> DatasetConsumptionDecision: + """Return a stable decision without mutating the Dataset. + + ``fixture_mode`` can be supplied only by an explicitly fixture-only + caller. It is evidence for a QA fixture path, never a relaxation for + a production boundary or an explicit unsafe state. + """ + + if purpose not in _VALID_PURPOSES: + raise ValueError(f"Unsupported dataset-consumption purpose: {purpose}") + + evidence = cls._evidence(dataset, purpose, reference_task) + reasons: list[str] = [] + explicit_fixture = cls.is_explicit_fixture(dataset) + phase2_absent = cls._phase2_state_is_absent(dataset) + + # These are irrevocable safety states. They are checked before a + # fixture exception, so fixture rows cannot hide a bad recorded state. + if evidence["dataset_status"] in {"failed", "quarantined"}: + reasons.append("dataset_status_unsafe") + if evidence["quarantine_status"] == "quarantined": + reasons.append("dataset_quarantined") + if evidence["validation_status"] == "failed": + reasons.append("validation_failed") + if evidence["provenance_status"] == "incomplete": + reasons.append("provenance_incomplete") + if evidence["lineage_status"] == "incomplete": + reasons.append("lineage_incomplete") + if evidence["snapshot_ingest_status"] in {"failed", "quarantined"}: + reasons.append("source_snapshot_unsafe") + # A source family can be authoritative while an individual snapshot + # remains too old or insufficiently described to trust. Historical + # data that is intentionally valid needs an explicit + # ``not_applicable`` contract policy; an omitted, due or stale status + # cannot silently enter a production boundary. + if evidence["source_snapshot_id"] and evidence["snapshot_freshness_status"] not in _CONSUMABLE_SNAPSHOT_FRESHNESS: + reasons.append("source_snapshot_freshness_not_eligible") + + if reasons: + return DatasetConsumptionDecision( + eligible=False, + purpose=purpose, + fixture_legacy_exception=False, + reasons=tuple(sorted(set(reasons))), + evidence=evidence, + ) + + # Fixtures are evidence for tests and QA only. They cannot become + # production inference, derived processing or export inputs merely by + # presenting a fixture flag at a public service boundary. + if phase2_absent and explicit_fixture: + if purpose == "quality_assessment": + return DatasetConsumptionDecision( + eligible=True, + purpose=purpose, + fixture_legacy_exception=True, + reasons=(), + evidence=evidence, + ) + if purpose == "authoritative_coverage": + reasons.append("fixture_not_authoritative_coverage") + else: + reasons.append("fixture_qa_only") + elif phase2_absent and fixture_mode: + # `fixture_mode` is a caller flag, not a source trust claim. A + # manual/unknown production dataset must never self-designate as a + # fixture merely by supplying this parameter. + reasons.append("fixture_source_required") + + # Existing service tests construct transient SQLAlchemy Dataset objects + # directly rather than retrieving a persisted row. A production + # `db.get()` result is persistent and never enters this branch. This + # compatibility path is intentionally unavailable to coverage, where + # a fixture must never appear authoritative. + if ( + phase2_absent + and not reasons + and cls._is_transient_orm_dataset(dataset) + and purpose == "quality_assessment" + ): + return DatasetConsumptionDecision( + eligible=True, + purpose=purpose, + fixture_legacy_exception=True, + reasons=(), + evidence=evidence, + ) + + # QA unit tests intentionally use projection objects + # instead of persisted ORM Datasets. Those projections cannot enter an + # application API boundary; keep the exception isolated to read-only + # candidate verification. Inference, reference validation, derived + # processing, export and coverage never accept a projection. + if phase2_absent and not reasons and not isinstance(dataset, Dataset) and purpose == "quality_assessment": + return DatasetConsumptionDecision( + eligible=True, + purpose=purpose, + fixture_legacy_exception=True, + reasons=(), + evidence=evidence, + ) + + if phase2_absent: + reasons.append("phase2_provenance_missing") + if evidence["dataset_status"] != "ready": + reasons.append("dataset_not_ready") + if evidence["validation_status"] != "passed": + reasons.append("validation_not_passed") + if evidence["provenance_status"] != "complete": + reasons.append("provenance_not_complete") + if evidence["lineage_status"] not in {"complete", "not_applicable"}: + reasons.append("lineage_not_complete") + if evidence["quarantine_status"] != "not_quarantined": + reasons.append("quarantine_status_not_clear") + if not evidence["data_contract_key"] or not evidence["data_contract_version"]: + reasons.append("data_contract_not_versioned") + if not _CHECKSUM_SHA256.fullmatch(str(evidence["checksum_sha256"] or "")): + reasons.append("dataset_checksum_invalid") + if not evidence["source_registry_id"]: + reasons.append("source_registry_missing") + if not evidence["source_snapshot_id"]: + reasons.append("source_snapshot_missing") + if evidence["source_registry_id"] and not evidence["source_classification"]: + reasons.append("source_registry_unresolved") + if evidence["source_snapshot_id"] and evidence["snapshot_ingest_status"] != "ingested": + reasons.append("source_snapshot_not_ingested") + if evidence["source_snapshot_id"] and not _CHECKSUM_SHA256.fullmatch( + str(evidence["snapshot_checksum_sha256"] or "") + ): + reasons.append("source_snapshot_checksum_invalid") + if ( + evidence["source_registry_id"] + and evidence["source_snapshot_id"] + and evidence["snapshot_source_registry_id"] + and evidence["source_registry_id"] != evidence["snapshot_source_registry_id"] + ): + reasons.append("source_snapshot_registry_mismatch") + if ( + _CHECKSUM_SHA256.fullmatch(str(evidence["checksum_sha256"] or "")) + and _CHECKSUM_SHA256.fullmatch(str(evidence["snapshot_checksum_sha256"] or "")) + and str(evidence["checksum_sha256"]).lower() != str(evidence["snapshot_checksum_sha256"]).lower() + ): + reasons.append("source_snapshot_checksum_mismatch") + + if evidence["source_key"] and evidence["source_name"] and evidence["source_key"] != evidence["source_name"]: + reasons.append("source_registry_identity_mismatch") + + # Contextual, corroborative, authoritative and properly derived + # sources can serve their declared non-ground-truth roles once the + # full governed contract passes. Experimental/manual sources cannot + # cross a production boundary; only an explicitly marked fixture may + # participate in candidate QA. + experimental_source = ( + evidence["source_classification"] == "experimental" + or evidence["source_key"] in _UNTRUSTED_SOURCE_KEYS + ) + if experimental_source: + if purpose == "quality_assessment" and explicit_fixture: + pass + elif purpose == "quality_assessment": + reasons.append("experimental_source_requires_fixture_qa") + else: + reasons.append("experimental_source_not_allowed_for_purpose") + + if purpose == "authoritative_coverage": + if evidence["source_classification"] != "authoritative": + reasons.append("coverage_source_not_authoritative") + if evidence["source_key"] and evidence["source_name"] and evidence["source_key"] != evidence["source_name"]: + reasons.append("coverage_source_identity_mismatch") + + if purpose == "reference_validation": + if cls._normalise(cls._value(dataset, "dataset_role")) != "reference": + reasons.append("reference_dataset_role_required") + if evidence["source_classification"] != "authoritative": + reasons.append("reference_source_not_authoritative") + if evidence["source_ground_truth_allowed"] is not True: + reasons.append("reference_source_not_ground_truth_allowed") + if not evidence["reference_task"]: + reasons.append("reference_validation_task_required") + elif not cls._reference_task_is_approved(evidence): + reasons.append("reference_task_authority_not_approved") + + return DatasetConsumptionDecision( + eligible=not reasons, + purpose=purpose, + fixture_legacy_exception=False, + reasons=tuple(sorted(set(reasons))), + evidence=evidence, + ) + + @classmethod + def assert_eligible( + cls, + dataset: Any, + *, + purpose: DatasetConsumptionPurpose, + fixture_mode: bool = False, + reference_task: str | None = None, + ) -> DatasetConsumptionDecision: + decision = cls.evaluate( + dataset, + purpose=purpose, + fixture_mode=fixture_mode, + reference_task=reference_task, + ) + if decision.eligible: + return decision + code = "DATASET_QUARANTINED" if any( + reason in {"dataset_status_unsafe", "dataset_quarantined", "validation_failed", "source_snapshot_unsafe"} + for reason in decision.reasons + ) else "DATASET_PROVENANCE_INCOMPLETE" + raise AppError( + code=code, + message="Dataset cannot be consumed until its provenance and validation gates are satisfied.", + status_code=409, + details={ + "dataset_id": decision.evidence["dataset_id"], + "purpose": purpose, + "reasons": list(decision.reasons), + "fixture_legacy_exception": decision.fixture_legacy_exception, + }, + ) + + @classmethod + def eligible_for_authoritative_coverage(cls, dataset: Any) -> bool: + """Return false instead of raising so coverage can report a gap safely.""" + + return cls.evaluate(dataset, purpose="authoritative_coverage").eligible + + @staticmethod + def _reference_task_is_approved(evidence: Mapping[str, Any]) -> bool: + """Require task-specific primary authority or an explicit zone approval. + + ``classification=authoritative`` is deliberately not a blanket + permission to serve as building truth. A source may be authoritative + for an address lifecycle or elevation product while remaining only + corroborative for footprint QA. A regional product that is marked + ``*_pending_contract`` similarly remains blocked until an operator + records a narrow product-and-zone approval in its server-owned policy. + """ + + task = str(evidence.get("reference_task") or "").strip().lower() + authority = DatasetConsumptionGate._normalise( + DatasetConsumptionGate._mapping(evidence.get("source_validation_authority")).get(task) + ) + if authority == "primary": + return True + if authority not in {"approved", "approved_product_zone"}: + return False + + approvals = DatasetConsumptionGate._mapping(evidence.get("source_reference_validation_approvals")) + approval = DatasetConsumptionGate._mapping(approvals.get(task)) + if approval.get("approved") is not True: + return False + + source_key = str(evidence.get("source_key") or "").strip().lower() + source_scope = DatasetConsumptionGate._mapping(evidence.get("source_authority_scope")) + source_zone = str(source_scope.get("zone") or source_scope.get("scope") or "").strip() + approved_keys = approval.get("source_keys") + approved_zones = approval.get("zones") + if not isinstance(approved_keys, list) or source_key not in { + str(value).strip().lower() for value in approved_keys + }: + return False + if not isinstance(approved_zones, list) or source_zone not in { + str(value).strip() for value in approved_zones + }: + return False + return True diff --git a/backend/app/services/dataset_service.py b/backend/app/services/dataset_service.py index ca245183..0d93b8a8 100644 --- a/backend/app/services/dataset_service.py +++ b/backend/app/services/dataset_service.py @@ -2,17 +2,43 @@ from __future__ import annotations import json import pathlib +import re +from dataclasses import dataclass +from hashlib import sha256 from datetime import datetime, timezone +from math import isfinite from pathlib import Path from typing import Any from uuid import UUID import uuid from fastapi import UploadFile +from shapely.geometry import MultiPoint, shape from sqlalchemy.orm import Session from app.core.errors import AppError from app.models import Area, Dataset, DatasetVersion, Project +from app.services.data_contract_validation import ( + ContractKind, + DataAssetValidationInput, + GeometryRecord, + LineageEvidence, + LineageStatus, + ProvenanceStatus, + QuarantineStatus, + RASTER_GEOTIFF_CONTRACT_KEY, + RASTER_GEOTIFF_CONTRACT_VERSION, + TransformationEvidence, + VECTOR_GEOJSON_CONTRACT_KEY, + VECTOR_GEOJSON_CONTRACT_VERSION, + ValidationIssue, + ValidationReport, + ValidationStatus, + build_raster_ingest_input, + build_vector_ingest_input, + validate_registered_asset, +) +from app.services.data_quarantine_service import DataQuarantineService from app.schemas.dataset import ( DatasetCreateResponse, DatasetStorageResponse, @@ -22,10 +48,348 @@ from app.schemas.dataset import ( ) from app.services.geojson_service import parse_geojson_payload, load_dataset_text from app.services.raster_service import extract_raster_metadata +from app.services.source_registry_service import SourceRegistryService from app.services.storage_service import StorageService from app.services.vector_feature_service import VectorFeatureService +_CHECKSUM_SHA256 = re.compile(r"^[0-9a-f]{64}$") + + +@dataclass(frozen=True) +class _SourceVectorSchema: + """Server-owned vector schema expectations attached to a source registry row.""" + + source_key: str + expected_geometry_types: frozenset[str] + required_attributes: tuple[str, ...] + + @classmethod + def from_source(cls, source: Any) -> "_SourceVectorSchema": + geometry_values = getattr(source, "expected_geometry_types_json", ()) + expected_geometry_types = ( + frozenset(str(value).strip() for value in geometry_values if str(value).strip()) + if isinstance(geometry_values, (list, tuple, set)) + else frozenset() + ) + attributes = getattr(source, "expected_attributes_json", {}) + required_values = attributes.get("required") if isinstance(attributes, dict) else () + if isinstance(required_values, str): + required_values = (required_values,) + required_attributes = ( + tuple(sorted({str(value).strip() for value in required_values if str(value).strip()})) + if isinstance(required_values, (list, tuple, set)) + else () + ) + return cls( + source_key=str(getattr(source, "source_key", "") or "").strip().lower(), + expected_geometry_types=expected_geometry_types, + required_attributes=required_attributes, + ) + + def to_metadata(self, *, checked_feature_count: int) -> dict[str, Any]: + return { + "status": "passed", + "source_key": self.source_key, + "expected_geometry_types": sorted(self.expected_geometry_types), + "required_attributes": list(self.required_attributes), + "checked_feature_count": checked_feature_count, + } + + +def _feature_source_identifier(feature: dict[str, Any], properties: dict[str, Any]) -> Any: + """Return the source identity under the GeoJSON and registry conventions.""" + + return feature.get("id") or properties.get("id") or properties.get("source_feature_id") + + +def _validate_vector_feature_source_schema( + *, + feature: dict[str, Any], + properties: dict[str, Any], + geometry: Any, + schema: _SourceVectorSchema, + feature_context: str, +) -> None: + """Fail closed when a source-specific vector expectation is violated.""" + + if schema.expected_geometry_types and geometry.geom_type not in schema.expected_geometry_types: + raise AppError( + code="SOURCE_SCHEMA_GEOMETRY_TYPE_NOT_ALLOWED", + message=( + f"{feature_context} has geometry type {geometry.geom_type}, which is not " + f"allowed by source registry {schema.source_key or 'unknown'}" + ), + details={ + "source_key": schema.source_key, + "expected_geometry_types": sorted(schema.expected_geometry_types), + "observed_geometry_type": geometry.geom_type, + }, + status_code=400, + ) + for attribute in schema.required_attributes: + value = _feature_source_identifier(feature, properties) if attribute == "id" else properties.get(attribute) + if value is None or (isinstance(value, str) and not value.strip()): + raise AppError( + code="SOURCE_SCHEMA_REQUIRED_ATTRIBUTE_MISSING", + message=( + f"{feature_context} is missing required source attribute {attribute!r} " + f"for registry {schema.source_key or 'unknown'}" + ), + details={"source_key": schema.source_key, "required_attribute": attribute}, + status_code=400, + ) + + +@dataclass(frozen=True) +class _PartitionedVectorAudit: + """Aggregate evidence from a full per-feature partition audit. + + The importer materializes one GeoJSON partition at a time because the + current parser is ``json.loads`` based. It never materializes every + regional partition or every regional Shapely geometry at once. + """ + + feature_count: int + geometry_types: tuple[str, ...] + bounds_json: dict[str, float] + partition_checksums_sha256: dict[str, str] + source_schema_validation: dict[str, Any] + representative_record: GeometryRecord + + def to_metadata(self) -> dict[str, Any]: + return { + "feature_count": self.feature_count, + "geometry_types": list(self.geometry_types), + "bounds_json": dict(self.bounds_json), + "partition_checksums_sha256": dict(self.partition_checksums_sha256), + "source_schema_validation": dict(self.source_schema_validation), + "validation_mode": "partition_bounded_per_feature_with_aggregate_contract_record", + } + + +class _PartitionedGeoJsonRecords: + """Perform a full, partition-bounded feature audit across GeoJSON partitions. + + The generic vector contract materializes its supplied geometry records. A + regional artifact can contain hundreds of thousands of features, so this + class validates one materialized partition at a time and emits a compact + aggregate record for the generic source/checksum/CRS/bounds contract. + Memory is bounded to the largest single partition, not to one feature. + """ + + def __init__( + self, + partition_paths: list[str | Path], + *, + expected_feature_count: int, + declared_partition_checksums: dict[str, Any] | None, + source_schema: _SourceVectorSchema | None = None, + ) -> None: + self._partition_paths = tuple(Path(path) for path in partition_paths) + self._expected_feature_count = expected_feature_count + self._declared_checksums = declared_partition_checksums + self._source_schema = source_schema or _SourceVectorSchema( + source_key="", + expected_geometry_types=frozenset(), + required_attributes=(), + ) + + def audit(self) -> _PartitionedVectorAudit: + declared_checksums = self._validated_declared_checksums() + observed_checksums: dict[str, str] = {} + feature_count = 0 + source_feature_ids: set[str] = set() + geometry_types: set[str] = set() + min_x: float | None = None + min_y: float | None = None + max_x: float | None = None + max_y: float | None = None + for partition_path in self._partition_paths: + try: + raw = partition_path.read_bytes() + payload = json.loads(raw.decode("utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise AppError( + code="INVALID_GEOJSON_PARTITION", + message=f"Could not read GeoJSON partition {partition_path.name}", + status_code=400, + ) from exc + features = payload.get("features") if isinstance(payload, dict) else None + if not isinstance(payload, dict) or payload.get("type") != "FeatureCollection" or not isinstance(features, list): + raise AppError( + code="INVALID_GEOJSON_PARTITION", + message=f"GeoJSON partition {partition_path.name} must be a FeatureCollection", + status_code=400, + ) + observed_checksum = sha256(raw).hexdigest() + expected_checksum = declared_checksums[partition_path.name] + if observed_checksum != expected_checksum: + raise AppError( + code="PARTITION_CHECKSUM_MISMATCH", + message=( + f"Checksum for partition {partition_path.name} does not match " + "the governed acquisition manifest." + ), + status_code=400, + ) + observed_checksums[partition_path.name] = observed_checksum + for index, feature in enumerate(features): + if not isinstance(feature, dict): + raise AppError( + code="INVALID_GEOJSON_PARTITION", + message=f"Feature {index} in {partition_path.name} must be an object", + status_code=400, + ) + properties = feature.get("properties") if isinstance(feature.get("properties"), dict) else {} + source_feature_id = _feature_source_identifier(feature, properties) + if source_feature_id is not None: + normalized_id = str(source_feature_id).strip() + if normalized_id: + if normalized_id in source_feature_ids: + raise AppError( + code="DUPLICATE_SOURCE_FEATURE", + message=( + f"Duplicate source feature {normalized_id} across regional partitions" + ), + status_code=400, + ) + source_feature_ids.add(normalized_id) + try: + geometry = shape(feature.get("geometry")) + except Exception as exc: + raise AppError( + code="GEOMETRY_PARSE_FAILED", + message=f"Feature {index} in {partition_path.name} has invalid GeoJSON geometry", + status_code=400, + ) from exc + if geometry.is_empty: + raise AppError( + code="GEOMETRY_EMPTY", + message=f"Feature {index} in {partition_path.name} has an empty geometry", + status_code=400, + ) + if not geometry.is_valid: + raise AppError( + code="GEOMETRY_INVALID", + message=( + f"Feature {index} in {partition_path.name} is invalid; " + "partitioned ingestion never silently repairs geometry" + ), + status_code=400, + ) + _validate_vector_feature_source_schema( + feature=feature, + properties=properties, + geometry=geometry, + schema=self._source_schema, + feature_context=f"Feature {index} in {partition_path.name}", + ) + feature_bounds = geometry.bounds + if not all(isfinite(value) for value in feature_bounds): + raise AppError( + code="GEOMETRY_BOUNDS_INVALID", + message=f"Feature {index} in {partition_path.name} has non-finite bounds", + status_code=400, + ) + geometry_types.add(geometry.geom_type) + min_x = feature_bounds[0] if min_x is None else min(min_x, feature_bounds[0]) + min_y = feature_bounds[1] if min_y is None else min(min_y, feature_bounds[1]) + max_x = feature_bounds[2] if max_x is None else max(max_x, feature_bounds[2]) + max_y = feature_bounds[3] if max_y is None else max(max_y, feature_bounds[3]) + feature_count += 1 + + if feature_count != self._expected_feature_count: + raise AppError( + code="PARTITION_FEATURE_COUNT_MISMATCH", + message=( + f"Regional artifact declares {self._expected_feature_count} features but " + f"partitions contain {feature_count} features" + ), + status_code=400, + ) + if None in {min_x, min_y, max_x, max_y}: # pragma: no cover - feature-count invariant above + raise AppError( + code="VECTOR_FEATURES_REQUIRED", + message="Partitioned vector artifact has no geometry records.", + status_code=400, + ) + bounds_json = { + "min_x": float(min_x), + "min_y": float(min_y), + "max_x": float(max_x), + "max_y": float(max_y), + } + # A MultiPoint envelope is validation evidence only, not a replacement + # for persisted source features. It gives the generic contract the + # audited aggregate bounds without retaining all Shapely objects. + representative_geometry = MultiPoint( + [ + (bounds_json["min_x"], bounds_json["min_y"]), + (bounds_json["max_x"], bounds_json["min_y"]), + (bounds_json["max_x"], bounds_json["max_y"]), + (bounds_json["min_x"], bounds_json["max_y"]), + ] + ) + return _PartitionedVectorAudit( + feature_count=feature_count, + geometry_types=tuple(sorted(geometry_types)), + bounds_json=bounds_json, + partition_checksums_sha256=dict(sorted(observed_checksums.items())), + source_schema_validation=self._source_schema.to_metadata(checked_feature_count=feature_count), + representative_record=GeometryRecord( + geometry=representative_geometry, + properties={"partitioned_geometry_audit": True}, + identifier="partitioned-geometry-audit", + ), + ) + + def _validated_declared_checksums(self) -> dict[str, str]: + """Require an exact filename-to-SHA256 manifest for every partition. + + A list of checksum values is insufficient: it cannot establish which + municipality/source partition produced which persisted feature set. + The explicit map is also retained with the aggregate audit evidence. + """ + + if not isinstance(self._declared_checksums, dict) or not self._declared_checksums: + raise AppError( + code="PARTITION_CHECKSUM_MANIFEST_REQUIRED", + message="Partitioned ingestion requires a non-empty filename-to-checksum manifest.", + status_code=400, + ) + partition_names = [path.name for path in self._partition_paths] + if len(set(partition_names)) != len(partition_names): + raise AppError( + code="DUPLICATE_PARTITION_IDENTITY", + message="Partitioned ingestion requires unique partition filenames.", + status_code=400, + ) + declared = { + str(key): str(value).strip().lower() + for key, value in self._declared_checksums.items() + } + if len(declared) != len(partition_names) or set(declared) != set(partition_names): + raise AppError( + code="PARTITION_CHECKSUM_MANIFEST_MISMATCH", + message="Partition checksum manifest must contain exactly one entry for each partition filename.", + details={ + "expected_partition_filenames": sorted(partition_names), + "declared_partition_filenames": sorted(declared), + }, + status_code=400, + ) + invalid = sorted(name for name, checksum in declared.items() if not _CHECKSUM_SHA256.fullmatch(checksum)) + if invalid: + raise AppError( + code="PARTITION_CHECKSUM_INVALID", + message="Partition checksum manifest contains a non-SHA256 value.", + details={"partition_filenames": invalid}, + status_code=400, + ) + return declared + + class DatasetService: VECTOR_EXTENSIONS = {".geojson", ".json"} RASTER_EXTENSIONS = {".tif", ".tiff", ".geotiff"} @@ -33,6 +397,302 @@ class DatasetService: RASTER_TYPES = {"raster", "tif", "tiff", "geotiff"} VALID_DATASET_ROLES = {"source", "derived", "reference"} VALID_TEMPORAL_GRANULARITIES = {"snapshot", "day", "month", "year", "period"} + CANONICAL_VECTOR_CRS = "EPSG:4326" + + @staticmethod + def _registry_persistence_available(db: Session) -> bool: + """Return true only for real ORM-backed ingestion transactions. + + Production request handling always supplies a SQLAlchemy Session. The + narrow fallback keeps historical lightweight unit fakes (which predate + the registry tables) isolated; it cannot bypass the database-backed + production import path. + """ + return callable(getattr(db, "query", None)) + + @staticmethod + def _stable_hash(payload: Any) -> str: + return sha256( + json.dumps(payload, sort_keys=True, separators=(",", ":"), default=str).encode("utf-8") + ).hexdigest() + + @classmethod + def _canonical_vector_storage_bytes(cls, payload: dict[str, Any]) -> bytes: + """Serialize the consumable GeoJSON representation deterministically. + + ``VectorFeatureService.canonicalize_geojson_payload`` is the one + place that transforms source coordinates to EPSG:4326. This helper + makes the exact result of that transform the persisted, checksummed + dataset artifact too; it must never remain merely an in-memory view. + """ + + return json.dumps( + payload, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + + @classmethod + def _vector_storage_requires_canonicalization(cls, source_crs: str | None) -> bool: + """Return whether the source file cannot itself be the canonical view. + + A missing CRS is intentionally treated as the GeoJSON/RFC-7946 + default EPSG:4326. Other aliases (for example ``CRS:84``) are + rewritten so every transformed consumption artifact explicitly says + ``EPSG:4326``. + """ + + return str(source_crs or cls.CANONICAL_VECTOR_CRS).strip().upper() != cls.CANONICAL_VECTOR_CRS + + @classmethod + def _persist_vector_source_evidence( + cls, + *, + project_id: UUID, + dataset_id: UUID, + original_filename: str, + content: bytes, + content_type: str | None, + ) -> dict[str, Any]: + """Retain a non-canonical source file outside the consumption path. + + The Dataset's normal ``storage_path`` always points at the canonical + artifact. The source bytes are retained only below ``provenance/`` + and are referenced through structured provenance metadata; consumers + must never treat this location as a dataset input. + """ + + safe_filename = StorageService._safe_filename(original_filename) + evidence_path = ( + StorageService.dataset_root(str(project_id), str(dataset_id), "vector") + / "provenance" + / f"{dataset_id}_source_{safe_filename}" + ) + return StorageService.persist_file( + str(evidence_path), + content, + original_filename=safe_filename, + content_type=content_type, + ) + + @classmethod + def _record_vector_source_evidence( + cls, + *, + source_metadata: dict[str, Any], + provenance_metadata: dict[str, Any], + source_crs: str, + evidence: dict[str, Any], + canonical_checksum_sha256: str, + ) -> None: + """Bind original source bytes to their canonical consumption artifact.""" + + source_artifact = { + "storage_path": evidence["storage_path"], + "checksum_sha256": evidence["checksum_sha256"], + "size_bytes": evidence["size_bytes"], + "content_type": evidence["content_type"], + "source_crs": source_crs, + "retention": "provenance_evidence_only", + } + transformation = { + "name": "vector_crs_normalization", + "version": "1.0.0", + "source_crs": source_crs, + "storage_crs": cls.CANONICAL_VECTOR_CRS, + "source_checksum_sha256": evidence["checksum_sha256"], + "canonical_checksum_sha256": canonical_checksum_sha256, + } + source_metadata["source_artifact"] = source_artifact + provenance_metadata["source_artifact"] = source_artifact + provenance_metadata["canonical_consumption_artifact"] = { + "checksum_sha256": canonical_checksum_sha256, + "crs": cls.CANONICAL_VECTOR_CRS, + "storage_role": "dataset_consumption", + } + provenance_metadata["transformations"] = [ + *( + provenance_metadata.get("transformations") + if isinstance(provenance_metadata.get("transformations"), list) + else [] + ), + transformation, + ] + + @staticmethod + def _calculate_file_checksum_sha256(path: str | Path) -> str: + """Stream an operator artifact before storage for an idempotent ingest key.""" + + artifact = Path(path) + if not artifact.is_file(): + raise AppError( + code="DATASET_FILE_MISSING", + message="Partitioned vector artifact is missing", + details={"artifact_path": str(artifact)}, + status_code=404, + ) + digest = sha256() + with artifact.open("rb") as stream: + for chunk in iter(lambda: stream.read(8 * 1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + @classmethod + def _ingest_key( + cls, + *, + project_id: UUID, + source_key: str, + checksum_sha256: str, + dataset_type: str, + dataset_role: str, + area_id: UUID | None, + reference_layer_name: str | None, + source_version: str | None, + ) -> str: + return cls._stable_hash( + { + "project_id": str(project_id), + "source_key": source_key, + "checksum_sha256": checksum_sha256.lower(), + "dataset_type": dataset_type, + "dataset_role": dataset_role, + "area_id": str(area_id) if area_id else None, + "reference_layer_name": reference_layer_name or None, + "source_version": source_version or None, + "ingest_contract": "phase2-source-provenance-v1", + } + ) + + @staticmethod + def _contract_metadata( + *, + metadata: dict[str, Any], + source_metadata: dict[str, Any] | None, + provenance_metadata: dict[str, Any] | None, + source: Any, + ) -> dict[str, Any]: + result = dict(metadata) + source_values = source_metadata if isinstance(source_metadata, dict) else {} + provenance_values = provenance_metadata if isinstance(provenance_metadata, dict) else {} + result["license"] = ( + result.get("license") + or source_values.get("license") + or source_values.get("license_note") + or provenance_values.get("license") + or getattr(source, "license_name", None) + or "unknown" + ) + result.setdefault( + "usage_restrictions", + source_values.get("usage_restrictions") + or getattr(source, "usage_restrictions", None) + or "unknown", + ) + return result + + @staticmethod + def _validate_vector_source_schema(source: Any, feature_collection: dict[str, Any]) -> dict[str, Any]: + """Validate the server-owned source schema after canonicalization. + + Generic GeoJSON validation proves that a feature collection is + structurally valid. This additional pass proves that it also matches + the geometry and required-attribute expectations recorded for the + selected source registry entry. It deliberately uses the + canonical-storage payload so the evidence describes exactly what will + be persisted in ``vector_features``. + """ + + schema = _SourceVectorSchema.from_source(source) + features = feature_collection.get("features") if isinstance(feature_collection, dict) else None + if not isinstance(features, list): + raise AppError( + code="SOURCE_SCHEMA_FEATURE_COLLECTION_INVALID", + message="Source-schema validation requires a GeoJSON FeatureCollection.", + status_code=400, + ) + for index, feature in enumerate(features): + if not isinstance(feature, dict): + raise AppError( + code="SOURCE_SCHEMA_FEATURE_INVALID", + message=f"Feature {index} is not an object during source-schema validation.", + status_code=400, + ) + properties = feature.get("properties") if isinstance(feature.get("properties"), dict) else {} + try: + geometry = shape(feature.get("geometry")) + except Exception as exc: + raise AppError( + code="SOURCE_SCHEMA_GEOMETRY_INVALID", + message=f"Feature {index} has no parseable geometry during source-schema validation.", + status_code=400, + ) from exc + _validate_vector_feature_source_schema( + feature=feature, + properties=properties, + geometry=geometry, + schema=schema, + feature_context=f"Feature {index}", + ) + return schema.to_metadata(checked_feature_count=len(features)) + + @staticmethod + def _snapshot_freshness_status( + source_key: str, + source_metadata: dict[str, Any] | None, + *, + observed_at: datetime | None, + source_version: str | None, + ) -> str: + metadata = source_metadata if isinstance(source_metadata, dict) else {} + supplied = str(metadata.get("freshness_status") or "").strip().lower() + allowed = {"unknown", "current", "due", "stale", "not_applicable", "review_required"} + if supplied in allowed: + return supplied + if source_key in {"manual", "fixture", "map_selection", "derived", "experimental"}: + return "not_applicable" + return "current" if observed_at is not None or bool((source_version or "").strip()) else "review_required" + + @staticmethod + def _resolution_unit_for_crs(crs: str | None) -> str: + normalized = str(crs or "").strip().upper() + return "degree" if normalized in {"EPSG:4326", "CRS:84", "OGC:CRS84"} else "m" + + @staticmethod + def _failed_validation_report( + *, + asset_id: str, + dataset_type: str, + code: str, + message: str, + now: datetime, + category: str = "parser", + ) -> ValidationReport: + if dataset_type == "vector": + contract_key, contract_version = VECTOR_GEOJSON_CONTRACT_KEY, VECTOR_GEOJSON_CONTRACT_VERSION + else: + contract_key, contract_version = RASTER_GEOTIFF_CONTRACT_KEY, RASTER_GEOTIFF_CONTRACT_VERSION + return ValidationReport( + asset_id=asset_id, + data_contract_key=contract_key, + data_contract_version=contract_version, + contract_fingerprint_sha256=None, + validation_status=ValidationStatus.FAILED, + provenance_status=ProvenanceStatus.INCOMPLETE, + lineage_status=LineageStatus.INCOMPLETE, + quarantine_status=QuarantineStatus.QUARANTINED, + validation_scope=("ingest", dataset_type), + checked_at=now, + issues=( + ValidationIssue( + code=code, + category=category, + field="artifact", + message=message, + ), + ), + ) @staticmethod def _normalize_datetime(value: datetime | None) -> datetime | None: @@ -107,6 +767,16 @@ class DatasetService: reference_layer_name=dataset.reference_layer_name, source_metadata=dataset.source_metadata, provenance_metadata=dataset.provenance_metadata, + ingest_key=dataset.ingest_key, + source_registry_id=dataset.source_registry_id, + source_snapshot_id=dataset.source_snapshot_id, + data_contract_key=dataset.data_contract_key, + data_contract_version=dataset.data_contract_version, + validation_status=dataset.validation_status, + validation_report_json=dataset.validation_report_json, + provenance_status=dataset.provenance_status, + lineage_status=dataset.lineage_status, + quarantine_status=dataset.quarantine_status, imported_at=dataset.imported_at, temporal_series_key=dataset.temporal_series_key, observed_at=dataset.observed_at, @@ -250,8 +920,173 @@ class DatasetService: bands_json["dtype"] = metadata_json["dtype"] return bands_json or None + @classmethod + def _find_existing_ingest(cls, db: Session, project_id: UUID, ingest_key: str) -> Dataset | None: + if not cls._registry_persistence_available(db): + return None + return SourceRegistryService.find_dataset_by_ingest_key(db, project_id, ingest_key) + + @classmethod + def _record_snapshot( + cls, + db: Session, + *, + source_key: str, + checksum_sha256: str, + source_version: str | None, + observed_at: datetime | None, + valid_from: datetime | None, + valid_to: datetime | None, + source_crs: str | None, + source_metadata: dict[str, Any] | None, + metadata: dict[str, Any], + ) -> tuple[Any | None, Any | None]: + if not cls._registry_persistence_available(db): + return None, None + source = SourceRegistryService.ensure_server_owned_source(db, source_key) + source_values = source_metadata if isinstance(source_metadata, dict) else {} + resolution = metadata.get("resolution_json") or metadata.get("resolution") or {} + if isinstance(resolution, (list, tuple)) and len(resolution) >= 2: + resolution = {"x": resolution[0], "y": resolution[1], "unit": cls._resolution_unit_for_crs(source_crs)} + if not isinstance(resolution, dict): + resolution = {"status": "unknown"} + snapshot_key = f"{source_key}:{source_version or 'unversioned'}:{checksum_sha256.lower()}" + snapshot = SourceRegistryService.record_snapshot( + db, + source_key=source_key, + snapshot_key=snapshot_key, + checksum_sha256=checksum_sha256, + source_version=source_version, + snapshot_at=observed_at, + fetched_at=datetime.now(timezone.utc), + reuse_existing_snapshot=True, + source_url=( + source_values.get("source_url") + or source_values.get("catalogue_url") + or source_values.get("service_url") + ), + crs=source_crs, + units=source_values.get("units") or source.default_units, + spatial_resolution=resolution, + temporal_coverage={ + "observed_at": observed_at.isoformat() if observed_at else None, + "valid_from": valid_from.isoformat() if valid_from else None, + "valid_to": valid_to.isoformat() if valid_to else None, + }, + geographic_coverage={ + "bbox": metadata.get("source_bounds_json") or metadata.get("bounds_json") or metadata.get("bounds"), + "coverage_zones": source_values.get("coverage_zones") or source_values.get("coverage_zone"), + }, + observed_schema={ + "dataset_type": metadata.get("dataset_type"), + "geometry_types": metadata.get("geometry_types"), + "bands": metadata.get("band_count"), + "attributes": source_values.get("expected_attributes"), + }, + freshness_status=cls._snapshot_freshness_status( + source_key, + source_metadata, + observed_at=observed_at, + source_version=source_version, + ), + ingest_status="ingested", + known_limitations=list(source_values.get("known_limitations") or []), + snapshot_metadata={ + "source_metadata": source_values, + "source_checksum_sha256": checksum_sha256.lower(), + }, + ) + return source, snapshot + + @classmethod + def _apply_validation_report( + cls, + db: Session, + *, + dataset: Dataset, + dataset_version: DatasetVersion, + report: ValidationReport, + source: Any | None, + snapshot: Any | None, + artifact_path: str | None, + ) -> None: + fields = report.persistence_fields() + dataset.validation_report_json = fields["validation_report_json"] + dataset.quarantine_status = fields["quarantine_status"] + dataset_version.validation_report_json = fields["validation_report_json"] + if source is not None and snapshot is not None: + SourceRegistryService.bind_dataset_provenance( + dataset, + source=source, + snapshot=snapshot, + data_contract_key=fields["data_contract_key"], + data_contract_version=fields["data_contract_version"], + validation_status=fields["validation_status"], + provenance_status=fields["provenance_status"], + lineage_status=fields["lineage_status"], + ) + SourceRegistryService.bind_dataset_version_provenance( + dataset_version, + source=source, + snapshot=snapshot, + data_contract_key=fields["data_contract_key"], + data_contract_version=fields["data_contract_version"], + validation_status=fields["validation_status"], + provenance_status=fields["provenance_status"], + lineage_status=fields["lineage_status"], + ) + else: + for target in (dataset, dataset_version): + target.data_contract_key = fields["data_contract_key"] + target.data_contract_version = fields["data_contract_version"] + target.validation_status = fields["validation_status"] + target.provenance_status = fields["provenance_status"] + target.lineage_status = fields["lineage_status"] + + decision = DataQuarantineService.decide(report) + if decision.eligible_for_use: + dataset.status = "ready" + dataset.quarantine_status = "not_quarantined" + return + dataset.status = "quarantined" + dataset.quarantine_status = "quarantined" + if source is not None and snapshot is not None: + SourceRegistryService.quarantine_dataset( + db, + dataset=dataset, + dataset_version=dataset_version, + source_snapshot=snapshot, + stage="ingest_validation", + reason_code=(decision.reason_codes[0] if decision.reason_codes else "DATA_CONTRACT_FAILED"), + details={"validation_report": report.to_dict(), "quarantine_decision": decision.to_dict()}, + artifact_path=artifact_path, + artifact_checksum_sha256=dataset.checksum_sha256, + ) + + @classmethod + def _new_dataset_version( + cls, + dataset: Dataset, + *, + ingest_key: str | None, + ) -> DatasetVersion: + return DatasetVersion( + id=uuid.uuid4(), + dataset_id=dataset.id, + version=1, + storage_path=dataset.storage_path, + source_version=dataset.source_version, + observed_at=dataset.observed_at, + valid_from=dataset.valid_from, + valid_to=dataset.valid_to, + checksum_sha256=dataset.checksum_sha256, + ingest_key=f"{ingest_key}:v1" if ingest_key else None, + source_metadata=dataset.source_metadata, + provenance_metadata=dataset.provenance_metadata, + ) + @staticmethod - async def upload_dataset( + async def _upload_dataset_legacy( db: Session, project_id: UUID, file: UploadFile, @@ -411,7 +1246,385 @@ class DatasetService: return DatasetService._to_response(dataset) @staticmethod - def import_vector_bytes( + async def upload_dataset( + db: Session, + project_id: UUID, + file: UploadFile, + dataset_type: str, + source: str, + dataset_role: str = "source", + source_name: str | None = None, + reference_layer_name: str | None = None, + source_metadata: dict | None = None, + provenance_metadata: dict | None = None, + area_id: UUID | None = None, + temporal_series_key: str | None = None, + observed_at: datetime | None = None, + valid_from: datetime | None = None, + valid_to: datetime | None = None, + temporal_granularity: str | None = None, + source_version: str | None = None, + ) -> DatasetCreateResponse: + """Stage a user upload as an explicitly manual, non-authoritative source. + + Client text such as ``source_name=grb`` is retained only as a claim in + provenance. It cannot select an authoritative registry entry; only a + server-owned acquisition adapter reaches those entries. + """ + if not DatasetService._registry_persistence_available(db): + return await DatasetService._upload_dataset_legacy( + db=db, + project_id=project_id, + file=file, + dataset_type=dataset_type, + source=source, + dataset_role=dataset_role, + source_name=source_name, + reference_layer_name=reference_layer_name, + source_metadata=source_metadata, + provenance_metadata=provenance_metadata, + area_id=area_id, + temporal_series_key=temporal_series_key, + observed_at=observed_at, + valid_from=valid_from, + valid_to=valid_to, + temporal_granularity=temporal_granularity, + source_version=source_version, + ) + if not db.get(Project, project_id): + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + if area_id is not None: + area = db.get(Area, area_id) + if not area: + raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) + if area.project_id != project_id: + raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400) + + filename = DatasetService._validate_upload_filename(file.filename) + canonical_type = DatasetService._canonical_dataset_type(dataset_type) + normalized_role = DatasetService._normalize_dataset_role(dataset_role) + if normalized_role == "reference" and canonical_type == "raster": + raise AppError(code="INVALID_DATASET_ROLE", message="reference role requires vector dataset type", status_code=400) + extension = DatasetService._extension_for_path(filename) + if canonical_type == "vector" and extension not in DatasetService.VECTOR_EXTENSIONS: + raise AppError(code="INVALID_UPLOAD", message="Vector uploads require .geojson or .json files", status_code=415) + if canonical_type == "raster" and extension not in DatasetService.RASTER_EXTENSIONS: + raise AppError(code="INVALID_UPLOAD", message="Raster uploads require .tif, .tiff or .geotiff files", status_code=415) + + temporal = DatasetService._validate_temporal_metadata( + temporal_series_key=temporal_series_key, + observed_at=observed_at, + valid_from=valid_from, + valid_to=valid_to, + temporal_granularity=temporal_granularity, + source_version=source_version, + ) + raw = await file.read() + checksum_sha256 = StorageService.calculate_checksum_sha256(raw) + ingest_key = DatasetService._ingest_key( + project_id=project_id, + source_key="manual", + checksum_sha256=checksum_sha256, + dataset_type=canonical_type, + dataset_role=normalized_role, + area_id=area_id, + reference_layer_name=reference_layer_name if normalized_role == "reference" else None, + source_version=temporal["source_version"], + ) + existing = DatasetService._find_existing_ingest(db, project_id, ingest_key) + if existing is not None: + return DatasetService._to_response(existing) + + raw_source_metadata = dict(source_metadata or {}) + raw_provenance_metadata = dict(provenance_metadata or {}) + raw_source_metadata.update( + { + "ingest_origin": "manual_upload", + "claimed_source": source, + "claimed_source_name": source_name, + "authority_claim_accepted": False, + } + ) + raw_source_metadata.setdefault( + "temporal_unknown_reason", + "The manual upload does not assert a precise source observation timestamp.", + ) + raw_source_metadata.setdefault( + "source_version_unknown_reason", + "The manual upload has no server-attested source edition or snapshot version.", + ) + raw_provenance_metadata.update( + { + "ingest_origin": "manual_upload", + "ingest_key": ingest_key, + "claimed_source": {"source": source, "source_name": source_name}, + } + ) + + dataset_id = uuid.uuid4() + storage_info: dict[str, Any] | None = None + storage_content = raw + source_evidence: dict[str, Any] | None = None + imported_at = datetime.now(timezone.utc) + metadata: dict[str, Any] = {"dataset_type": canonical_type} + source_crs: str | None = None + canonical_vector_payload: dict[str, Any] | None = None + parser_error: tuple[str, str] | None = None + try: + if canonical_type == "vector": + try: + payload = json.loads(raw.decode("utf-8")) + except UnicodeDecodeError as exc: + raise AppError(code="INVALID_UPLOAD", message="Upload must be UTF-8 encoded", status_code=400) from exc + raw_metadata = parse_geojson_payload(payload) + source_crs = str(raw_metadata.get("crs") or "").strip() or None + canonical_vector_payload = VectorFeatureService.canonicalize_geojson_payload( + payload, + source_crs=source_crs or DatasetService.CANONICAL_VECTOR_CRS, + ) + metadata = parse_geojson_payload(canonical_vector_payload) + metadata.update( + { + "dataset_type": "vector", + "source_crs": source_crs, + "source_bounds_json": raw_metadata.get("bounds_json"), + "source_crs_assumed": raw_metadata.get("crs_assumed", False), + "canonical_storage_crs": DatasetService.CANONICAL_VECTOR_CRS, + } + ) + if DatasetService._vector_storage_requires_canonicalization(source_crs): + storage_content = DatasetService._canonical_vector_storage_bytes(canonical_vector_payload) + source_evidence = DatasetService._persist_vector_source_evidence( + project_id=project_id, + dataset_id=dataset_id, + original_filename=filename, + content=raw, + content_type=file.content_type, + ) + else: + storage_info = StorageService.persist_dataset_file( + project_id=str(project_id), + dataset_id=str(dataset_id), + dataset_type=canonical_type, + original_filename=filename, + content=raw, + content_type=file.content_type, + ) + metadata = extract_raster_metadata(storage_info["storage_path"]) + metadata["dataset_type"] = "raster" + source_crs = metadata.get("crs") + except (ValueError, json.JSONDecodeError, AppError) as exc: + code = exc.code if isinstance(exc, AppError) else "INVALID_GEOJSON" + parser_error = (code, str(exc)) + metadata = { + "dataset_type": canonical_type, + "processing_error": str(exc), + "processing_code": code, + } + + if storage_info is None: + storage_info = StorageService.persist_dataset_file( + project_id=str(project_id), + dataset_id=str(dataset_id), + dataset_type=canonical_type, + original_filename=filename, + content=storage_content, + content_type=file.content_type, + ) + computed_storage_checksum_sha256 = StorageService.calculate_checksum_sha256(storage_content) + if source_evidence is not None: + resolved_source_crs = source_crs or DatasetService.CANONICAL_VECTOR_CRS + DatasetService._record_vector_source_evidence( + source_metadata=raw_source_metadata, + provenance_metadata=raw_provenance_metadata, + source_crs=resolved_source_crs, + evidence=source_evidence, + canonical_checksum_sha256=computed_storage_checksum_sha256, + ) + metadata.update( + { + "source_artifact_checksum_sha256": source_evidence["checksum_sha256"], + "canonical_artifact_checksum_sha256": computed_storage_checksum_sha256, + } + ) + + source_registry, source_snapshot = DatasetService._record_snapshot( + db, + source_key="manual", + checksum_sha256=storage_info["checksum_sha256"], + source_version=temporal["source_version"], + observed_at=temporal["observed_at"], + valid_from=temporal["valid_from"], + valid_to=temporal["valid_to"], + source_crs=source_crs, + source_metadata=raw_source_metadata, + metadata=metadata, + ) + contract_metadata = DatasetService._contract_metadata( + metadata=metadata, + source_metadata=raw_source_metadata, + provenance_metadata=raw_provenance_metadata, + source=source_registry, + ) + if parser_error is not None: + report = DatasetService._failed_validation_report( + asset_id=ingest_key, + dataset_type=canonical_type, + code=parser_error[0], + message=parser_error[1], + now=imported_at, + ) + elif canonical_type == "vector": + source_crs = source_crs or DatasetService.CANONICAL_VECTOR_CRS + lineage = LineageEvidence() + if source_crs.upper() != DatasetService.CANONICAL_VECTOR_CRS: + lineage = LineageEvidence( + transformations=( + TransformationEvidence( + name="vector_crs_normalization", + version="1.0.0", + checksum_sha256=DatasetService._stable_hash( + {"source_crs": source_crs, "storage_crs": DatasetService.CANONICAL_VECTOR_CRS} + ), + ), + ) + ) + try: + contract_metadata["source_schema_validation"] = DatasetService._validate_vector_source_schema( + source_registry, + canonical_vector_payload or {"type": "FeatureCollection", "features": []}, + ) + report = validate_registered_asset( + build_vector_ingest_input( + asset_id=ingest_key, + source_crs=source_crs, + storage_crs=DatasetService.CANONICAL_VECTOR_CRS, + feature_collection=canonical_vector_payload or {"type": "FeatureCollection", "features": []}, + checksum_sha256=storage_info["checksum_sha256"], + computed_checksum_sha256=computed_storage_checksum_sha256, + content=storage_content, + source_registry_id=str(source_registry.id), + source_snapshot_id=str(source_snapshot.id), + imported_at=imported_at, + metadata=contract_metadata, + observed_at=temporal["observed_at"], + valid_from=temporal["valid_from"], + valid_to=temporal["valid_to"], + temporal_unknown_reason=raw_source_metadata["temporal_unknown_reason"], + source_version=temporal["source_version"], + source_version_unknown_reason=raw_source_metadata["source_version_unknown_reason"], + lineage=lineage, + ) + ) + except AppError as exc: + report = DatasetService._failed_validation_report( + asset_id=ingest_key, + dataset_type="vector", + code=exc.code, + message=exc.message, + now=imported_at, + category="source_schema", + ) + else: + resolution_json = DatasetService._extract_raster_resolution_json(metadata) + resolution = ( + {"x": resolution_json["x"], "y": resolution_json["y"], "unit": DatasetService._resolution_unit_for_crs(source_crs)} + if resolution_json + else None + ) + report = validate_registered_asset( + build_raster_ingest_input( + asset_id=ingest_key, + source_crs=source_crs, + storage_crs=source_crs, + raster_profile=metadata, + bounds=DatasetService._extract_raster_bounds_json(metadata), + resolution=resolution, + checksum_sha256=storage_info["checksum_sha256"], + computed_checksum_sha256=checksum_sha256, + content=raw, + source_registry_id=str(source_registry.id), + source_snapshot_id=str(source_snapshot.id), + imported_at=imported_at, + metadata=contract_metadata, + observed_at=temporal["observed_at"], + valid_from=temporal["valid_from"], + valid_to=temporal["valid_to"], + temporal_unknown_reason=raw_source_metadata["temporal_unknown_reason"], + source_version=temporal["source_version"], + source_version_unknown_reason=raw_source_metadata["source_version_unknown_reason"], + ) + ) + + bounds_json = metadata.get("bounds_json") + resolution_json = metadata.get("resolution_json") + bands_json = metadata.get("bands_json") + if canonical_type == "raster": + bounds_json = DatasetService._extract_raster_bounds_json(metadata) + resolution_json = DatasetService._extract_raster_resolution_json(metadata) + bands_json = DatasetService._extract_raster_bands_json(metadata) + dataset = Dataset( + id=dataset_id, + project_id=project_id, + area_id=area_id, + name=filename, + dataset_type=canonical_type, + source="manual_upload", + dataset_role=normalized_role, + source_name="manual", + reference_layer_name=reference_layer_name if normalized_role == "reference" else None, + source_metadata=raw_source_metadata, + provenance_metadata=raw_provenance_metadata, + imported_at=imported_at, + ingest_key=ingest_key, + **temporal, + storage_path=storage_info["storage_path"], + original_filename=storage_info["original_filename"], + stored_filename=storage_info["stored_filename"], + content_type=storage_info["content_type"], + size_bytes=storage_info["size_bytes"], + checksum_sha256=storage_info["checksum_sha256"], + crs=(DatasetService.CANONICAL_VECTOR_CRS if canonical_type == "vector" else source_crs), + bounds_json=bounds_json, + resolution_json=resolution_json, + bands_json=bands_json, + metadata_json=contract_metadata, + status="validating", + ) + dataset_version = DatasetService._new_dataset_version(dataset, ingest_key=ingest_key) + try: + db.add(dataset) + db.add(dataset_version) + db.flush() + DatasetService._apply_validation_report( + db, + dataset=dataset, + dataset_version=dataset_version, + report=report, + source=source_registry, + snapshot=source_snapshot, + artifact_path=storage_info["storage_path"], + ) + if report.validation_status == ValidationStatus.PASSED and canonical_vector_payload is not None: + VectorFeatureService.persist_geojson_features( + db=db, + dataset_id=dataset.id, + payload=canonical_vector_payload, + feature_class=reference_layer_name if normalized_role == "reference" else None, + source_crs=DatasetService.CANONICAL_VECTOR_CRS, + commit=False, + ) + db.commit() + db.refresh(dataset) + except Exception: + db.rollback() + # Keep the staged bytes. A transport/database failure must remain + # inspectable instead of silently deleting the only evidence. + raise + return DatasetService._to_response(dataset) + + @staticmethod + def _import_vector_bytes_legacy( db: Session, *, project_id: UUID, @@ -531,7 +1744,7 @@ class DatasetService: return DatasetService._to_response(dataset) @staticmethod - def import_raster_bytes( + def _import_raster_bytes_legacy( db: Session, *, project_id: UUID, @@ -632,6 +1845,501 @@ class DatasetService: StorageService.remove_dataset_file(storage_info["storage_path"]) raise + @staticmethod + def _governed_import_bytes( + db: Session, + *, + project_id: UUID, + filename: str, + content: bytes, + dataset_type: str, + source: str, + source_name: str, + dataset_role: str, + reference_layer_name: str | None, + source_metadata: dict[str, Any] | None, + provenance_metadata: dict[str, Any] | None, + area_id: UUID | None, + temporal_series_key: str | None, + observed_at: datetime | None, + valid_from: datetime | None, + valid_to: datetime | None, + temporal_granularity: str | None, + source_version: str | None, + content_type: str, + ) -> DatasetCreateResponse: + """Persist an adapter-owned source through one governed ingestion path. + + Acquisition adapters choose an entry from the server-owned registry; + they cannot create authority identities dynamically. The original + artifact is deliberately retained if parsing or validation fails so + the immutable checksum, source snapshot and quarantine record remain + reviewable. + """ + if not db.get(Project, project_id): + raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) + if area_id is not None: + area = db.get(Area, area_id) + if not area: + raise AppError(code="AREA_NOT_FOUND", message="Area not found", status_code=404) + if area.project_id != project_id: + raise AppError(code="INVALID_DATASET_SCOPE", message="Area does not belong to this project", status_code=400) + if not content: + raise AppError(code="INVALID_UPLOAD", message=f"{dataset_type.title()} artifact is empty", status_code=400) + + canonical_type = DatasetService._canonical_dataset_type(dataset_type) + normalized_role = DatasetService._normalize_dataset_role(dataset_role) + if normalized_role == "reference" and canonical_type == "raster": + raise AppError(code="INVALID_DATASET_ROLE", message="reference role requires vector dataset type", status_code=400) + safe_filename = DatasetService._validate_upload_filename(filename) + extension = DatasetService._extension_for_path(safe_filename) + allowed_extensions = DatasetService.VECTOR_EXTENSIONS if canonical_type == "vector" else DatasetService.RASTER_EXTENSIONS + if extension not in allowed_extensions: + expected = ".geojson or .json" if canonical_type == "vector" else ".tif, .tiff or .geotiff" + raise AppError(code="INVALID_UPLOAD", message=f"{canonical_type.title()} artifacts require {expected} files", status_code=415) + + source_key = SourceRegistryService.normalize_source_key(source_name) + # This lookup deliberately happens before writing the artifact. A + # typo in an internal adapter must not acquire an unregistered source + # identity or silently downgrade itself to a manual source. + SourceRegistryService.definition_for(source_key) + temporal = DatasetService._validate_temporal_metadata( + temporal_series_key=temporal_series_key, + observed_at=observed_at, + valid_from=valid_from, + valid_to=valid_to, + temporal_granularity=temporal_granularity, + source_version=source_version, + ) + computed_checksum_sha256 = StorageService.calculate_checksum_sha256(content) + ingest_key = DatasetService._ingest_key( + project_id=project_id, + source_key=source_key, + checksum_sha256=computed_checksum_sha256, + dataset_type=canonical_type, + dataset_role=normalized_role, + area_id=area_id, + reference_layer_name=reference_layer_name if normalized_role == "reference" else None, + source_version=temporal["source_version"], + ) + existing = DatasetService._find_existing_ingest(db, project_id, ingest_key) + if existing is not None: + return DatasetService._to_response(existing) + + governed_source_metadata = dict(source_metadata or {}) + governed_provenance_metadata = dict(provenance_metadata or {}) + governed_source_metadata.update( + { + "ingest_origin": "governed_acquisition_adapter", + "source_registry_key": source_key, + "authority_claim_accepted": True, + } + ) + governed_source_metadata.setdefault( + "temporal_unknown_reason", + "The governed source did not publish a precise observation timestamp for this snapshot.", + ) + governed_source_metadata.setdefault( + "source_version_unknown_reason", + "The governed source did not publish a stable source edition; the immutable checksum identifies this snapshot.", + ) + governed_provenance_metadata.update( + { + "ingest_origin": "governed_acquisition_adapter", + "source_registry_key": source_key, + "ingest_key": ingest_key, + } + ) + + dataset_id = uuid.uuid4() + storage_info: dict[str, Any] | None = None + storage_content = content + source_evidence: dict[str, Any] | None = None + imported_at = datetime.now(timezone.utc) + metadata: dict[str, Any] = {"dataset_type": canonical_type} + source_crs: str | None = None + canonical_vector_payload: dict[str, Any] | None = None + parser_error: tuple[str, str] | None = None + try: + if canonical_type == "vector": + try: + payload = json.loads(content.decode("utf-8")) + except UnicodeDecodeError as exc: + raise AppError(code="INVALID_UPLOAD", message="Vector artifact must be UTF-8 encoded", status_code=400) from exc + raw_metadata = parse_geojson_payload(payload) + source_crs = str(raw_metadata.get("crs") or "").strip() or None + canonical_vector_payload = VectorFeatureService.canonicalize_geojson_payload( + payload, + source_crs=source_crs or DatasetService.CANONICAL_VECTOR_CRS, + ) + metadata = parse_geojson_payload(canonical_vector_payload) + metadata.update( + { + "dataset_type": "vector", + "source_crs": source_crs, + "source_bounds_json": raw_metadata.get("bounds_json"), + "source_crs_assumed": raw_metadata.get("crs_assumed", False), + "canonical_storage_crs": DatasetService.CANONICAL_VECTOR_CRS, + } + ) + if DatasetService._vector_storage_requires_canonicalization(source_crs): + storage_content = DatasetService._canonical_vector_storage_bytes(canonical_vector_payload) + source_evidence = DatasetService._persist_vector_source_evidence( + project_id=project_id, + dataset_id=dataset_id, + original_filename=safe_filename, + content=content, + content_type=content_type, + ) + else: + storage_info = StorageService.persist_dataset_file( + project_id=str(project_id), + dataset_id=str(dataset_id), + dataset_type=canonical_type, + original_filename=safe_filename, + content=content, + content_type=content_type, + ) + metadata = extract_raster_metadata(storage_info["storage_path"]) + metadata["dataset_type"] = "raster" + source_crs = str(metadata.get("crs") or "").strip() or None + except (ValueError, json.JSONDecodeError, AppError) as exc: + code = exc.code if isinstance(exc, AppError) else "INVALID_GEOJSON" + parser_error = (code, str(exc)) + metadata = {"dataset_type": canonical_type, "processing_error": str(exc), "processing_code": code} + + if storage_info is None: + storage_info = StorageService.persist_dataset_file( + project_id=str(project_id), + dataset_id=str(dataset_id), + dataset_type=canonical_type, + original_filename=safe_filename, + content=storage_content, + content_type=content_type, + ) + computed_storage_checksum_sha256 = StorageService.calculate_checksum_sha256(storage_content) + if source_evidence is not None: + resolved_source_crs = source_crs or DatasetService.CANONICAL_VECTOR_CRS + DatasetService._record_vector_source_evidence( + source_metadata=governed_source_metadata, + provenance_metadata=governed_provenance_metadata, + source_crs=resolved_source_crs, + evidence=source_evidence, + canonical_checksum_sha256=computed_storage_checksum_sha256, + ) + metadata.update( + { + "source_artifact_checksum_sha256": source_evidence["checksum_sha256"], + "canonical_artifact_checksum_sha256": computed_storage_checksum_sha256, + } + ) + + source_registry, source_snapshot = DatasetService._record_snapshot( + db, + source_key=source_key, + checksum_sha256=storage_info["checksum_sha256"], + source_version=temporal["source_version"], + observed_at=temporal["observed_at"], + valid_from=temporal["valid_from"], + valid_to=temporal["valid_to"], + source_crs=source_crs, + source_metadata=governed_source_metadata, + metadata=metadata, + ) + contract_metadata = DatasetService._contract_metadata( + metadata=metadata, + source_metadata=governed_source_metadata, + provenance_metadata=governed_provenance_metadata, + source=source_registry, + ) + if parser_error is not None: + report = DatasetService._failed_validation_report( + asset_id=ingest_key, + dataset_type=canonical_type, + code=parser_error[0], + message=parser_error[1], + now=imported_at, + ) + elif canonical_type == "vector": + resolved_source_crs = source_crs or DatasetService.CANONICAL_VECTOR_CRS + lineage = LineageEvidence() + if resolved_source_crs.upper() != DatasetService.CANONICAL_VECTOR_CRS: + lineage = LineageEvidence( + transformations=( + TransformationEvidence( + name="vector_crs_normalization", + version="1.0.0", + checksum_sha256=DatasetService._stable_hash( + {"source_crs": resolved_source_crs, "storage_crs": DatasetService.CANONICAL_VECTOR_CRS} + ), + ), + ) + ) + try: + contract_metadata["source_schema_validation"] = DatasetService._validate_vector_source_schema( + source_registry, + canonical_vector_payload or {"type": "FeatureCollection", "features": []}, + ) + report = validate_registered_asset( + build_vector_ingest_input( + asset_id=ingest_key, + source_crs=resolved_source_crs, + storage_crs=DatasetService.CANONICAL_VECTOR_CRS, + feature_collection=canonical_vector_payload or {"type": "FeatureCollection", "features": []}, + checksum_sha256=storage_info["checksum_sha256"], + computed_checksum_sha256=computed_storage_checksum_sha256, + content=storage_content, + source_registry_id=str(source_registry.id), + source_snapshot_id=str(source_snapshot.id), + imported_at=imported_at, + metadata=contract_metadata, + observed_at=temporal["observed_at"], + valid_from=temporal["valid_from"], + valid_to=temporal["valid_to"], + temporal_unknown_reason=governed_source_metadata["temporal_unknown_reason"], + source_version=temporal["source_version"], + source_version_unknown_reason=governed_source_metadata["source_version_unknown_reason"], + lineage=lineage, + ) + ) + except AppError as exc: + report = DatasetService._failed_validation_report( + asset_id=ingest_key, + dataset_type="vector", + code=exc.code, + message=exc.message, + now=imported_at, + category="source_schema", + ) + else: + resolution_json = DatasetService._extract_raster_resolution_json(metadata) + resolution = ( + { + "x": resolution_json["x"], + "y": resolution_json["y"], + "unit": DatasetService._resolution_unit_for_crs(source_crs), + } + if resolution_json + else None + ) + report = validate_registered_asset( + build_raster_ingest_input( + asset_id=ingest_key, + source_crs=source_crs, + storage_crs=source_crs, + raster_profile=metadata, + bounds=DatasetService._extract_raster_bounds_json(metadata), + resolution=resolution, + checksum_sha256=storage_info["checksum_sha256"], + computed_checksum_sha256=computed_checksum_sha256, + content=content, + source_registry_id=str(source_registry.id), + source_snapshot_id=str(source_snapshot.id), + imported_at=imported_at, + metadata=contract_metadata, + observed_at=temporal["observed_at"], + valid_from=temporal["valid_from"], + valid_to=temporal["valid_to"], + temporal_unknown_reason=governed_source_metadata["temporal_unknown_reason"], + source_version=temporal["source_version"], + source_version_unknown_reason=governed_source_metadata["source_version_unknown_reason"], + ) + ) + + bounds_json = metadata.get("bounds_json") + resolution_json = metadata.get("resolution_json") + bands_json = metadata.get("bands_json") + if canonical_type == "raster": + bounds_json = DatasetService._extract_raster_bounds_json(metadata) + resolution_json = DatasetService._extract_raster_resolution_json(metadata) + bands_json = DatasetService._extract_raster_bands_json(metadata) + dataset = Dataset( + id=dataset_id, + project_id=project_id, + area_id=area_id, + name=safe_filename, + dataset_type=canonical_type, + source=source, + dataset_role=normalized_role, + source_name=source_key, + reference_layer_name=reference_layer_name if normalized_role == "reference" else None, + source_metadata=governed_source_metadata, + provenance_metadata=governed_provenance_metadata, + imported_at=imported_at, + ingest_key=ingest_key, + **temporal, + storage_path=storage_info["storage_path"], + original_filename=storage_info["original_filename"], + stored_filename=storage_info["stored_filename"], + content_type=storage_info["content_type"], + size_bytes=storage_info["size_bytes"], + checksum_sha256=storage_info["checksum_sha256"], + crs=(DatasetService.CANONICAL_VECTOR_CRS if canonical_type == "vector" else source_crs), + bounds_json=bounds_json, + resolution_json=resolution_json, + bands_json=bands_json, + metadata_json=contract_metadata, + status="validating", + ) + dataset_version = DatasetService._new_dataset_version(dataset, ingest_key=ingest_key) + try: + db.add(dataset) + db.add(dataset_version) + db.flush() + DatasetService._apply_validation_report( + db, + dataset=dataset, + dataset_version=dataset_version, + report=report, + source=source_registry, + snapshot=source_snapshot, + artifact_path=storage_info["storage_path"], + ) + if report.validation_status == ValidationStatus.PASSED and canonical_vector_payload is not None: + VectorFeatureService.persist_geojson_features( + db=db, + dataset_id=dataset.id, + payload=canonical_vector_payload, + feature_class=reference_layer_name if normalized_role == "reference" else None, + source_crs=DatasetService.CANONICAL_VECTOR_CRS, + commit=False, + ) + db.commit() + db.refresh(dataset) + except Exception: + db.rollback() + # Leave the staged artifact untouched. A failed persistence + # transaction is not evidence that the source bytes were safe to + # delete or that an acquisition can be repeated silently. + raise + return DatasetService._to_response(dataset) + + @staticmethod + def import_vector_bytes( + db: Session, + *, + project_id: UUID, + filename: str, + content: bytes, + source: str, + source_name: str, + dataset_role: str, + reference_layer_name: str | None, + source_metadata: dict[str, Any], + provenance_metadata: dict[str, Any], + area_id: UUID | None = None, + temporal_series_key: str | None = None, + observed_at: datetime | None = None, + valid_from: datetime | None = None, + valid_to: datetime | None = None, + temporal_granularity: str | None = None, + source_version: str | None = None, + content_type: str = "application/geo+json", + ) -> DatasetCreateResponse: + if not DatasetService._registry_persistence_available(db): + return DatasetService._import_vector_bytes_legacy( + db, + project_id=project_id, + filename=filename, + content=content, + source=source, + source_name=source_name, + dataset_role=dataset_role, + reference_layer_name=reference_layer_name, + source_metadata=source_metadata, + provenance_metadata=provenance_metadata, + area_id=area_id, + temporal_series_key=temporal_series_key, + observed_at=observed_at, + valid_from=valid_from, + valid_to=valid_to, + temporal_granularity=temporal_granularity, + source_version=source_version, + content_type=content_type, + ) + return DatasetService._governed_import_bytes( + db, + project_id=project_id, + filename=filename, + content=content, + dataset_type="vector", + source=source, + source_name=source_name, + dataset_role=dataset_role, + reference_layer_name=reference_layer_name, + source_metadata=source_metadata, + provenance_metadata=provenance_metadata, + area_id=area_id, + temporal_series_key=temporal_series_key, + observed_at=observed_at, + valid_from=valid_from, + valid_to=valid_to, + temporal_granularity=temporal_granularity, + source_version=source_version, + content_type=content_type, + ) + + @staticmethod + def import_raster_bytes( + db: Session, + *, + project_id: UUID, + filename: str, + content: bytes, + source: str, + source_name: str, + source_metadata: dict[str, Any], + provenance_metadata: dict[str, Any], + area_id: UUID | None = None, + temporal_series_key: str | None = None, + observed_at: datetime | None = None, + valid_from: datetime | None = None, + valid_to: datetime | None = None, + temporal_granularity: str | None = None, + source_version: str | None = None, + content_type: str = "image/tiff", + ) -> DatasetCreateResponse: + if not DatasetService._registry_persistence_available(db): + return DatasetService._import_raster_bytes_legacy( + db, + project_id=project_id, + filename=filename, + content=content, + source=source, + source_name=source_name, + source_metadata=source_metadata, + provenance_metadata=provenance_metadata, + area_id=area_id, + temporal_series_key=temporal_series_key, + observed_at=observed_at, + valid_from=valid_from, + valid_to=valid_to, + temporal_granularity=temporal_granularity, + source_version=source_version, + content_type=content_type, + ) + return DatasetService._governed_import_bytes( + db, + project_id=project_id, + filename=filename, + content=content, + dataset_type="raster", + source=source, + source_name=source_name, + dataset_role="source", + reference_layer_name=None, + source_metadata=source_metadata, + provenance_metadata=provenance_metadata, + area_id=area_id, + temporal_series_key=temporal_series_key, + observed_at=observed_at, + valid_from=valid_from, + valid_to=valid_to, + temporal_granularity=temporal_granularity, + source_version=source_version, + content_type=content_type, + ) + @staticmethod def import_partitioned_vector_artifact( db: Session, @@ -654,6 +2362,12 @@ class DatasetService: source_version: str | None = None, batch_size: int = 1000, ) -> DatasetCreateResponse: + if not DatasetService._registry_persistence_available(db): + raise AppError( + code="SOURCE_REGISTRY_PERSISTENCE_UNAVAILABLE", + message="Partitioned authoritative imports require registry and provenance persistence.", + status_code=503, + ) if not db.get(Project, project_id): raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404) area = db.get(Area, area_id) @@ -672,6 +2386,10 @@ class DatasetService: if DatasetService._extension_for_path(filename) not in DatasetService.VECTOR_EXTENSIONS: raise AppError(code="INVALID_UPLOAD", message="Vector artifacts require .geojson or .json files", status_code=415) normalized_role = DatasetService._normalize_dataset_role(dataset_role) + source_key = SourceRegistryService.normalize_source_key(source_name) + # A partitioned operator artifact is never allowed to manufacture a + # source identity from its caller-provided label. + SourceRegistryService.definition_for(source_key) temporal = DatasetService._validate_temporal_metadata( temporal_series_key=temporal_series_key, observed_at=observed_at, @@ -689,6 +2407,50 @@ class DatasetService: status_code=400, ) + artifact_checksum_sha256 = DatasetService._calculate_file_checksum_sha256(artifact_path) + ingest_key = DatasetService._ingest_key( + project_id=project_id, + source_key=source_key, + checksum_sha256=artifact_checksum_sha256, + dataset_type="vector", + dataset_role=normalized_role, + area_id=area_id, + reference_layer_name=reference_layer_name if normalized_role == "reference" else None, + source_version=temporal["source_version"], + ) + existing = DatasetService._find_existing_ingest(db, project_id, ingest_key) + if existing is not None: + return DatasetService._to_response(existing) + + governed_source_metadata = dict(source_metadata or {}) + governed_provenance_metadata = dict(provenance_metadata or {}) + governed_source_metadata.update( + { + "ingest_origin": "governed_partitioned_acquisition_adapter", + "source_registry_key": source_key, + "authority_claim_accepted": True, + "partitioned_artifact": True, + } + ) + governed_source_metadata.setdefault( + "temporal_unknown_reason", + "The governed source did not publish a precise observation timestamp for this snapshot.", + ) + governed_source_metadata.setdefault( + "source_version_unknown_reason", + "The governed source did not publish a stable source edition; the immutable checksum identifies this snapshot.", + ) + governed_provenance_metadata.update( + { + "ingest_origin": "governed_partitioned_acquisition_adapter", + "source_registry_key": source_key, + "ingest_key": ingest_key, + "artifact_checksum_sha256": artifact_checksum_sha256, + "combined_artifact_checksum_sha256": artifact_checksum_sha256, + "partition_count": len(partition_paths), + } + ) + dataset_id = uuid.uuid4() storage_info = StorageService.persist_dataset_file_from_path( project_id=str(project_id), @@ -698,6 +2460,177 @@ class DatasetService: source_path=artifact_path, content_type="application/geo+json", ) + # The source file may have changed while it was copied. Re-key on the + # bytes actually retained; never bind a snapshot to a stale pre-copy + # checksum. + persisted_checksum_sha256 = str(storage_info["checksum_sha256"]) + if persisted_checksum_sha256 != artifact_checksum_sha256: + ingest_key = DatasetService._ingest_key( + project_id=project_id, + source_key=source_key, + checksum_sha256=persisted_checksum_sha256, + dataset_type="vector", + dataset_role=normalized_role, + area_id=area_id, + reference_layer_name=reference_layer_name if normalized_role == "reference" else None, + source_version=temporal["source_version"], + ) + existing = DatasetService._find_existing_ingest(db, project_id, ingest_key) + if existing is not None: + StorageService.remove_dataset_file(str(storage_info["storage_path"])) + return DatasetService._to_response(existing) + governed_provenance_metadata["ingest_key"] = ingest_key + governed_provenance_metadata["artifact_checksum_sha256"] = persisted_checksum_sha256 + governed_provenance_metadata["combined_artifact_checksum_sha256"] = persisted_checksum_sha256 + + storage_crs = str( + metadata.get("canonical_storage_crs") + or metadata.get("storage_crs") + or metadata.get("crs") + or DatasetService.CANONICAL_VECTOR_CRS + ).strip() + source_crs = str(metadata.get("source_crs") or storage_crs).strip() or None + metadata.update( + { + "dataset_type": "vector", + "canonical_storage_crs": DatasetService.CANONICAL_VECTOR_CRS, + "partitioned_artifact": True, + "partition_count": len(partition_paths), + } + ) + source_registry, source_snapshot = DatasetService._record_snapshot( + db, + source_key=source_key, + checksum_sha256=persisted_checksum_sha256, + source_version=temporal["source_version"], + observed_at=temporal["observed_at"], + valid_from=temporal["valid_from"], + valid_to=temporal["valid_to"], + source_crs=source_crs, + source_metadata=governed_source_metadata, + metadata=metadata, + ) + # Registry persistence is checked above, so absence here is an + # infrastructure fault rather than a state that can be imported. + if source_registry is None or source_snapshot is None: # pragma: no cover - defensive invariant + raise AppError( + code="SOURCE_REGISTRY_PERSISTENCE_UNAVAILABLE", + message="Source registry persistence did not return a governed snapshot.", + status_code=503, + ) + contract_metadata = DatasetService._contract_metadata( + metadata=metadata, + source_metadata=governed_source_metadata, + provenance_metadata=governed_provenance_metadata, + source=source_registry, + ) + partition_records = _PartitionedGeoJsonRecords( + partition_paths, + expected_feature_count=expected_feature_count, + declared_partition_checksums=governed_provenance_metadata.get("partition_checksums"), + source_schema=_SourceVectorSchema.from_source(source_registry), + ) + lineage = LineageEvidence() + if source_crs and storage_crs.upper() != source_crs.upper(): + lineage = LineageEvidence( + transformations=( + TransformationEvidence( + name="partitioned_vector_crs_normalization", + version="1.0.0", + checksum_sha256=DatasetService._stable_hash( + { + "source_crs": source_crs, + "storage_crs": storage_crs, + "partition_count": len(partition_paths), + } + ), + ), + ) + ) + declared_artifact_checksum = str(governed_provenance_metadata.get("artifact_sha256") or "").strip().lower() + artifact_binding_error: tuple[str, str] | None = None + if not declared_artifact_checksum: + artifact_binding_error = ( + "ARTIFACT_CHECKSUM_REQUIRED", + "Partitioned ingestion requires the acquisition manifest's combined artifact checksum.", + ) + elif not _CHECKSUM_SHA256.fullmatch(declared_artifact_checksum): + artifact_binding_error = ( + "ARTIFACT_CHECKSUM_INVALID", + "Declared partitioned artifact checksum must be a lowercase SHA-256 value.", + ) + elif declared_artifact_checksum != persisted_checksum_sha256: + artifact_binding_error = ( + "ARTIFACT_CHECKSUM_MISMATCH", + "Declared artifact checksum does not match the retained partitioned artifact.", + ) + try: + partition_audit = partition_records.audit() + contract_metadata["partitioned_geometry_audit"] = partition_audit.to_metadata() + contract_metadata["source_schema_validation"] = partition_audit.source_schema_validation + governed_provenance_metadata["partition_checksum_manifest_sha256"] = DatasetService._stable_hash( + partition_audit.partition_checksums_sha256 + ) + governed_provenance_metadata["partitioned_artifact_binding_sha256"] = DatasetService._stable_hash( + { + "combined_artifact_checksum_sha256": persisted_checksum_sha256, + "partition_checksum_manifest_sha256": governed_provenance_metadata[ + "partition_checksum_manifest_sha256" + ], + "feature_count": partition_audit.feature_count, + "storage_crs": storage_crs, + } + ) + if artifact_binding_error is not None: + report = DatasetService._failed_validation_report( + asset_id=ingest_key, + dataset_type="vector", + code=artifact_binding_error[0], + message=artifact_binding_error[1], + now=datetime.now(timezone.utc), + category="checksum", + ) + else: + report = validate_registered_asset( + DataAssetValidationInput( + asset_id=ingest_key, + data_contract_key=VECTOR_GEOJSON_CONTRACT_KEY, + data_contract_version=VECTOR_GEOJSON_CONTRACT_VERSION, + kind=ContractKind.VECTOR, + source_crs=source_crs, + storage_crs=storage_crs, + bounds=contract_metadata.get("bounds_json"), + checksum_sha256=persisted_checksum_sha256, + computed_checksum_sha256=persisted_checksum_sha256, + metadata=contract_metadata, + # The partition-bounded audit above validates every + # source feature. The generic contract receives only + # compact aggregate geometry evidence and therefore + # cannot materialize the complete regional artifact. + geometry_records=(partition_audit.representative_record,), + source_registry_id=str(source_registry.id), + source_snapshot_id=str(source_snapshot.id), + lineage=lineage, + imported_at=datetime.now(timezone.utc), + observed_at=temporal["observed_at"], + valid_from=temporal["valid_from"], + valid_to=temporal["valid_to"], + temporal_unknown_reason=governed_source_metadata["temporal_unknown_reason"], + source_version=temporal["source_version"], + source_version_unknown_reason=governed_source_metadata[ + "source_version_unknown_reason" + ], + ) + ) + except AppError as exc: + report = DatasetService._failed_validation_report( + asset_id=ingest_key, + dataset_type="vector", + code=exc.code, + message=exc.message, + now=datetime.now(timezone.utc), + category="source_schema" if exc.code.startswith("SOURCE_SCHEMA") else "parser", + ) dataset = Dataset( id=dataset_id, project_id=project_id, @@ -706,65 +2639,131 @@ class DatasetService: dataset_type="vector", source=source, dataset_role=normalized_role, - source_name=source_name, + source_name=source_key, reference_layer_name=reference_layer_name if normalized_role == "reference" else None, - source_metadata=source_metadata, - provenance_metadata=provenance_metadata, + source_metadata=governed_source_metadata, + provenance_metadata=governed_provenance_metadata, imported_at=datetime.now(timezone.utc), **temporal, + ingest_key=ingest_key, storage_path=storage_info["storage_path"], original_filename=storage_info["original_filename"], stored_filename=storage_info["stored_filename"], content_type=storage_info["content_type"], size_bytes=storage_info["size_bytes"], - checksum_sha256=storage_info["checksum_sha256"], - crs=str(metadata.get("crs") or "EPSG:4326"), + checksum_sha256=persisted_checksum_sha256, + crs=storage_crs, bounds_json=metadata.get("bounds_json"), - metadata_json=metadata, - status="ready", + metadata_json=contract_metadata, + status="validating", ) + dataset_version = DatasetService._new_dataset_version(dataset, ingest_key=ingest_key) try: db.add(dataset) - db.add( - DatasetVersion( - dataset_id=dataset.id, - version=1, - storage_path=dataset.storage_path, - source_version=dataset.source_version, - observed_at=dataset.observed_at, - valid_from=dataset.valid_from, - checksum_sha256=dataset.checksum_sha256, - source_metadata=dataset.source_metadata, - provenance_metadata=dataset.provenance_metadata, - ) - ) - persisted_count = VectorFeatureService.persist_geojson_partitions( + db.add(dataset_version) + db.flush() + if report.validation_status == ValidationStatus.PASSED: + try: + begin_nested = getattr(db, "begin_nested", None) + if callable(begin_nested): + with begin_nested(): + persisted_count = VectorFeatureService.persist_geojson_partitions( + db, + dataset.id, + partition_paths, + feature_class=reference_layer_name if normalized_role == "reference" else None, + batch_size=batch_size, + source_crs=storage_crs, + ) + if persisted_count != expected_feature_count: + raise AppError( + code="PARTITION_FEATURE_COUNT_MISMATCH", + message=( + f"Regional artifact declares {expected_feature_count} features but " + f"{persisted_count} queryable features were indexed" + ), + status_code=400, + ) + else: # lightweight test sessions only; production uses a savepoint + persisted_count = VectorFeatureService.persist_geojson_partitions( + db, + dataset.id, + partition_paths, + feature_class=reference_layer_name if normalized_role == "reference" else None, + batch_size=batch_size, + source_crs=storage_crs, + ) + if persisted_count != expected_feature_count: + raise AppError( + code="PARTITION_FEATURE_COUNT_MISMATCH", + message=( + f"Regional artifact declares {expected_feature_count} features but " + f"{persisted_count} queryable features were indexed" + ), + status_code=400, + ) + except AppError as exc: + report = DatasetService._failed_validation_report( + asset_id=ingest_key, + dataset_type="vector", + code=exc.code, + message=exc.message, + now=datetime.now(timezone.utc), + ) + DatasetService._apply_validation_report( db, - dataset.id, - partition_paths, - feature_class=reference_layer_name if normalized_role == "reference" else None, - batch_size=batch_size, + dataset=dataset, + dataset_version=dataset_version, + report=report, + source=source_registry, + snapshot=source_snapshot, + artifact_path=str(storage_info["storage_path"]), ) - if persisted_count != expected_feature_count: - raise AppError( - code="PARTITION_FEATURE_COUNT_MISMATCH", - message=( - f"Regional artifact declares {expected_feature_count} features but " - f"{persisted_count} queryable features were indexed" - ), - status_code=400, - ) db.commit() db.refresh(dataset) except Exception: db.rollback() - StorageService.remove_dataset_file(storage_info["storage_path"]) + # Retain staged bytes for forensic review. A transaction error is + # not evidence that the source artifact may be safely discarded. raise return DatasetService._to_response(dataset) @staticmethod def refresh_metadata(db: Session, dataset_id: UUID) -> DatasetCreateResponse: dataset = DatasetService._get_dataset(db, dataset_id) + if dataset.quarantine_status == "quarantined" or dataset.status == "quarantined": + raise AppError( + code="DATASET_QUARANTINED", + message="Quarantined datasets cannot be refreshed into an eligible state; re-ingest a new governed snapshot.", + status_code=409, + ) + # A governed dataset's source snapshot and validation report bind the + # exact bytes, CRS and extracted metadata that were inspected at + # ingest. Re-reading a mutable storage path here would otherwise let + # an in-place replacement change the operational artifact while its + # persisted checksum/report still says ``passed``. Such a change must + # create a new immutable source snapshot through the governed ingest + # path; metadata refresh remains intentionally available only to rows + # without Phase-2 contract evidence. + has_governed_contract_evidence = any( + ( + dataset.source_registry_id is not None, + dataset.source_snapshot_id is not None, + bool(str(dataset.data_contract_key or "").strip()), + bool(str(dataset.data_contract_version or "").strip()), + dataset.validation_report_json is not None, + dataset.validation_status == "passed", + ) + ) + if has_governed_contract_evidence: + raise AppError( + code="GOVERNED_DATASET_REINGEST_REQUIRED", + message=( + "Governed dataset metadata is immutable evidence. Re-ingest the artifact to create a new " + "source snapshot and validation report." + ), + status_code=409, + ) if not dataset.storage_path: raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404) if not Path(dataset.storage_path).exists(): @@ -777,16 +2776,12 @@ class DatasetService: metadata = extract_raster_metadata(dataset.storage_path) else: raise AppError(code="INVALID_DATASET_TYPE", message="Cannot refresh metadata for this dataset type", status_code=400) - dataset.status = "ready" except ValueError as exc: - dataset.status = "failed" raise AppError(code="INVALID_GEOJSON", message=str(exc), status_code=400) from exc except AppError as exc: if DatasetService._is_raster_type(dataset.dataset_type) and exc.code == "RASTER_PROCESSING_UNAVAILABLE": - dataset.status = "failed" metadata = {"processing_error": exc.message, "processing_code": exc.code} else: - dataset.status = "failed" raise bounds_json = metadata.get("bounds_json") if isinstance(metadata, dict) else dataset.bounds_json @@ -797,9 +2792,15 @@ class DatasetService: resolution_json = DatasetService._extract_raster_resolution_json(metadata) bands_json = DatasetService._extract_raster_bands_json(metadata) - dataset.crs = metadata.get("crs") if isinstance(metadata, dict) else dataset.crs + # Metadata extraction is observational only. It must never turn an + # unvalidated historical row into a ready, authoritative dataset. + if DatasetService._is_vector_type(dataset.dataset_type): + dataset.crs = DatasetService.CANONICAL_VECTOR_CRS + else: + dataset.crs = metadata.get("crs") if isinstance(metadata, dict) else dataset.crs dataset.bounds_json = bounds_json - dataset.metadata_json = metadata + existing_metadata = dataset.metadata_json if isinstance(dataset.metadata_json, dict) else {} + dataset.metadata_json = {**existing_metadata, **metadata} dataset.resolution_json = resolution_json dataset.bands_json = bands_json @@ -812,6 +2813,12 @@ class DatasetService: @staticmethod def update_temporal_metadata(db: Session, dataset_id: UUID, payload: DatasetTemporalUpdate) -> DatasetCreateResponse: dataset = DatasetService._get_dataset(db, dataset_id) + if dataset.quarantine_status == "quarantined" or dataset.status == "quarantined": + raise AppError( + code="DATASET_QUARANTINED", + message="Quarantined datasets require a new governed ingest rather than an in-place temporal edit.", + status_code=409, + ) temporal = DatasetService._validate_temporal_metadata(**payload.model_dump()) if all(getattr(dataset, field) == value for field, value in temporal.items()): return DatasetService._to_response(dataset) @@ -819,6 +2826,15 @@ class DatasetService: for field, value in temporal.items(): setattr(dataset, field, value) + # Observation/source-version fields are contract inputs. Their edit + # invalidates the prior report, so later training/inference gates fail + # closed until a governed re-ingest persists a new snapshot/report. + dataset.status = "validating" + dataset.validation_status = "not_validated" + dataset.validation_report_json = None + dataset.provenance_status = "incomplete" + dataset.quarantine_status = "not_quarantined" + latest_version = ( db.query(DatasetVersion) .filter(DatasetVersion.dataset_id == dataset.id) @@ -836,8 +2852,21 @@ class DatasetService: valid_from=dataset.valid_from, valid_to=dataset.valid_to, checksum_sha256=dataset.checksum_sha256, + ingest_key=( + f"{dataset.ingest_key}:temporal:{(latest_version.version + 1) if latest_version else 1}" + if dataset.ingest_key + else None + ), source_metadata=dataset.source_metadata, provenance_metadata=dataset.provenance_metadata, + source_registry_id=dataset.source_registry_id, + source_snapshot_id=dataset.source_snapshot_id, + data_contract_key=dataset.data_contract_key, + data_contract_version=dataset.data_contract_version, + validation_status="not_validated", + validation_report_json=None, + provenance_status="incomplete", + lineage_status=dataset.lineage_status, ) ) db.commit() @@ -878,7 +2907,32 @@ class DatasetService: raw = load_dataset_text(dataset.storage_path) try: - return json.loads(raw) + payload = json.loads(raw) + metadata_value = getattr(dataset, "metadata_json", None) + metadata = metadata_value if isinstance(metadata_value, dict) else {} + provenance_value = getattr(dataset, "provenance_metadata", None) + provenance = provenance_value if isinstance(provenance_value, dict) else {} + canonical_evidence = provenance.get("canonical_consumption_artifact") + canonical_checksum = ( + canonical_evidence.get("checksum_sha256") + if isinstance(canonical_evidence, dict) + else metadata.get("canonical_artifact_checksum_sha256") + ) + # Post-normalization imports persist canonical bytes. Reapplying + # their original source CRS here would transform those coordinates + # a second time. Historical rows without this immutable binding + # retain the legacy read-time canonicalization behavior until they + # are re-ingested through the governed path. + source_crs = ( + DatasetService.CANONICAL_VECTOR_CRS + if canonical_checksum == getattr(dataset, "checksum_sha256", None) + else ( + metadata.get("source_crs") + or getattr(dataset, "crs", None) + or DatasetService.CANONICAL_VECTOR_CRS + ) + ) + return VectorFeatureService.canonicalize_geojson_payload(payload, source_crs=str(source_crs)) except Exception as exc: raise AppError(code="INVALID_GEOJSON", message="Stored dataset is not valid JSON", status_code=500) from exc @@ -931,6 +2985,12 @@ class DatasetService: @staticmethod def raster_metadata(db: Session, dataset_id: UUID) -> dict[str, Any]: dataset = DatasetService._get_dataset(db, dataset_id) + if dataset.quarantine_status == "quarantined" or dataset.status == "quarantined": + raise AppError( + code="DATASET_QUARANTINED", + message="Quarantined datasets cannot be read as production-ready raster metadata.", + status_code=409, + ) if not DatasetService._is_raster_type(dataset.dataset_type): raise AppError(code="INVALID_DATASET_TYPE", message="Dataset is not a raster dataset", status_code=400) if not dataset.storage_path: @@ -944,7 +3004,6 @@ class DatasetService: metadata = extract_raster_metadata(dataset.storage_path) dataset.metadata_json = dict(dataset.metadata_json or {}) dataset.metadata_json.update(metadata) - dataset.status = "ready" db.add(dataset) db.commit() db.refresh(dataset) diff --git a/backend/app/services/demo_workflow_service.py b/backend/app/services/demo_workflow_service.py index efa2dae4..309e90e8 100644 --- a/backend/app/services/demo_workflow_service.py +++ b/backend/app/services/demo_workflow_service.py @@ -12,6 +12,7 @@ from sqlalchemy.orm import Session from app.models import Area, Dataset, DatasetVersion, Metric, Project, QualityCheck from app.schemas.demo import DemoWorkflowResponse +from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService from app.services.geojson_service import parse_geojson_payload from app.services.qa_service import QaService from app.services.quality_service import QualityService @@ -30,17 +31,17 @@ class DemoWorkflowService: EXPECTED_METRICS_FILENAME = "expected_qa_metrics.json" @staticmethod - def _add_initial_version(db: Session, dataset: Dataset) -> None: - db.add( - DatasetVersion( - dataset_id=dataset.id, - version=1, - storage_path=dataset.storage_path, - checksum_sha256=dataset.checksum_sha256, - source_metadata=dataset.source_metadata, - provenance_metadata=dataset.provenance_metadata, - ) + def _add_initial_version(db: Session, dataset: Dataset) -> DatasetVersion: + version = DatasetVersion( + dataset_id=dataset.id, + version=1, + storage_path=dataset.storage_path, + checksum_sha256=dataset.checksum_sha256, + source_metadata=dataset.source_metadata, + provenance_metadata=dataset.provenance_metadata, ) + db.add(version) + return version @staticmethod def _repo_root() -> Path: @@ -233,24 +234,34 @@ class DemoWorkflowService: crs=metadata.get("crs"), bounds_json=metadata.get("bounds_json"), metadata_json=metadata, - status="ready", + status="validating", ) db.add(dataset) - DemoWorkflowService._add_initial_version(db, dataset) + version = DemoWorkflowService._add_initial_version(db, dataset) + is_ready = DerivedDatasetGovernanceService.govern_vector( + db, + dataset=dataset, + dataset_version=version, + feature_collection=payload, + source_key="fixture", + operation="demo.fixture_vector", + operation_parameters={"fixture_name": filename, "role": role}, + ) + if is_ready: + VectorFeatureService.persist_geojson_features( + db=db, + dataset_id=dataset.id, + payload=payload, + feature_class=reference_layer_name or "building", + commit=False, + ) db.commit() db.refresh(dataset) - VectorFeatureService.persist_geojson_features( - db=db, - dataset_id=dataset.id, - payload=payload, - feature_class=reference_layer_name or "building", - ) return dataset @staticmethod def _create_demo_raster_bytes() -> bytes: numpy = importlib.import_module("numpy") - rasterio = importlib.import_module("rasterio") rasterio_io = importlib.import_module("rasterio.io") rasterio_transform = importlib.import_module("rasterio.transform") @@ -318,10 +329,19 @@ class DemoWorkflowService: crs=metadata.get("crs"), bounds_json=bounds_json, metadata_json=metadata, - status="ready", + status="validating", ) db.add(dataset) - DemoWorkflowService._add_initial_version(db, dataset) + version = DemoWorkflowService._add_initial_version(db, dataset) + DerivedDatasetGovernanceService.govern_raster( + db, + dataset=dataset, + dataset_version=version, + raster_metadata=metadata, + source_key="fixture", + operation="demo.fixture_raster", + operation_parameters={"fixture_name": DemoWorkflowService.RASTER_FILENAME}, + ) db.commit() db.refresh(dataset) return dataset diff --git a/backend/app/services/derived_dataset_governance_service.py b/backend/app/services/derived_dataset_governance_service.py new file mode 100644 index 00000000..6d47adc8 --- /dev/null +++ b/backend/app/services/derived_dataset_governance_service.py @@ -0,0 +1,516 @@ +"""Governance for persisted derived and fixture datasets. + +Dataset importers own raw-source ingestion. This small service owns the +other persistence boundary: artifacts produced inside the workbench (vector +and raster operations) and the explicitly local demo fixtures. It is kept +separate from :mod:`dataset_service` so an operation can never create a ready +dataset without a source registry binding, immutable snapshot, validation +report and, for derived results, a durable lineage edge. + +The ``Session.query`` capability check deliberately preserves lightweight +unit-test doubles used by pre-Phase-2 tests. Real SQLAlchemy sessions always +take the governed branch; the compatibility branch is not reachable in the +application runtime. +""" + +from __future__ import annotations + +from datetime import datetime, timezone +from hashlib import sha256 +import json +import re +from typing import Any, Mapping + +from sqlalchemy.orm import Session + +from app.models import Dataset, DatasetVersion +from app.services.data_contract_validation import ( + IssueSeverity, + LineageStatus, + LineageEvidence, + ProvenanceStatus, + QuarantineStatus, + TransformationEvidence, + ValidationIssue, + ValidationReport, + ValidationStatus, + build_raster_ingest_input, + build_vector_ingest_input, + validate_registered_asset, +) +from app.services.data_quarantine_service import DataQuarantineService +from app.services.dataset_consumption_gate_service import DatasetConsumptionDecision, DatasetConsumptionGate +from app.services.source_registry_service import SourceRegistryService + + +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_CANONICAL_VECTOR_CRS = "EPSG:4326" + + +class DerivedDatasetGovernanceService: + """Apply Phase-2 provenance rules to non-importer Dataset creation. + + Callers add the dataset and first immutable version, then call one of the + ``govern_*`` methods before committing. A failed contract deliberately + leaves the artifact and its durable quarantine record in the transaction; + it is never silently promoted to ``ready``. + """ + + @staticmethod + def persistence_available(db: Session) -> bool: + """Return whether this is a real ORM persistence session. + + Historical unit tests use minimal fakes with ``add``/``commit`` only. + Keeping that explicitly isolated avoids pretending a fake test store + has source-registry guarantees while production remains fail-closed. + """ + + return callable(getattr(db, "query", None)) and callable(getattr(db, "flush", None)) + + @classmethod + def govern_vector( + cls, + db: Session, + *, + dataset: Dataset, + dataset_version: DatasetVersion, + feature_collection: Mapping[str, Any], + source_key: str, + operation: str, + parent_dataset: Dataset | None = None, + operation_parameters: Mapping[str, Any] | None = None, + ) -> bool: + """Validate and bind a vector result, returning ``True`` when ready.""" + + if not cls.persistence_available(db): + # Explicit compatibility for historical minimal test fixtures. + # Production sessions always have query/flush and never take this + # branch. + dataset.status = "ready" + return True + + parent_gate = cls._parent_derived_processing_gate(parent_dataset) + metadata = dict(dataset.metadata_json or {}) + output_crs = str(metadata.get("crs") or dataset.crs or _CANONICAL_VECTOR_CRS) + source = SourceRegistryService.ensure_server_owned_source(db, source_key) + snapshot = cls._record_snapshot( + db, + source_key=source_key, + dataset=dataset, + operation=operation, + source_crs=output_crs, + spatial_resolution=metadata.get("resolution_json") or {"status": "not_applicable"}, + geographic_coverage={"bounds": metadata.get("bounds_json") or dataset.bounds_json}, + observed_schema={ + "dataset_type": "vector", + "geometry_types": metadata.get("geometry_types") or [], + "feature_count": metadata.get("feature_count"), + }, + ) + lineage = cls._lineage_evidence(parent_dataset, operation, operation_parameters) + report = validate_registered_asset( + build_vector_ingest_input( + asset_id=str(dataset.id), + source_crs=output_crs, + storage_crs=output_crs, + feature_collection=feature_collection, + checksum_sha256=dataset.checksum_sha256, + computed_checksum_sha256=dataset.checksum_sha256, + source_registry_id=str(source.id), + source_snapshot_id=str(snapshot.id), + imported_at=dataset.imported_at or datetime.now(timezone.utc), + metadata=cls._contract_metadata(metadata, source.license_name), + observed_at=dataset.observed_at, + valid_from=dataset.valid_from, + valid_to=dataset.valid_to, + temporal_unknown_reason=cls._temporal_unknown_reason(dataset), + source_version=dataset.source_version, + source_version_unknown_reason=cls._source_version_unknown_reason(dataset), + lineage=lineage, + ) + ) + if parent_gate is not None and not parent_gate.eligible: + report = cls._with_parent_gate_failure(report, parent_gate) + return cls._apply( + db, + dataset=dataset, + dataset_version=dataset_version, + source=source, + snapshot=snapshot, + report=report, + stage="derived_vector_validation", + parent_dataset=parent_dataset, + operation=operation, + operation_parameters=operation_parameters, + ) + + @classmethod + def govern_raster( + cls, + db: Session, + *, + dataset: Dataset, + dataset_version: DatasetVersion, + raster_metadata: Mapping[str, Any], + source_key: str, + operation: str, + parent_dataset: Dataset | None = None, + operation_parameters: Mapping[str, Any] | None = None, + ) -> bool: + """Validate and bind a raster result, returning ``True`` when ready.""" + + if not cls.persistence_available(db): + dataset.status = "ready" + return True + + parent_gate = cls._parent_derived_processing_gate(parent_dataset) + metadata = dict(raster_metadata or {}) + output_crs = str(metadata.get("crs") or dataset.crs or "") or None + source = SourceRegistryService.ensure_server_owned_source(db, source_key) + snapshot = cls._record_snapshot( + db, + source_key=source_key, + dataset=dataset, + operation=operation, + source_crs=output_crs, + spatial_resolution=cls._raster_resolution(metadata, output_crs), + geographic_coverage={"bounds": metadata.get("bounds") or dataset.bounds_json}, + observed_schema={ + "dataset_type": "raster", + "width": metadata.get("width"), + "height": metadata.get("height"), + "band_count": metadata.get("band_count"), + "dtype": metadata.get("dtype"), + }, + ) + lineage = cls._lineage_evidence(parent_dataset, operation, operation_parameters) + report = validate_registered_asset( + build_raster_ingest_input( + asset_id=str(dataset.id), + source_crs=output_crs, + storage_crs=output_crs, + raster_profile=metadata, + bounds=metadata.get("bounds") or dataset.bounds_json, + resolution=cls._raster_resolution(metadata, output_crs), + checksum_sha256=dataset.checksum_sha256, + computed_checksum_sha256=dataset.checksum_sha256, + source_registry_id=str(source.id), + source_snapshot_id=str(snapshot.id), + imported_at=dataset.imported_at or datetime.now(timezone.utc), + metadata=cls._contract_metadata(metadata, source.license_name), + observed_at=dataset.observed_at, + valid_from=dataset.valid_from, + valid_to=dataset.valid_to, + temporal_unknown_reason=cls._temporal_unknown_reason(dataset), + source_version=dataset.source_version, + source_version_unknown_reason=cls._source_version_unknown_reason(dataset), + lineage=lineage, + ) + ) + if parent_gate is not None and not parent_gate.eligible: + report = cls._with_parent_gate_failure(report, parent_gate) + return cls._apply( + db, + dataset=dataset, + dataset_version=dataset_version, + source=source, + snapshot=snapshot, + report=report, + stage="derived_raster_validation", + parent_dataset=parent_dataset, + operation=operation, + operation_parameters=operation_parameters, + ) + + @classmethod + def _record_snapshot( + cls, + db: Session, + *, + source_key: str, + dataset: Dataset, + operation: str, + source_crs: str | None, + spatial_resolution: Mapping[str, Any] | None, + geographic_coverage: Mapping[str, Any] | None, + observed_schema: Mapping[str, Any] | None, + ): + checksum = cls._checksum_or_placeholder(dataset.checksum_sha256) + # A derived artifact has a distinct creation event even when its bytes + # equal a prior output. Include the immutable dataset id so source + # snapshots never collide on a different fetched_at timestamp. + snapshot_key = f"{source_key}:{operation}:{dataset.id}:{checksum}" + return SourceRegistryService.record_snapshot( + db, + source_key=source_key, + snapshot_key=snapshot_key, + checksum_sha256=checksum, + source_version=dataset.source_version or f"{operation}:1.0.0", + snapshot_at=dataset.observed_at, + fetched_at=dataset.imported_at or datetime.now(timezone.utc), + crs=source_crs, + units=cls._units_for_crs(source_crs), + spatial_resolution=dict(spatial_resolution or {"status": "unknown"}), + temporal_coverage={ + "observed_at": cls._datetime_value(dataset.observed_at), + "valid_from": cls._datetime_value(dataset.valid_from), + "valid_to": cls._datetime_value(dataset.valid_to), + }, + geographic_coverage=dict(geographic_coverage or {"status": "unknown"}), + observed_schema=dict(observed_schema or {"status": "unknown"}), + # A transform cannot establish source freshness. With no source + # observation it is not applicable; with one it still needs an + # explicit policy review rather than a fabricated "current" flag. + freshness_status="not_applicable" if dataset.observed_at is None else "review_required", + ingest_status="ingested", + known_limitations=[ + "Derived and fixture artifacts inherit no automatic source authority beyond their explicit registry entry and lineage.", + ], + snapshot_metadata={ + "operation": operation, + "dataset_id": str(dataset.id), + "storage_path": dataset.storage_path, + "artifact_checksum_sha256": dataset.checksum_sha256, + }, + ) + + @classmethod + def _apply( + cls, + db: Session, + *, + dataset: Dataset, + dataset_version: DatasetVersion, + source: Any, + snapshot: Any, + report: ValidationReport, + stage: str, + parent_dataset: Dataset | None, + operation: str, + operation_parameters: Mapping[str, Any] | None, + ) -> bool: + fields = report.persistence_fields() + dataset.validation_report_json = fields["validation_report_json"] + dataset_version.validation_report_json = fields["validation_report_json"] + SourceRegistryService.bind_dataset_provenance( + dataset, + source=source, + snapshot=snapshot, + data_contract_key=fields["data_contract_key"], + data_contract_version=fields["data_contract_version"], + validation_status=fields["validation_status"], + provenance_status=fields["provenance_status"], + lineage_status=fields["lineage_status"], + ) + SourceRegistryService.bind_dataset_version_provenance( + dataset_version, + source=source, + snapshot=snapshot, + data_contract_key=fields["data_contract_key"], + data_contract_version=fields["data_contract_version"], + validation_status=fields["validation_status"], + provenance_status=fields["provenance_status"], + lineage_status=fields["lineage_status"], + ) + + # IDs for DatasetVersion defaults exist only after the caller adds and + # flushes both rows. The operation services call this before commit. + db.flush() + if parent_dataset is not None: + SourceRegistryService.record_lineage_edge( + db, + parent_dataset_id=parent_dataset.id, + child_dataset_id=dataset.id, + parent_dataset_version_id=cls._latest_parent_version_id(db, parent_dataset), + child_dataset_version_id=dataset_version.id, + relation_type="derived_from", + transformation_name=operation, + transformation_version="1.0.0", + parameters=dict(operation_parameters or {}), + input_checksum_sha256=cls._valid_checksum(parent_dataset.checksum_sha256), + output_checksum_sha256=cls._valid_checksum(dataset.checksum_sha256), + ) + + decision = DataQuarantineService.decide(report) + if decision.eligible_for_use: + dataset.status = "ready" + dataset.quarantine_status = "not_quarantined" + return True + + SourceRegistryService.quarantine_dataset( + db, + dataset=dataset, + dataset_version=dataset_version, + source_snapshot=snapshot, + stage=stage, + reason_code=(decision.reason_codes[0] if decision.reason_codes else "DATA_CONTRACT_FAILED"), + details={"validation_report": report.to_dict(), "quarantine_decision": decision.to_dict()}, + artifact_path=dataset.storage_path, + artifact_checksum_sha256=cls._valid_checksum(dataset.checksum_sha256), + ) + return False + + @staticmethod + def _contract_metadata(metadata: Mapping[str, Any], license_name: str) -> dict[str, Any]: + values = dict(metadata) + values.setdefault("license", license_name) + return values + + @classmethod + def _lineage_evidence( + cls, + parent_dataset: Dataset | None, + operation: str, + operation_parameters: Mapping[str, Any] | None, + ) -> LineageEvidence: + upstream_ids: tuple[str, ...] = () + upstream_checksums: tuple[str, ...] = () + if parent_dataset is not None: + upstream_ids = (str(parent_dataset.id),) + # An invalid/missing parent checksum intentionally fails the + # derived contract rather than inventing traceability. The same + # applies to a legacy/unvalidated parent: it may remain visible + # as evidence, but cannot create a new ready derived asset. + parent_is_governed = ( + parent_dataset.status == "ready" + and parent_dataset.quarantine_status == "not_quarantined" + and parent_dataset.validation_status == "passed" + and parent_dataset.provenance_status == "complete" + and parent_dataset.source_registry_id is not None + and parent_dataset.source_snapshot_id is not None + ) + upstream_checksums = ( + parent_dataset.checksum_sha256 if parent_is_governed else "parent_dataset_not_governed", + ) + transform_checksum = cls._stable_hash( + {"operation": operation, "version": "1.0.0", "parameters": dict(operation_parameters or {})} + ) + return LineageEvidence( + upstream_asset_ids=upstream_ids, + upstream_checksums_sha256=upstream_checksums, + transformations=( + TransformationEvidence( + name=operation, + version="1.0.0", + checksum_sha256=transform_checksum, + ), + ), + ) + + @staticmethod + def _parent_derived_processing_gate(parent_dataset: Dataset | None) -> DatasetConsumptionDecision | None: + """Evaluate the durable parent boundary before creating a ready child. + + We deliberately turn a rejected parent into a child validation failure + (instead of simply raising): the output is then persisted with its + source snapshot, validation evidence and durable quarantine record. + This makes an attempted derivation from a manual or experimental + dataset observable and prevents a caller from bypassing the boundary + by invoking the governance service directly. + """ + + if parent_dataset is None: + return None + return DatasetConsumptionGate.evaluate(parent_dataset, purpose="derived_processing") + + @staticmethod + def _with_parent_gate_failure( + report: ValidationReport, + decision: DatasetConsumptionDecision, + ) -> ValidationReport: + """Attach an auditable, fail-closed lineage failure to a report.""" + + issue = ValidationIssue( + code="PARENT_DATASET_NOT_ELIGIBLE_FOR_DERIVED_PROCESSING", + category="lineage", + field="lineage.parent_dataset", + message="Parent dataset failed the governed derived-processing consumption gate.", + severity=IssueSeverity.ERROR, + expected="eligible governed parent dataset", + observed={ + "dataset_id": decision.evidence.get("dataset_id"), + "reasons": list(decision.reasons), + "source_key": decision.evidence.get("source_key"), + "source_classification": decision.evidence.get("source_classification"), + }, + ) + return ValidationReport( + asset_id=report.asset_id, + data_contract_key=report.data_contract_key, + data_contract_version=report.data_contract_version, + contract_fingerprint_sha256=report.contract_fingerprint_sha256, + validation_status=ValidationStatus.FAILED, + provenance_status=( + ProvenanceStatus.INCOMPLETE + if report.provenance_status == ProvenanceStatus.COMPLETE + else report.provenance_status + ), + lineage_status=LineageStatus.INCOMPLETE, + quarantine_status=QuarantineStatus.QUARANTINED, + validation_scope=report.validation_scope, + checked_at=report.checked_at, + issues=(*report.issues, issue), + ) + + @staticmethod + def _latest_parent_version_id(db: Session, dataset: Dataset): + version = ( + db.query(DatasetVersion) + .filter(DatasetVersion.dataset_id == dataset.id) + .order_by(DatasetVersion.version.desc()) + .first() + ) + return version.id if version is not None else None + + @staticmethod + def _raster_resolution(metadata: Mapping[str, Any], crs: str | None) -> dict[str, Any] | None: + values = metadata.get("resolution") + if not isinstance(values, (list, tuple)) or len(values) < 2: + return None + try: + return {"x": abs(float(values[0])), "y": abs(float(values[1])), "unit": DerivedDatasetGovernanceService._resolution_unit(crs)} + except (TypeError, ValueError): + return None + + @staticmethod + def _resolution_unit(crs: str | None) -> str: + return "degree" if str(crs or "").upper() == _CANONICAL_VECTOR_CRS else "m" + + @staticmethod + def _units_for_crs(crs: str | None) -> str: + return "degrees" if str(crs or "").upper() == _CANONICAL_VECTOR_CRS else "metres" + + @staticmethod + def _temporal_unknown_reason(dataset: Dataset) -> str | None: + if dataset.observed_at is not None: + return None + return "Derived or fixture artifact inherits no precise observation timestamp from its input." + + @staticmethod + def _source_version_unknown_reason(dataset: Dataset) -> str | None: + if dataset.source_version: + return None + return "Derived or fixture artifact has no source edition; the transform version is recorded separately." + + @staticmethod + def _datetime_value(value: datetime | None) -> str | None: + return value.astimezone(timezone.utc).isoformat() if value is not None else None + + @staticmethod + def _stable_hash(value: Mapping[str, Any]) -> str: + return sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")).hexdigest() + + @staticmethod + def _valid_checksum(value: str | None) -> str | None: + normalized = str(value or "").strip().lower() + return normalized if _SHA256.fullmatch(normalized) else None + + @classmethod + def _checksum_or_placeholder(cls, value: str | None) -> str: + checksum = cls._valid_checksum(value) + if checksum is not None: + return checksum + # The contract receives the original invalid/missing checksum and + # quarantines it. A deterministic placeholder only permits storing + # the rejected snapshot without fabricating a valid artifact hash. + return cls._stable_hash({"invalid_dataset_checksum": value or "missing"}) diff --git a/backend/app/services/detection_service.py b/backend/app/services/detection_service.py index e4b9a22a..ea5d1c4b 100644 --- a/backend/app/services/detection_service.py +++ b/backend/app/services/detection_service.py @@ -19,10 +19,12 @@ from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project, Vect from app.schemas.detection import DetectionListResponse, DetectionRead, DetectionRunListResponse, DetectionRunRead, DetectionRunResponse from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon from app.services.detection_qa_service import DetectionQaService +from app.services.dataset_consumption_gate_service import DatasetConsumptionGate from app.services.model_asset_catalog_service import ModelAssetCatalogService from app.services.model_registry_service import ModelRegistryService from app.services.qa_service import QaService from app.services.quality_service import QualityService +from app.services.runtime_model_provenance_service import RuntimeModelProvenance, RuntimeModelProvenanceService from app.services.temporal_compatibility_service import TemporalCompatibilityService from app.services.yolo_adapter import YoloDetectionAdapter @@ -97,6 +99,18 @@ class DetectionService: if model.model_id == resolved_settings.yolo_model_id and resolved_settings.yolo_enforce_validation_scope: DetectionService._validate_model_area_scope(db, dataset, resolved_settings) + # Never enter a production inference path with a persisted dataset + # that has failed validation, incomplete provenance, or an active + # quarantine. Fixture detection is a separate QA/test-only path. + if model.model_id == "manual-fixture-detector": + DatasetConsumptionGate.assert_eligible( + dataset, + purpose="quality_assessment", + fixture_mode=True, + ) + elif model.model_id == resolved_settings.yolo_model_id and model.configured: + DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference") + run_parameters = { "model_id": model.model_id, "model_asset_id": selected_model_asset.model_asset_id if selected_model_asset else None, @@ -352,15 +366,6 @@ class DetectionService: reference_dataset, ) - detections = DetectionService._query_detection_rows( - db, - analysis_run_id=analysis_run_id, - dataset_id=run.dataset_id, - class_name=class_name, - min_confidence=min_confidence, - ) - raw_candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in detections] - candidate_geometries = raw_candidate_geometries run_parameters = run.parameters_json if isinstance(run.parameters_json, dict) else {} manifest_path = DetectionQaService.tile_manifest_path(run_parameters) resolved_settings = get_settings() @@ -375,6 +380,33 @@ class DetectionService: status_code=422, ) + fixture_parameters = run_parameters.get("parameters_json") + fixture_mode = bool( + run.model_name == "manual-fixture-detector" + and isinstance(fixture_parameters, dict) + and fixture_parameters.get("fixture_mode") is True + ) + DatasetConsumptionGate.assert_eligible( + candidate_dataset, + purpose="quality_assessment", + fixture_mode=fixture_mode, + ) + DatasetConsumptionGate.assert_eligible( + reference_dataset, + purpose="reference_validation", + reference_task="building_validation", + ) + + detections = DetectionService._query_detection_rows( + db, + analysis_run_id=analysis_run_id, + dataset_id=run.dataset_id, + class_name=class_name, + min_confidence=min_confidence, + ) + raw_candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in detections] + candidate_geometries = raw_candidate_geometries + coverage = None if manifest_path: manifest = DetectionService._load_tile_manifest(manifest_path, resolved_settings.yolo_max_tiles) @@ -728,6 +760,19 @@ class DetectionService: ) -> tuple[list[Detection], dict[str, Any]]: manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles) model_path = Path(settings.yolo_model_path or "").expanduser() + runtime_model_provenance = RuntimeModelProvenanceService.validate_for_production_runtime( + db=db, + model_path=model_path, + model_id=model_name, + task_type="object_detection", + expected_model_version=model_version, + allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"), + ) + DetectionService._attach_runtime_model_provenance( + analysis_run, + job, + runtime_model_provenance, + ) adapter = yolo_adapter_class(settings) model = adapter.load_model(model_path) allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)} @@ -785,7 +830,10 @@ class DetectionService: "y_max": float(bbox[3]), }, source_tile_path=candidate["source_tile_path"], - properties_json=candidate["properties"], + properties_json={ + **candidate["properties"], + "runtime_model_provenance": runtime_model_provenance.as_dict(), + }, ) db.add(detection) persisted.append(detection) @@ -796,8 +844,30 @@ class DetectionService: "raw_detection_count": len(candidates), "suppressed_detection_count": len(candidates) - len(filtered_candidates), "duplicate_iou_threshold": float(settings.yolo_duplicate_iou_threshold), + "runtime_model_provenance": runtime_model_provenance.as_dict(), } + @staticmethod + def _attach_runtime_model_provenance( + analysis_run: AnalysisRun, + job: Job, + provenance: RuntimeModelProvenance, + ) -> None: + """Persist byte-bound model evidence with the run before adapter loading. + + Individual detections retain the same evidence in ``properties_json``; + this run-level copy is the compact audit root for a complete inference. + Assigning fresh dictionaries matters for SQLAlchemy JSON change tracking. + """ + + evidence = provenance.as_dict() + analysis_parameters = dict(analysis_run.parameters_json or {}) + analysis_parameters["runtime_model_provenance"] = evidence + analysis_run.parameters_json = analysis_parameters + job_parameters = dict(job.parameters_json or {}) + job_parameters["runtime_model_provenance"] = evidence + job.parameters_json = job_parameters + @staticmethod def _canonical_class_name(value: Any) -> str: return str(value or "").strip().casefold() diff --git a/backend/app/services/export_service.py b/backend/app/services/export_service.py index 9cd99f35..7ae60683 100644 --- a/backend/app/services/export_service.py +++ b/backend/app/services/export_service.py @@ -23,6 +23,7 @@ from app.schemas.flood_hazard import FloodHazardPartitionSelectionRequest, Flood from app.schemas.temporal import TemporalComparisonRequest from app.schemas.thematic_raster import ThematicRasterSelectionRequest from app.services.dataset_service import DatasetService +from app.services.dataset_consumption_gate_service import DatasetConsumptionGate from app.services.detection_service import DetectionService from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService from app.services.segmentation_service import SegmentationService @@ -34,12 +35,39 @@ from app.services.vector_feature_service import VectorFeatureService class ExportService: + @staticmethod + def _assert_run_source_dataset_exportable(db: Session, run: AnalysisRun) -> Dataset: + """Block an output export when its persisted source dataset is unsafe.""" + + if not run.dataset_id: + raise AppError( + code="DATASET_PROVENANCE_INCOMPLETE", + message="Analysis output cannot be exported without a persisted source dataset.", + status_code=409, + ) + dataset = db.get(Dataset, run.dataset_id) + if not dataset or dataset.project_id != run.project_id: + raise AppError(code="DATASET_NOT_FOUND", message="Analysis source dataset not found", status_code=404) + DatasetConsumptionGate.assert_eligible(dataset, purpose="export") + return dataset + @staticmethod def export_map_result( db: Session, payload: MapResultExportRequest, ) -> ExportCreateResponse: if payload.mode == "evolution": + earlier_dataset = db.get(Dataset, payload.earlier_dataset_id) + later_dataset = db.get(Dataset, payload.later_dataset_id) + if ( + not earlier_dataset + or not later_dataset + or earlier_dataset.project_id != payload.project_id + or later_dataset.project_id != payload.project_id + ): + raise AppError(code="DATASET_NOT_FOUND", message="Temporal export dataset not found", status_code=404) + DatasetConsumptionGate.assert_eligible(earlier_dataset, purpose="export") + DatasetConsumptionGate.assert_eligible(later_dataset, purpose="export") comparison = TemporalAnalysisService.compare( db, project_id=payload.project_id, @@ -86,6 +114,7 @@ class ExportService: dataset = db.get(Dataset, payload.dataset_id) if not dataset or dataset.project_id != payload.project_id: raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404) + DatasetConsumptionGate.assert_eligible(dataset, purpose="export") if dataset.dataset_type in DatasetService.VECTOR_TYPES: if payload.partitioned: return ExportService.export_partitioned_vector_selection_geojson( @@ -219,6 +248,7 @@ class ExportService: limit: int = 1000, name: str | None = None, ) -> ExportCreateResponse: + DatasetConsumptionGate.assert_eligible(dataset, purpose="export") if dataset.source_name != "vmm_vha_bathymetry_profiles" or partition_scope_key != "flanders": raise AppError( code="PARTITIONED_VECTOR_EXPORT_UNSUPPORTED", @@ -308,6 +338,7 @@ class ExportService: details={"dataset_type": dataset.dataset_type}, status_code=400, ) + DatasetConsumptionGate.assert_eligible(dataset, purpose="export") selection_kwargs: dict[str, Any] = { "dataset_id": dataset_id, @@ -374,6 +405,7 @@ class ExportService: details={"dataset_type": dataset.dataset_type}, status_code=400, ) + DatasetConsumptionGate.assert_eligible(dataset, purpose="export") feature_collection = DatasetService.get_dataset_geojson(db, dataset_id) filename = ExportService._filename(name, f"{dataset.id}.geojson", ".geojson") @@ -401,6 +433,7 @@ class ExportService: run = db.get(AnalysisRun, analysis_run_id) if not run or run.analysis_type != "detection": raise AppError(code="DETECTION_RUN_NOT_FOUND", message="Detection run not found", status_code=404) + ExportService._assert_run_source_dataset_exportable(db, run) feature_collection = DetectionService.detections_to_geojson(db, analysis_run_id=analysis_run_id) filename = ExportService._filename(name, f"{run.id}-detections.geojson", ".geojson") @@ -428,6 +461,7 @@ class ExportService: run = db.get(AnalysisRun, analysis_run_id) if not run or run.analysis_type != "segmentation": raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404) + ExportService._assert_run_source_dataset_exportable(db, run) feature_collection = SegmentationService.segmentations_to_geojson(db, analysis_run_id=analysis_run_id) filename = ExportService._filename(name, f"{run.id}-segmentations.geojson", ".geojson") diff --git a/backend/app/services/model_registry_service.py b/backend/app/services/model_registry_service.py index b4db00a5..ecd25aa3 100644 --- a/backend/app/services/model_registry_service.py +++ b/backend/app/services/model_registry_service.py @@ -9,6 +9,7 @@ from app.services.segmentation_adapter import ( SamSegmentationAdapter, YoloSegmentationAdapter, ) +from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService from app.services.yolo_adapter import YoloDetectionAdapter from app.core.errors import AppError @@ -144,9 +145,24 @@ class ModelRegistryService: elif not model_path.exists() or not model_path.is_file(): limitation = "YOLO_SEG_MODEL_PATH does not point to an existing local model file. GeoIntel will not download segmentation model weights automatically." else: - configured = True - status = "configured" - limitation = "Configured for local YOLO segmentation inference over an existing raster tile manifest." + try: + RuntimeModelProvenanceService.validate_for_runtime( + model_path=model_path, + model_id=settings.yolo_seg_model_id, + task_type="segmentation", + expected_model_version=settings.yolo_seg_model_version, + allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"), + ) + except AppError as exc: + status = "contract_incomplete" + limitation = ( + "Configured YOLO segmentation weights are not runnable until their immutable " + f"runtime provenance sidecar validates: {exc.message}" + ) + else: + configured = True + status = "configured" + limitation = "Configured for local YOLO segmentation inference over an existing raster tile manifest." return DetectionModelCapability( model_id=settings.yolo_seg_model_id, @@ -186,9 +202,24 @@ class ModelRegistryService: elif not model_path.exists() or not model_path.is_file(): limitation = "SAM_MODEL_PATH does not point to an existing local model file. GeoIntel will not download segmentation model weights automatically." else: - configured = True - status = "configured" - limitation = "Configured for local class-agnostic SAM segmentation over an existing raster tile manifest." + try: + RuntimeModelProvenanceService.validate_for_runtime( + model_path=model_path, + model_id=settings.sam_model_id, + task_type="segmentation", + expected_model_version=settings.sam_model_version, + allowed_frameworks=("ultralytics/sam", "sam", "ultralytics", "pytorch"), + ) + except AppError as exc: + status = "contract_incomplete" + limitation = ( + "Configured SAM weights are not runnable until their immutable runtime provenance " + f"sidecar validates: {exc.message}" + ) + else: + configured = True + status = "configured" + limitation = "Configured for local class-agnostic SAM segmentation over an existing raster tile manifest." return DetectionModelCapability( model_id=settings.sam_model_id, @@ -233,9 +264,24 @@ class ModelRegistryService: status = "accelerator_unavailable" limitation = exc.message else: - configured = True - status = "configured" - limitation = "Configured for local YOLO inference over an existing raster tile manifest within its validated area scope." + try: + RuntimeModelProvenanceService.validate_for_runtime( + model_path=model_path, + model_id=settings.yolo_model_id, + task_type="object_detection", + expected_model_version=settings.yolo_model_version, + allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"), + ) + except AppError as exc: + status = "contract_incomplete" + limitation = ( + "Configured YOLO weights are not runnable until their immutable runtime provenance " + f"sidecar validates: {exc.message}" + ) + else: + configured = True + status = "configured" + limitation = "Configured for local YOLO inference over an existing raster tile manifest within its validated area scope." return DetectionModelCapability( model_id=settings.yolo_model_id, diff --git a/backend/app/services/qa_service.py b/backend/app/services/qa_service.py index c0528ea4..864456de 100644 --- a/backend/app/services/qa_service.py +++ b/backend/app/services/qa_service.py @@ -11,10 +11,10 @@ from shapely.geometry.base import BaseGeometry from shapely.strtree import STRtree from shapely.ops import unary_union from shapely.validation import make_valid -from shapely.geometry import shape from app.core.errors import AppError from app.models import Area, Dataset from app.schemas.qa import QaProviderComparisonResult +from app.services.dataset_consumption_gate_service import DatasetConsumptionGate from app.services.vector_operations_service import VectorOperationsService @@ -264,6 +264,12 @@ class QaService: reference_dataset_id, expected_project_id=project_id, ) + DatasetConsumptionGate.assert_eligible(candidate_dataset, purpose="quality_assessment") + DatasetConsumptionGate.assert_eligible( + reference_dataset, + purpose="reference_validation", + reference_task="building_validation", + ) area_geometry = QaService._validate_area( db, diff --git a/backend/app/services/raster_operations_service.py b/backend/app/services/raster_operations_service.py index b72b8b75..bf8f49ba 100644 --- a/backend/app/services/raster_operations_service.py +++ b/backend/app/services/raster_operations_service.py @@ -13,6 +13,7 @@ from shapely.validation import make_valid from app.core.errors import AppError from app.models import Area, Dataset, DatasetVersion +from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService from app.services.raster_service import extract_raster_metadata from app.services.storage_service import StorageService @@ -334,7 +335,10 @@ class RasterOperationsService: dataset_type="raster", source=f"operation:{operation_name}", dataset_role="derived", - source_name=source_dataset.source_name, + # The source identity of this artifact is the server-owned + # derived-operation registry entry. The parent remains explicit + # in provenance and the lineage edge below. + source_name="derived", source_metadata=source_dataset.source_metadata, provenance_metadata=provenance, imported_at=datetime.now(timezone.utc), @@ -360,22 +364,31 @@ class RasterOperationsService: resolution_json=metadata_payload.get("resolution"), bands_json={"dtype": metadata_payload.get("dtype")} if metadata_payload.get("dtype") is not None else None, metadata_json=metadata_payload, - status="ready", + status="validating", ) db.add(derived_dataset) - db.add( - DatasetVersion( - dataset_id=derived_dataset.id, - version=1, - storage_path=derived_dataset.storage_path, - source_version=derived_dataset.source_version, - observed_at=derived_dataset.observed_at, - valid_from=derived_dataset.valid_from, - valid_to=derived_dataset.valid_to, - checksum_sha256=derived_dataset.checksum_sha256, - source_metadata=derived_dataset.source_metadata, - provenance_metadata=derived_dataset.provenance_metadata, - ) + dataset_version = DatasetVersion( + dataset_id=derived_dataset.id, + version=1, + storage_path=derived_dataset.storage_path, + source_version=derived_dataset.source_version, + observed_at=derived_dataset.observed_at, + valid_from=derived_dataset.valid_from, + valid_to=derived_dataset.valid_to, + checksum_sha256=derived_dataset.checksum_sha256, + source_metadata=derived_dataset.source_metadata, + provenance_metadata=derived_dataset.provenance_metadata, + ) + db.add(dataset_version) + DerivedDatasetGovernanceService.govern_raster( + db, + dataset=derived_dataset, + dataset_version=dataset_version, + raster_metadata=metadata_payload, + source_key="derived", + operation=operation_name, + parent_dataset=source_dataset, + operation_parameters=metadata_payload.get("operation_parameters"), ) db.commit() db.refresh(derived_dataset) diff --git a/backend/app/services/runtime_model_provenance_service.py b/backend/app/services/runtime_model_provenance_service.py new file mode 100644 index 00000000..4710ec64 --- /dev/null +++ b/backend/app/services/runtime_model_provenance_service.py @@ -0,0 +1,652 @@ +"""Fail-closed provenance validation for configured local model files. + +Configured YOLO and SAM weights are intentionally not trusted merely because a +file exists on the server. Before an adapter is allowed to load local weights, +this service verifies a neighbouring immutable sidecar manifest, validates the +model artifact against ``geointel.model.pytorch@1.0.0`` and binds that sidecar +to the server-owned source registry and source snapshot recorded in Postgres. + +``validate_for_runtime`` remains a structural sidecar check for catalog and +preflight inspection. Production inference must call +``validate_for_production_runtime`` with a database session; it rejects an +unregistered, mismatched, stale, unsafe or quarantined source snapshot before +an adapter can load model bytes. This keeps focused unit tests able to inspect +sidecars without inventing database rows while keeping the production boundary +strict. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from hashlib import sha256 +import json +from pathlib import Path +import re +from typing import Any, Mapping +from uuid import UUID + +from app.core.errors import AppError +from app.models import SourceRegistry, SourceSnapshot +from app.services.data_contract_validation import ( + PYTORCH_MODEL_CONTRACT_KEY, + PYTORCH_MODEL_CONTRACT_VERSION, + LineageEvidence, + TransformationEvidence, + build_model_validation_input, + validate_registered_asset, +) + + +_SHA256 = re.compile(r"^[0-9a-f]{64}$") +_CONSUMABLE_FRESHNESS = {"current", "not_applicable"} +_SAFE_SOURCE_REGISTRY_INGEST_STATUSES = {"configured", "ingested"} +_SAFE_SOURCE_SNAPSHOT_INGEST_STATUS = "ingested" + + +@dataclass(frozen=True) +class RuntimeModelProvenance: + """Validated, immutable evidence attached to one local inference run.""" + + model_id: str + task_type: str + model_path: str + manifest_path: str + model_sha256: str + manifest_sha256: str + runtime_manifest_sha256: str + data_contract_key: str + data_contract_version: str + validation_report_sha256: str + source_registry_id: str + source_snapshot_id: str + source_snapshot_checksum_sha256: str + source_version: str + + def as_dict(self) -> dict[str, str]: + return { + "model_id": self.model_id, + "task_type": self.task_type, + "model_path": self.model_path, + "manifest_path": self.manifest_path, + "model_sha256": self.model_sha256, + "manifest_sha256": self.manifest_sha256, + "runtime_manifest_sha256": self.runtime_manifest_sha256, + "data_contract_key": self.data_contract_key, + "data_contract_version": self.data_contract_version, + "validation_report_sha256": self.validation_report_sha256, + "source_registry_id": self.source_registry_id, + "source_snapshot_id": self.source_snapshot_id, + "source_snapshot_checksum_sha256": self.source_snapshot_checksum_sha256, + "source_version": self.source_version, + } + + +class RuntimeModelProvenanceService: + """Validate model sidecars and their production database binding. + + A sidecar lives next to its model as ``.geointel-model.json``. + Its ``metadata.runtime_manifest_sha256`` is the SHA-256 of canonical JSON + after omitting that one self-referential field. Any other mutation of the + manifest therefore invalidates it. Structural validation is deliberately + separate from :meth:`validate_for_production_runtime`: discovery and + preflight have no database session, whereas every production inference + path must prove an active source registry/snapshot binding. + """ + + MANIFEST_SUFFIX = ".geointel-model.json" + MANIFEST_SCHEMA_VERSION = "geointel.runtime-model-manifest/v1" + SOURCE_REGISTRY_KEY = "model" + + @classmethod + def manifest_path_for_model(cls, model_path: str | Path) -> Path: + path = Path(model_path).expanduser() + return Path(f"{path}{cls.MANIFEST_SUFFIX}") + + @staticmethod + def manifest_self_checksum(payload: Mapping[str, Any]) -> str: + """Hash sidecar semantics without its self-referential checksum field.""" + + canonical_payload = json.loads(json.dumps(payload, sort_keys=True, ensure_ascii=True)) + metadata = canonical_payload.get("metadata") + if isinstance(metadata, dict): + metadata.pop("runtime_manifest_sha256", None) + encoded = json.dumps( + canonical_payload, + sort_keys=True, + separators=(",", ":"), + ensure_ascii=True, + ).encode("utf-8") + return sha256(encoded).hexdigest() + + @classmethod + def validate_for_runtime( + cls, + *, + model_path: str | Path, + model_id: str, + task_type: str, + expected_model_version: str | None = None, + allowed_frameworks: tuple[str, ...] = (), + ) -> RuntimeModelProvenance: + """Return structural sidecar evidence without asserting database state. + + This method is appropriate for read-only catalog/preflight checks and + focused sidecar unit tests. It is insufficient for production + inference; adapters must use :meth:`validate_for_production_runtime`. + """ + + path = Path(model_path).expanduser() + if not path.exists() or not path.is_file(): + cls._raise( + "MODEL_PROVENANCE_MODEL_FILE_MISSING", + "Configured model file is missing; runtime provenance cannot be verified.", + model_path=str(path), + ) + manifest_path = cls.manifest_path_for_model(path) + if not manifest_path.exists() or not manifest_path.is_file(): + cls._raise( + "MODEL_PROVENANCE_MANIFEST_MISSING", + "Configured model requires an immutable .geointel-model.json sidecar before inference.", + model_path=str(path), + manifest_path=str(manifest_path), + ) + + try: + raw_manifest = manifest_path.read_bytes() + payload = json.loads(raw_manifest.decode("utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc: + cls._raise( + "MODEL_PROVENANCE_MANIFEST_INVALID", + "Configured model sidecar must be a readable UTF-8 JSON object.", + manifest_path=str(manifest_path), + error_type=type(exc).__name__, + ) + if not isinstance(payload, dict): + cls._raise( + "MODEL_PROVENANCE_MANIFEST_INVALID", + "Configured model sidecar must contain a JSON object.", + manifest_path=str(manifest_path), + ) + + cls._require_exact_text( + payload.get("schema_version"), + cls.MANIFEST_SCHEMA_VERSION, + field="schema_version", + manifest_path=manifest_path, + ) + contract = cls._require_mapping(payload, "data_contract", manifest_path) + contract_key = cls._require_text(contract, "key", manifest_path) + contract_version = cls._require_text(contract, "version", manifest_path) + # A valid manifest for a different artifact family must never make a + # local PyTorch/SAM weight executable. The structural validator below + # has a registry lookup too, but pinning the identity here keeps this + # runtime gate fail-closed if more model contracts are introduced. + cls._require_exact_text( + contract_key, + PYTORCH_MODEL_CONTRACT_KEY, + field="data_contract.key", + manifest_path=manifest_path, + ) + cls._require_exact_text( + contract_version, + PYTORCH_MODEL_CONTRACT_VERSION, + field="data_contract.version", + manifest_path=manifest_path, + ) + model = cls._require_mapping(payload, "model", manifest_path) + declared_model_id = cls._require_text(model, "model_id", manifest_path) + declared_task_type = cls._require_text(model, "task_type", manifest_path) + cls._require_exact_text(declared_model_id, model_id, field="model.model_id", manifest_path=manifest_path) + cls._require_exact_text(declared_task_type, task_type, field="model.task_type", manifest_path=manifest_path) + + declared_checksum = cls._require_checksum(model.get("sha256"), "model.sha256", manifest_path) + model_checksum = cls._file_sha256(path) + if declared_checksum != model_checksum: + cls._raise( + "MODEL_PROVENANCE_MODEL_CHECKSUM_MISMATCH", + "Model bytes do not match the checksum bound by the runtime sidecar.", + model_path=str(path), + expected=declared_checksum, + observed=model_checksum, + ) + + model_format = cls._require_text(model, "model_format", manifest_path) + framework = cls._require_text(model, "framework", manifest_path) + class_mapping = model.get("class_mapping") + if not isinstance(class_mapping, (dict, list, tuple)) or not class_mapping: + cls._raise( + "MODEL_PROVENANCE_MANIFEST_INVALID", + "model.class_mapping must be a non-empty mapping or sequence.", + manifest_path=str(manifest_path), + ) + normalized_framework = framework.strip().lower() + if allowed_frameworks and normalized_framework not in {value.strip().lower() for value in allowed_frameworks}: + cls._raise( + "MODEL_PROVENANCE_FRAMEWORK_MISMATCH", + "Model framework does not match the configured runtime adapter.", + manifest_path=str(manifest_path), + expected=sorted({value.strip().lower() for value in allowed_frameworks}), + observed=framework, + ) + source_version = cls._require_text(model, "source_version", manifest_path) + if expected_model_version and source_version != expected_model_version: + cls._raise( + "MODEL_PROVENANCE_VERSION_MISMATCH", + "Model sidecar version does not match the configured model version.", + manifest_path=str(manifest_path), + expected=expected_model_version, + observed=source_version, + ) + + source = cls._require_mapping(payload, "source", manifest_path) + source_registry_id = cls._require_uuid(source.get("source_registry_id"), "source.source_registry_id", manifest_path) + source_snapshot_id = cls._require_uuid(source.get("source_snapshot_id"), "source.source_snapshot_id", manifest_path) + cls._require_exact_text( + cls._require_text(source, "source_registry_key", manifest_path), + cls.SOURCE_REGISTRY_KEY, + field="source.source_registry_key", + manifest_path=manifest_path, + ) + snapshot_checksum = cls._require_checksum( + source.get("source_snapshot_checksum_sha256"), + "source.source_snapshot_checksum_sha256", + manifest_path, + ) + if snapshot_checksum != model_checksum: + cls._raise( + "MODEL_PROVENANCE_SNAPSHOT_CHECKSUM_MISMATCH", + "Model source snapshot checksum must bind the exact model bytes.", + manifest_path=str(manifest_path), + expected=model_checksum, + observed=snapshot_checksum, + ) + + metadata = cls._require_mapping(payload, "metadata", manifest_path) + training_manifest_sha256 = cls._require_checksum( + metadata.get("training_manifest_sha256"), + "metadata.training_manifest_sha256", + manifest_path, + ) + declared_runtime_manifest_sha256 = cls._require_checksum( + metadata.get("runtime_manifest_sha256"), + "metadata.runtime_manifest_sha256", + manifest_path, + ) + computed_runtime_manifest_sha256 = cls.manifest_self_checksum(payload) + if declared_runtime_manifest_sha256 != computed_runtime_manifest_sha256: + cls._raise( + "MODEL_PROVENANCE_MANIFEST_CHECKSUM_MISMATCH", + "Runtime model sidecar integrity checksum does not match its canonical contents.", + manifest_path=str(manifest_path), + expected=declared_runtime_manifest_sha256, + observed=computed_runtime_manifest_sha256, + ) + + lineage = cls._lineage_evidence(payload, manifest_path) + imported_at = cls._parse_imported_at(payload.get("imported_at"), manifest_path) + report = validate_registered_asset( + build_model_validation_input( + asset_id=f"{model_id}:{model_checksum}", + model_metadata={ + "model_format": model_format, + "framework": framework, + "class_mapping": class_mapping, + }, + checksum_sha256=declared_checksum, + computed_checksum_sha256=model_checksum, + source_registry_id=source_registry_id, + source_snapshot_id=source_snapshot_id, + imported_at=imported_at, + metadata={ + "training_manifest_sha256": training_manifest_sha256, + "runtime_manifest_sha256": declared_runtime_manifest_sha256, + }, + source_version=source_version, + lineage=lineage, + data_contract_key=contract_key, + data_contract_version=contract_version, + ) + ) + if report.failed: + cls._raise( + "MODEL_PROVENANCE_CONTRACT_FAILED", + "Configured model sidecar failed the exact versioned model data contract.", + manifest_path=str(manifest_path), + data_contract=f"{contract_key}@{contract_version}", + issue_codes=[issue.code for issue in report.issues], + validation_report_sha256=report.report_sha256, + ) + return RuntimeModelProvenance( + model_id=model_id, + task_type=task_type, + model_path=str(path.resolve()), + manifest_path=str(manifest_path.resolve()), + model_sha256=model_checksum, + manifest_sha256=sha256(raw_manifest).hexdigest(), + runtime_manifest_sha256=declared_runtime_manifest_sha256, + data_contract_key=contract_key, + data_contract_version=contract_version, + validation_report_sha256=report.report_sha256, + source_registry_id=source_registry_id, + source_snapshot_id=source_snapshot_id, + source_snapshot_checksum_sha256=snapshot_checksum, + source_version=source_version, + ) + + @classmethod + def validate_for_production_runtime( + cls, + *, + db: Any, + model_path: str | Path, + model_id: str, + task_type: str, + expected_model_version: str | None = None, + allowed_frameworks: tuple[str, ...] = (), + ) -> RuntimeModelProvenance: + """Validate byte-bound model evidence against governed database state. + + The sidecar is not itself a source of authority. The model bytes may + enter production inference only when the declared source registry and + immutable source snapshot both exist, belong together, are safe to + consume and bind the same SHA-256 and source version as the sidecar. + This check is intentionally invoked immediately before adapter loading. + """ + + if db is None or not callable(getattr(db, "get", None)): + cls._raise( + "MODEL_PROVENANCE_DATABASE_REQUIRED", + "Production model inference requires a database session for source snapshot provenance.", + ) + + evidence = cls.validate_for_runtime( + model_path=model_path, + model_id=model_id, + task_type=task_type, + expected_model_version=expected_model_version, + allowed_frameworks=allowed_frameworks, + ) + cls._assert_database_binding(db, evidence) + return evidence + + @classmethod + def _assert_database_binding(cls, db: Any, evidence: RuntimeModelProvenance) -> None: + registry_id = UUID(evidence.source_registry_id) + snapshot_id = UUID(evidence.source_snapshot_id) + source_registry = db.get(SourceRegistry, registry_id) + if not isinstance(source_registry, SourceRegistry): + cls._raise( + "MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND", + "Configured model sidecar refers to a source registry record that does not exist.", + source_registry_id=evidence.source_registry_id, + model_id=evidence.model_id, + ) + source_snapshot = db.get(SourceSnapshot, snapshot_id) + if not isinstance(source_snapshot, SourceSnapshot): + cls._raise( + "MODEL_PROVENANCE_SOURCE_SNAPSHOT_NOT_FOUND", + "Configured model sidecar refers to a source snapshot record that does not exist.", + source_snapshot_id=evidence.source_snapshot_id, + model_id=evidence.model_id, + ) + + if str(source_registry.id) != evidence.source_registry_id or source_registry.source_key != cls.SOURCE_REGISTRY_KEY: + cls._raise( + "MODEL_PROVENANCE_SOURCE_REGISTRY_IDENTITY_MISMATCH", + "Database source registry does not match the immutable model sidecar identity.", + expected_source_registry_id=evidence.source_registry_id, + observed_source_registry_id=str(source_registry.id), + expected_source_key=cls.SOURCE_REGISTRY_KEY, + observed_source_key=source_registry.source_key, + ) + if str(source_snapshot.id) != evidence.source_snapshot_id or str(source_snapshot.source_registry_id) != evidence.source_registry_id: + cls._raise( + "MODEL_PROVENANCE_SOURCE_SNAPSHOT_REGISTRY_MISMATCH", + "Model source snapshot does not belong to the declared source registry.", + source_registry_id=evidence.source_registry_id, + source_snapshot_id=evidence.source_snapshot_id, + observed_snapshot_registry_id=str(source_snapshot.source_registry_id), + ) + + registry_ingest_status = cls._normalise_status(source_registry.ingest_status) + registry_freshness_status = cls._normalise_status(source_registry.freshness_status) + if ( + registry_ingest_status not in _SAFE_SOURCE_REGISTRY_INGEST_STATUSES + or registry_freshness_status not in _CONSUMABLE_FRESHNESS + ): + cls._raise( + "MODEL_PROVENANCE_SOURCE_REGISTRY_UNSAFE", + "Configured model source registry is not in a safe configured/current state.", + source_registry_id=evidence.source_registry_id, + ingest_status=registry_ingest_status, + freshness_status=registry_freshness_status, + ) + + snapshot_ingest_status = cls._normalise_status(source_snapshot.ingest_status) + snapshot_freshness_status = cls._normalise_status(source_snapshot.freshness_status) + if ( + snapshot_ingest_status != _SAFE_SOURCE_SNAPSHOT_INGEST_STATUS + or snapshot_freshness_status not in _CONSUMABLE_FRESHNESS + ): + cls._raise( + "MODEL_PROVENANCE_SOURCE_SNAPSHOT_UNSAFE", + "Configured model source snapshot is not an ingested current immutable artifact.", + source_snapshot_id=evidence.source_snapshot_id, + ingest_status=snapshot_ingest_status, + freshness_status=snapshot_freshness_status, + ) + + snapshot_checksum = str(source_snapshot.checksum_sha256 or "").strip().lower() + if snapshot_checksum != evidence.source_snapshot_checksum_sha256 or snapshot_checksum != evidence.model_sha256: + cls._raise( + "MODEL_PROVENANCE_DATABASE_SNAPSHOT_CHECKSUM_MISMATCH", + "Database model source snapshot checksum does not bind the exact sidecar and model bytes.", + source_snapshot_id=evidence.source_snapshot_id, + expected_model_sha256=evidence.model_sha256, + expected_sidecar_snapshot_sha256=evidence.source_snapshot_checksum_sha256, + observed_snapshot_sha256=snapshot_checksum, + ) + + snapshot_version = str(source_snapshot.source_version or "").strip() + if snapshot_version != evidence.source_version: + cls._raise( + "MODEL_PROVENANCE_SOURCE_SNAPSHOT_VERSION_MISMATCH", + "Database model source snapshot version does not match the immutable sidecar model version.", + source_snapshot_id=evidence.source_snapshot_id, + expected_source_version=evidence.source_version, + observed_source_version=snapshot_version or None, + ) + + # The persisted snapshot state is the primary quarantine signal. The + # relationship check is a defensive second line for an active + # quarantine record that predates or bypassed a status transition. + active_quarantines = getattr(source_snapshot, "quarantines", ()) + if any(cls._normalise_status(getattr(item, "status", None)) == "quarantined" for item in active_quarantines): + cls._raise( + "MODEL_PROVENANCE_SOURCE_SNAPSHOT_QUARANTINED", + "Configured model source snapshot has an active quarantine record.", + source_snapshot_id=evidence.source_snapshot_id, + ) + + @staticmethod + def _normalise_status(value: Any) -> str: + return str(value or "").strip().lower() + + @classmethod + def _lineage_evidence(cls, payload: Mapping[str, Any], manifest_path: Path) -> LineageEvidence: + lineage = cls._require_mapping(payload, "lineage", manifest_path) + raw_asset_ids = lineage.get("upstream_asset_ids") + raw_checksums = lineage.get("upstream_checksums_sha256") + raw_transformations = lineage.get("transformations") + if not isinstance(raw_asset_ids, list) or not raw_asset_ids or not all( + isinstance(value, str) and value.strip() for value in raw_asset_ids + ): + cls._raise( + "MODEL_PROVENANCE_MANIFEST_INVALID", + "lineage.upstream_asset_ids must be a non-empty string list.", + manifest_path=str(manifest_path), + ) + if not isinstance(raw_checksums, list) or len(raw_checksums) != len(raw_asset_ids): + cls._raise( + "MODEL_PROVENANCE_MANIFEST_INVALID", + "lineage.upstream_checksums_sha256 must match upstream_asset_ids one-for-one.", + manifest_path=str(manifest_path), + ) + upstream_checksums = tuple( + cls._require_checksum(value, f"lineage.upstream_checksums_sha256[{index}]", manifest_path) + for index, value in enumerate(raw_checksums) + ) + if not isinstance(raw_transformations, list) or not raw_transformations: + cls._raise( + "MODEL_PROVENANCE_MANIFEST_INVALID", + "lineage.transformations must contain at least one immutable transformation record.", + manifest_path=str(manifest_path), + ) + transformations: list[TransformationEvidence] = [] + for index, raw in enumerate(raw_transformations): + if not isinstance(raw, Mapping): + cls._raise( + "MODEL_PROVENANCE_MANIFEST_INVALID", + "Each lineage transformation must be an object.", + manifest_path=str(manifest_path), + index=index, + ) + transformations.append( + TransformationEvidence( + name=cls._require_text(raw, "name", manifest_path, prefix=f"lineage.transformations[{index}]."), + version=cls._require_text(raw, "version", manifest_path, prefix=f"lineage.transformations[{index}]."), + checksum_sha256=cls._require_checksum( + raw.get("checksum_sha256"), + f"lineage.transformations[{index}].checksum_sha256", + manifest_path, + ), + ) + ) + return LineageEvidence( + upstream_asset_ids=tuple(raw_asset_ids), + upstream_checksums_sha256=upstream_checksums, + transformations=tuple(transformations), + ) + + @staticmethod + def _file_sha256(path: Path) -> str: + digest = sha256() + try: + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + except OSError as exc: + RuntimeModelProvenanceService._raise( + "MODEL_PROVENANCE_MODEL_FILE_UNREADABLE", + "Configured model file could not be read for checksum validation.", + model_path=str(path), + error_type=type(exc).__name__, + ) + return digest.hexdigest() + + @classmethod + def _parse_imported_at(cls, value: Any, manifest_path: Path) -> datetime: + if not isinstance(value, str) or not value.strip(): + cls._raise( + "MODEL_PROVENANCE_MANIFEST_INVALID", + "imported_at must be a timezone-aware ISO-8601 timestamp.", + manifest_path=str(manifest_path), + ) + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + cls._raise( + "MODEL_PROVENANCE_MANIFEST_INVALID", + "imported_at must be a timezone-aware ISO-8601 timestamp.", + manifest_path=str(manifest_path), + observed=value, + ) + if parsed.tzinfo is None: + cls._raise( + "MODEL_PROVENANCE_MANIFEST_INVALID", + "imported_at must include a timezone offset.", + manifest_path=str(manifest_path), + observed=value, + ) + return parsed.astimezone(timezone.utc) + + @classmethod + def _require_mapping(cls, payload: Mapping[str, Any], key: str, manifest_path: Path) -> Mapping[str, Any]: + value = payload.get(key) + if not isinstance(value, Mapping): + cls._raise( + "MODEL_PROVENANCE_MANIFEST_INVALID", + f"{key} must be a JSON object.", + manifest_path=str(manifest_path), + ) + return value + + @classmethod + def _require_text( + cls, + payload: Mapping[str, Any], + key: str, + manifest_path: Path, + *, + prefix: str = "", + ) -> str: + value = payload.get(key) + if not isinstance(value, str) or not value.strip(): + cls._raise( + "MODEL_PROVENANCE_MANIFEST_INVALID", + f"{prefix}{key} must be a non-empty string.", + manifest_path=str(manifest_path), + ) + return value.strip() + + @classmethod + def _require_checksum(cls, value: Any, field: str, manifest_path: Path) -> str: + normalized = value.strip().lower() if isinstance(value, str) else "" + if not _SHA256.fullmatch(normalized) or value != normalized: + cls._raise( + "MODEL_PROVENANCE_MANIFEST_INVALID", + f"{field} must be a lowercase SHA-256 digest.", + manifest_path=str(manifest_path), + ) + return normalized + + @classmethod + def _require_uuid(cls, value: Any, field: str, manifest_path: Path) -> str: + if not isinstance(value, str) or not value.strip(): + cls._raise( + "MODEL_PROVENANCE_MANIFEST_INVALID", + f"{field} must be a UUID string.", + manifest_path=str(manifest_path), + ) + try: + return str(UUID(value)) + except (AttributeError, ValueError): + cls._raise( + "MODEL_PROVENANCE_MANIFEST_INVALID", + f"{field} must be a UUID string.", + manifest_path=str(manifest_path), + observed=value, + ) + + @classmethod + def _require_exact_text( + cls, + observed: Any, + expected: str, + *, + field: str, + manifest_path: Path, + ) -> None: + if not isinstance(observed, str) or observed.strip() != expected: + cls._raise( + "MODEL_PROVENANCE_MANIFEST_INVALID", + f"{field} does not match the configured runtime identity.", + manifest_path=str(manifest_path), + expected=expected, + observed=observed, + ) + + @staticmethod + def _raise(code: str, message: str, **details: Any) -> None: + raise AppError(code=code, message=message, details=details, status_code=422) diff --git a/backend/app/services/segmentation_service.py b/backend/app/services/segmentation_service.py index 62dc63fb..95c8bf24 100644 --- a/backend/app/services/segmentation_service.py +++ b/backend/app/services/segmentation_service.py @@ -21,9 +21,11 @@ from app.schemas.segmentation import ( ) from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon from app.services.detection_service import DetectionService +from app.services.dataset_consumption_gate_service import DatasetConsumptionGate from app.services.model_registry_service import ModelRegistryService from app.services.qa_service import QaService from app.services.quality_service import QualityService +from app.services.runtime_model_provenance_service import RuntimeModelProvenance, RuntimeModelProvenanceService from app.services.segmentation_adapter import ( FixtureSegmentationAdapter, SamSegmentationAdapter, @@ -89,6 +91,18 @@ class SegmentationService: status_code=400, ) + # Production segmentation must consume only a passed, complete and + # non-quarantined dataset. The fixture segmenter is QA/test-only and + # cannot be classified as production inference. + if model.model_id == "fixture-segmenter": + DatasetConsumptionGate.assert_eligible( + dataset, + purpose="quality_assessment", + fixture_mode=True, + ) + elif model.model_id in configured_model_ids and model.configured: + DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference") + run_parameters = { "model_id": model.model_id, "confidence_threshold": confidence_threshold, @@ -326,6 +340,27 @@ class SegmentationService: if reference_dataset.dataset_type not in {"vector", "geojson"}: raise AppError(code="INVALID_DATASET_TYPE", message="Reference dataset must be vector data", status_code=400) + candidate_dataset = db.get(Dataset, run.dataset_id) + if not candidate_dataset: + raise AppError(code="DATASET_NOT_FOUND", message="Segmentation source dataset not found", status_code=404) + run_parameters = run.parameters_json if isinstance(run.parameters_json, dict) else {} + fixture_parameters = run_parameters.get("parameters_json") + fixture_mode = bool( + run.model_name == "fixture-segmenter" + and isinstance(fixture_parameters, dict) + and fixture_parameters.get("fixture_mode") is True + ) + DatasetConsumptionGate.assert_eligible( + candidate_dataset, + purpose="quality_assessment", + fixture_mode=fixture_mode, + ) + DatasetConsumptionGate.assert_eligible( + reference_dataset, + purpose="reference_validation", + reference_task="building_validation", + ) + segmentations = SegmentationService._query_segmentation_rows( db, analysis_run_id=analysis_run_id, @@ -510,11 +545,26 @@ class SegmentationService: ) -> tuple[list[Segmentation], dict[str, Any]]: manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles) if model_name == settings.sam_model_id: - adapter = sam_adapter_class(settings) model_path = Path(settings.sam_model_path or "").expanduser() + allowed_frameworks = ("ultralytics/sam", "sam", "ultralytics", "pytorch") + adapter = sam_adapter_class(settings) else: - adapter = yolo_seg_adapter_class(settings) model_path = Path(settings.yolo_seg_model_path or "").expanduser() + allowed_frameworks = ("ultralytics/pytorch", "ultralytics", "pytorch") + adapter = yolo_seg_adapter_class(settings) + runtime_model_provenance = RuntimeModelProvenanceService.validate_for_production_runtime( + db=db, + model_path=model_path, + model_id=model_name, + task_type="segmentation", + expected_model_version=model_version, + allowed_frameworks=allowed_frameworks, + ) + SegmentationService._attach_runtime_model_provenance( + analysis_run, + job, + runtime_model_provenance, + ) model = adapter.load_model(model_path) allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)} @@ -591,6 +641,7 @@ class SegmentationService: "tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()), "tile_index": candidate["tile_index"], "device": settings.yolo_device, + "runtime_model_provenance": runtime_model_provenance.as_dict(), }, ) db.add(segmentation) @@ -603,8 +654,25 @@ class SegmentationService: "suppressed_segmentation_count": len(candidates) - len(filtered_candidates), "duplicate_iou_threshold": float(settings.segmentation_duplicate_iou_threshold), "tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()), + "runtime_model_provenance": runtime_model_provenance.as_dict(), } + @staticmethod + def _attach_runtime_model_provenance( + analysis_run: AnalysisRun, + job: Job, + provenance: RuntimeModelProvenance, + ) -> None: + """Record immutable model evidence with a configured segmentation run.""" + + evidence = provenance.as_dict() + analysis_parameters = dict(analysis_run.parameters_json or {}) + analysis_parameters["runtime_model_provenance"] = evidence + analysis_run.parameters_json = analysis_parameters + job_parameters = dict(job.parameters_json or {}) + job_parameters["runtime_model_provenance"] = evidence + job.parameters_json = job_parameters + @staticmethod def _geodesic_area_m2(geometry: MultiPolygon | Polygon) -> float | None: try: diff --git a/backend/app/services/source_registry_service.py b/backend/app/services/source_registry_service.py new file mode 100644 index 00000000..ac05ec60 --- /dev/null +++ b/backend/app/services/source_registry_service.py @@ -0,0 +1,1509 @@ +"""Server-owned source identity, snapshot and lineage persistence primitives. + +This service deliberately does not inspect caller-provided ``source_name`` +metadata. A governed adapter must select one of the static definitions below, +create an immutable snapshot and bind that exact snapshot to a dataset/version. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +import re +from typing import Any +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.core.errors import AppError +from app.models import ( + Dataset, + DatasetLineageEdge, + DatasetQuarantine, + DatasetVersion, + SourceRegistry, + SourceSnapshot, +) + + +_CHECKSUM = re.compile(r"^[a-fA-F0-9]{64}$") +_VALIDATION_STATUSES = {"not_validated", "passed", "failed"} +_PROVENANCE_STATUSES = {"complete", "incomplete", "not_applicable"} +_LINEAGE_STATUSES = {"complete", "incomplete", "not_applicable"} +_FRESHNESS_STATUSES = { + "unknown", + "current", + "due", + "stale", + "not_applicable", + "review_required", +} +_SNAPSHOT_INGEST_STATUSES = { + "registered", + "configured", + "not_configured", + "available", + "ingested", + "failed", + "quarantined", + "legacy_unverified", +} +_MAX_LINEAGE_GRAPH_NODES = 50_000 + + +def _usage_policy( + *, + ground_truth_allowed: bool = False, + training_allowed: bool = False, + allowed_tasks: tuple[str, ...] = (), + validation_authority: dict[str, str] | None = None, +) -> dict[str, Any]: + return { + "automatic_ground_truth": False, + "ground_truth_allowed": ground_truth_allowed, + "training_allowed": training_allowed, + "allowed_tasks": list(allowed_tasks), + "validation_authority": validation_authority or {}, + } + + +@dataclass(frozen=True) +class SourceRegistryDefinition: + source_key: str + display_name: str + classification: str + authority_name: str + authority_scope: dict[str, Any] + default_crs: str = "unknown" + default_units: str = "unknown" + provider_adapter_key: str | None = None + source_url: str | None = None + license_name: str = "Provider terms must be verified for each immutable snapshot." + usage_restrictions: str = ( + "Use only according to the source-specific snapshot terms and attribution." + ) + spatial_resolution: dict[str, Any] | None = None + temporal_coverage: dict[str, Any] | None = None + geographic_coverage: dict[str, Any] | None = None + expected_geometry_types: tuple[str, ...] = () + expected_attributes: dict[str, Any] | None = None + usage_policy: dict[str, Any] | None = None + freshness_status: str = "unknown" + ingest_status: str = "registered" + known_limitations: tuple[str, ...] = () + + def as_model_values(self) -> dict[str, Any]: + return { + "source_key": self.source_key, + "display_name": self.display_name, + "classification": self.classification, + "authority_name": self.authority_name, + "authority_scope_json": dict(self.authority_scope), + "provider_adapter_key": self.provider_adapter_key, + "source_url": self.source_url, + "license_name": self.license_name, + "usage_restrictions": self.usage_restrictions, + "default_crs": self.default_crs, + "default_units": self.default_units, + "spatial_resolution_json": dict( + self.spatial_resolution or {"status": "unknown"} + ), + "temporal_coverage_json": dict( + self.temporal_coverage or {"status": "unknown"} + ), + "geographic_coverage_json": dict( + self.geographic_coverage or {"status": "unknown"} + ), + "expected_geometry_types_json": list(self.expected_geometry_types), + "expected_attributes_json": dict( + self.expected_attributes or {"status": "unknown"} + ), + "usage_policy_json": dict(self.usage_policy or _usage_policy()), + "freshness_status": self.freshness_status, + "ingest_status": self.ingest_status, + "known_limitations_json": list( + self.known_limitations + or ( + "No authority, ground-truth, freshness or training claim is allowed without a governed snapshot and passed contract.", + ) + ), + "registry_metadata_json": { + "registry_owner": "server", + "definition_version": "phase2-v1", + }, + } + + +def _definition( + source_key: str, + display_name: str, + classification: str, + authority_name: str, + authority_scope: dict[str, Any], + **kwargs: Any, +) -> SourceRegistryDefinition: + return SourceRegistryDefinition( + source_key=source_key, + display_name=display_name, + classification=classification, + authority_name=authority_name, + authority_scope=authority_scope, + **kwargs, + ) + + +_VECTOR_CONTEXT = _usage_policy(allowed_tasks=("reference_context",)) +_IMAGERY_CONTEXT = _usage_policy( + training_allowed=True, + allowed_tasks=("imagery", "training_input", "visual_context"), +) +_REGIONAL_BUILDING_LABELS = _usage_policy( + ground_truth_allowed=True, + training_allowed=True, + allowed_tasks=("building_validation", "building_labels"), + validation_authority={"building_validation": "regional_primary_pending_contract"}, +) + + +SERVER_OWNED_SOURCE_DEFINITIONS: dict[str, SourceRegistryDefinition] = { + definition.source_key: definition + for definition in ( + _definition( + "grb", + "Grootschalig Referentie Bestand", + "authoritative", + "Digitaal Vlaanderen", + {"zone": "Flanders", "themes": ["buildings", "roads", "water", "parcels"]}, + provider_adapter_key="grb", + source_url="https://www.vlaanderen.be/datavindplaats/catalogus/basiskaart-vlaanderen-grb", + default_crs="EPSG:31370", + default_units="metres", + geographic_coverage={"zone": "Flanders"}, + expected_geometry_types=( + "Polygon", + "MultiPolygon", + "LineString", + "MultiLineString", + ), + expected_attributes={ + "required": ["id"], + "layers": ["GBG", "Wegsegment", "WTZ", "WLAS", "WGR", "ADP"], + }, + usage_policy=_usage_policy( + ground_truth_allowed=True, + training_allowed=True, + allowed_tasks=( + "building_validation", + "building_labels", + "reference_context", + ), + validation_authority={"building_validation": "primary"}, + ), + ingest_status="configured", + known_limitations=( + "GRB building geometry is authoritative only for a governed, versioned snapshot within Flanders.", + "GRB does not independently establish imagery-time alignment or national model validation.", + ), + ), + _definition( + "digitaal_vlaanderen", + "Digitaal Vlaanderen (bronportaal)", + "authoritative", + "Digitaal Vlaanderen", + {"zone": "Flanders", "role": "umbrella_catalogue_and_adapter_authority"}, + provider_adapter_key="digitaal_vlaanderen", + source_url="https://www.vlaanderen.be/datavindplaats", + default_crs="product_specific", + default_units="product_specific", + geographic_coverage={"zone": "Flanders"}, + usage_policy=_usage_policy( + allowed_tasks=("source_catalogue", "reference_context") + ), + ingest_status="configured", + known_limitations=( + "This umbrella authority is not a product-level ground-truth source.", + "A governed import must use a product-specific source key whenever one is available.", + ), + ), + _definition( + "digitaal_vlaanderen_buildings_addresses_register", + "Gebouwen- en adressenregister", + "authoritative", + "Digitaal Vlaanderen", + {"zone": "Flanders", "theme": "buildings_addresses"}, + provider_adapter_key="buildings_addresses_register", + default_crs="EPSG:31370", + default_units="metres", + geographic_coverage={"zone": "Flanders"}, + expected_geometry_types=("Point", "Polygon", "MultiPolygon"), + expected_attributes={ + "required": ["id"], + "role": "administrative_corroboration", + }, + usage_policy=_usage_policy( + allowed_tasks=( + "building_validation", + "address_corroboration", + "building_register_validation", + ), + validation_authority={ + "building_validation": "corroborative", + "building_register_validation": "primary", + }, + ), + known_limitations=( + "Administrative records do not replace a governed footprint-label contract.", + ), + ), + _definition( + "sentinel_2", + "Sentinel-2", + "contextual", + "Copernicus Programme", + {"scope": "Belgium and Belgian North Sea", "role": "multispectral_context"}, + provider_adapter_key="sentinel_2", + default_crs="product_specific", + default_units="reflectance", + spatial_resolution={"metres": [10, 20, 60]}, + temporal_coverage={"cadence_days": 5, "status": "product_specific"}, + geographic_coverage={"scope": "Belgium and Belgian North Sea"}, + expected_attributes={"required": ["product_id", "sensing_time"]}, + usage_policy=_usage_policy( + training_allowed=True, + allowed_tasks=("imagery_context", "change_context"), + ), + ingest_status="not_configured", + known_limitations=( + "Sentinel-2 is contextual imagery, never automatic building ground truth.", + ), + ), + _definition( + "digitaal_vlaanderen_dhmv", + "Digitaal Hoogtemodel Vlaanderen", + "authoritative", + "Digitaal Vlaanderen", + {"zone": "Flanders", "role": "terrain_height_corroboration"}, + provider_adapter_key="dhmv", + default_crs="EPSG:31370", + default_units="m TAW", + spatial_resolution={"metres": 1}, + temporal_coverage={"period": "2013-2015", "status": "product_specific"}, + geographic_coverage={"zone": "Flanders"}, + expected_attributes={"bands": 1, "nodata_required": True}, + usage_policy=_usage_policy( + training_allowed=True, + allowed_tasks=( + "terrain_context", + "height_corroboration", + "elevation_validation", + ), + validation_authority={ + "building_validation": "corroborative", + "elevation_validation": "primary", + }, + ), + ingest_status="configured", + known_limitations=( + "DHMV is height context and cannot independently establish building labels.", + ), + ), + _definition( + "osm", + "OpenStreetMap", + "contextual", + "OpenStreetMap contributors", + {"scope": "community-maintained", "role": "contextual"}, + provider_adapter_key="osm", + source_url="https://www.openstreetmap.org", + license_name="ODbL", + usage_restrictions="OpenStreetMap attribution and ODbL obligations apply.", + default_crs="EPSG:4326", + default_units="mixed", + geographic_coverage={"scope": "global"}, + expected_geometry_types=("Point", "LineString", "Polygon", "MultiPolygon"), + expected_attributes={"status": "community_tags"}, + usage_policy=_usage_policy( + allowed_tasks=("context", "candidate_discovery") + ), + ingest_status="not_configured", + known_limitations=( + "OSM is never automatic ground truth for GeoIntel validation or labels.", + ), + ), + _definition( + "manual", + "Handmatige upload", + "experimental", + "Operator supplied", + {"scope": "operator_supplied", "trust": "unverified"}, + known_limitations=( + "Manual uploads remain untrusted until a passed contract and governed provenance are attached.", + ), + ingest_status="configured", + ), + _definition( + "fixture", + "Test- en demo fixture", + "experimental", + "GeoIntel test fixture", + {"scope": "test_only"}, + default_crs="fixture_specific", + default_units="fixture_specific", + known_limitations=( + "Fixtures must never be presented as official data or used for production training/promotion.", + ), + ingest_status="configured", + ), + _definition( + "map_selection", + "Afgeleide kaartselectie", + "derived", + "GeoIntel derived operation", + {"scope": "derived_from_registered_input"}, + default_crs="EPSG:4326", + default_units="source_dependent", + known_limitations=( + "Derived selections inherit no authority beyond complete source snapshots and lineage edges.", + ), + ), + _definition( + "derived", + "Afgeleide dataset", + "derived", + "GeoIntel derived operation", + {"scope": "derived_from_registered_input"}, + ), + _definition( + "training_label", + "Afgeleide trainingslabels", + "derived", + "GeoIntel reviewed label pipeline", + {"scope": "derived_from_reviewed_source_snapshots"}, + known_limitations=( + "Training labels require complete source lineage and human-review evidence; they inherit no automatic authority.", + ), + ), + _definition( + "model", + "Model artifact", + "experimental", + "GeoIntel model pipeline", + {"scope": "internal_model_artifact"}, + known_limitations=( + "A model artifact is not a validated capability or promotion decision without its model card and evaluation evidence.", + ), + ), + _definition( + "experimental", + "Experimentele bron", + "experimental", + "Unverified", + {"scope": "unverified"}, + ), + _definition( + "legacy_unknown", + "Niet-geclassificeerde historische bron", + "experimental", + "Legacy import — unverified", + {"scope": "legacy", "trust": "unverified"}, + ingest_status="legacy_unverified", + known_limitations=( + "Historical source identity is descriptive only until re-ingested through a governed adapter.", + ), + ), + _definition( + "ngi_adminvector", + "NGI AdminVector", + "authoritative", + "Nationaal Geografisch Instituut", + {"scope": "Belgium"}, + default_crs="EPSG:31370", + default_units="metres", + geographic_coverage={"scope": "Belgium"}, + expected_geometry_types=("Polygon", "MultiPolygon"), + usage_policy=_VECTOR_CONTEXT, + ), + _definition( + "rbins_marine_reporting_units", + "RBINS mariene rapportage-eenheden", + "authoritative", + "RBINS", + {"zone": "Belgian North Sea"}, + default_crs="EPSG:4326", + default_units="degrees", + geographic_coverage={"zone": "Belgian North Sea"}, + expected_geometry_types=("Polygon", "MultiPolygon"), + usage_policy=_VECTOR_CONTEXT, + ), + _definition( + "rbins_msp_2026", + "Belgisch Marien Ruimtelijk Plan 2026-2034", + "authoritative", + "RBINS", + {"zone": "Belgian North Sea", "edition": "2026-2034"}, + default_crs="EPSG:4326", + default_units="degrees", + geographic_coverage={"zone": "Belgian North Sea"}, + expected_geometry_types=("Polygon", "MultiPolygon"), + usage_policy=_VECTOR_CONTEXT, + ), + _definition( + "vrbg", + "Vlaams Wegenregister", + "authoritative", + "Digitaal Vlaanderen", + {"zone": "Flanders", "theme": "roads"}, + default_crs="EPSG:31370", + default_units="metres", + geographic_coverage={"zone": "Flanders"}, + expected_geometry_types=("LineString", "MultiLineString"), + usage_policy=_VECTOR_CONTEXT, + ), + _definition( + "digitaal_vlaanderen_orthophoto", + "Orthofoto Vlaanderen", + "contextual", + "Digitaal Vlaanderen", + {"zone": "Flanders", "role": "imagery"}, + default_crs="EPSG:31370", + default_units="pixel", + spatial_resolution={"metres": 0.25}, + geographic_coverage={"zone": "Flanders"}, + usage_policy=_IMAGERY_CONTEXT, + ingest_status="configured", + ), + _definition( + "spw_orthophoto", + "Orthofoto Wallonië", + "contextual", + "Service public de Wallonie", + {"zone": "Wallonia", "role": "imagery"}, + license_name="CC BY 4.0", + default_crs="EPSG:31370", + default_units="pixel", + spatial_resolution={"metres": 0.25}, + geographic_coverage={"zone": "Wallonia"}, + usage_policy=_IMAGERY_CONTEXT, + ingest_status="configured", + ), + _definition( + "urbis_orthophoto", + "Orthofoto Brussel", + "contextual", + "UrbIS / Brussels Region", + {"zone": "Brussels-Capital Region", "role": "imagery"}, + license_name="CC0", + default_crs="EPSG:31370", + default_units="pixel", + spatial_resolution={"metres": 0.25}, + geographic_coverage={"zone": "Brussels-Capital Region"}, + usage_policy=_IMAGERY_CONTEXT, + ingest_status="configured", + ), + _definition( + "agentschap_landbouw_zeevisserij_agricultural_parcels", + "Landbouwgebruikspercelen", + "authoritative", + "Agentschap Landbouw en Zeevisserij", + {"zone": "Flanders", "theme": "agricultural_parcels"}, + default_crs="EPSG:31370", + default_units="metres", + geographic_coverage={"zone": "Flanders"}, + expected_geometry_types=("Polygon", "MultiPolygon"), + usage_policy=_VECTOR_CONTEXT, + ), + _definition( + "department_omgeving_land_use", + "Landgebruik Vlaanderen", + "authoritative", + "Departement Omgeving", + {"zone": "Flanders", "theme": "land_use"}, + default_crs="EPSG:31370", + default_units="metres", + geographic_coverage={"zone": "Flanders"}, + expected_geometry_types=("Polygon", "MultiPolygon"), + usage_policy=_VECTOR_CONTEXT, + ), + _definition( + "inbo_bwk_natura2000", + "BWK en Natura 2000", + "authoritative", + "INBO", + {"zone": "Flanders", "theme": "nature"}, + default_crs="EPSG:31370", + default_units="metres", + geographic_coverage={"zone": "Flanders"}, + expected_geometry_types=("Polygon", "MultiPolygon"), + usage_policy=_VECTOR_CONTEXT, + ), + _definition( + "statbel", + "Statbel bevolking", + "authoritative", + "Statbel", + {"scope": "Belgium", "theme": "population"}, + default_crs="EPSG:31370", + default_units="persons", + geographic_coverage={"scope": "Belgium"}, + expected_geometry_types=("Polygon", "MultiPolygon"), + usage_policy=_VECTOR_CONTEXT, + ), + _definition( + "waterinfo", + "Waterinfo", + "authoritative", + "Waterinfo Vlaanderen", + {"zone": "Flanders", "theme": "water"}, + default_crs="EPSG:31370", + default_units="source_specific", + geographic_coverage={"zone": "Flanders"}, + usage_policy=_VECTOR_CONTEXT, + ), + _definition( + "department_omgeving_thematic_raster", + "Omgeving thematische rasters", + "authoritative", + "Departement Omgeving", + {"zone": "Flanders", "theme": "thematic_raster"}, + default_crs="EPSG:31370", + default_units="source_specific", + geographic_coverage={"zone": "Flanders"}, + usage_policy=_VECTOR_CONTEXT, + ), + _definition( + "dov_soil_map", + "DOV bodemkaart", + "authoritative", + "Databank Ondergrond Vlaanderen", + {"zone": "Flanders", "theme": "soil"}, + default_crs="EPSG:31370", + default_units="metres", + geographic_coverage={"zone": "Flanders"}, + expected_geometry_types=("Polygon", "MultiPolygon"), + usage_policy=_VECTOR_CONTEXT, + ), + _definition( + "vmm_flood_hazard", + "VMM overstromingskaarten", + "authoritative", + "Vlaamse Milieumaatschappij", + {"zone": "Flanders", "theme": "flood_hazard"}, + default_crs="EPSG:31370", + default_units="metres", + geographic_coverage={"zone": "Flanders"}, + usage_policy=_VECTOR_CONTEXT, + ), + _definition( + "vmm_vha_bathymetry_profiles", + "VHA bathymetrieprofielen", + "authoritative", + "Vlaamse Milieumaatschappij", + {"zone": "Flanders", "theme": "bathymetry_profiles"}, + default_crs="EPSG:31370", + default_units="m TAW", + geographic_coverage={"zone": "Flanders"}, + expected_geometry_types=("Point",), + usage_policy=_VECTOR_CONTEXT, + ), + _definition( + "historical_landuse", + "Historisch landgebruik", + "corroborative", + "Historical archive provider", + {"scope": "Belgium", "theme": "historical_land_use"}, + default_crs="source_specific", + default_units="source_specific", + usage_policy=_VECTOR_CONTEXT, + ), + _definition( + "spw_geoportail", + "SPW Geoportail (bronportaal)", + "authoritative", + "Service public de Wallonie", + {"zone": "Wallonia", "role": "umbrella_catalogue_and_adapter_authority"}, + provider_adapter_key="spw_geoportail", + source_url="https://geoportail.wallonie.be/catalogue", + default_crs="product_specific", + default_units="product_specific", + geographic_coverage={"zone": "Wallonia"}, + usage_policy=_usage_policy( + allowed_tasks=("source_catalogue", "reference_context") + ), + ingest_status="configured", + known_limitations=( + "This umbrella authority is not a product-level ground-truth source.", + "A governed import must use PICC, WALOUS, terrain, flood, orthophoto or another product-specific key when available.", + ), + ), + _definition( + "spw_picc", + "PICC", + "authoritative", + "Service public de Wallonie", + {"zone": "Wallonia", "theme": "topography_buildings"}, + license_name="CC BY 4.0", + default_crs="EPSG:31370", + default_units="metres", + geographic_coverage={"zone": "Wallonia"}, + expected_geometry_types=( + "Polygon", + "MultiPolygon", + "LineString", + "MultiLineString", + ), + usage_policy=_REGIONAL_BUILDING_LABELS, + ), + _definition( + "urbis", + "UrbIS", + "authoritative", + "Brussels Region", + {"zone": "Brussels-Capital Region", "theme": "topography_buildings"}, + license_name="CC0", + default_crs="EPSG:31370", + default_units="metres", + geographic_coverage={"zone": "Brussels-Capital Region"}, + expected_geometry_types=( + "Polygon", + "MultiPolygon", + "LineString", + "MultiLineString", + ), + usage_policy=_REGIONAL_BUILDING_LABELS, + ), + _definition( + "spw_walous_land_cover", + "WALOUS landbedekking", + "authoritative", + "Service public de Wallonie", + {"zone": "Wallonia", "theme": "land_cover"}, + license_name="CC BY 4.0", + default_crs="EPSG:3812", + default_units="class_code", + spatial_resolution={"metres": 1}, + geographic_coverage={"zone": "Wallonia"}, + usage_policy=_VECTOR_CONTEXT, + ), + _definition( + "spw_bathymetry", + "SPW bathymetrie", + "authoritative", + "Service public de Wallonie", + {"zone": "Wallonia", "theme": "bathymetry"}, + default_crs="EPSG:3812", + default_units="mDNG", + geographic_coverage={"zone": "Wallonia"}, + usage_policy=_VECTOR_CONTEXT, + ), + _definition( + "spw_terrain", + "SPW terreinmodel", + "corroborative", + "Service public de Wallonie", + {"zone": "Wallonia", "theme": "terrain"}, + default_crs="EPSG:3812", + default_units="metres", + spatial_resolution={"metres": 1}, + geographic_coverage={"zone": "Wallonia"}, + usage_policy=_VECTOR_CONTEXT, + ), + _definition( + "spw_flood_hazard", + "SPW overstromingsgevaar", + "authoritative", + "Service public de Wallonie", + {"zone": "Wallonia", "theme": "flood_hazard"}, + default_crs="EPSG:3812", + default_units="metres", + geographic_coverage={"zone": "Wallonia"}, + usage_policy=_VECTOR_CONTEXT, + ), + _definition( + "mdk_bathymetry", + "MDK bathymetrie", + "authoritative", + "Maritieme Dienstverlening en Kust", + {"zone": "Belgian North Sea", "theme": "bathymetry"}, + default_crs="EPSG:3812", + default_units="metres", + geographic_coverage={"zone": "Belgian North Sea"}, + usage_policy=_VECTOR_CONTEXT, + ), + _definition( + "mdk_bcp_bathymetry", + "MDK BCP bathymetrie-probe en verwerving", + "authoritative", + "Maritieme Dienstverlening en Kust", + { + "zone": "Belgian North Sea", + "theme": "bathymetry", + "role": "coverage_probe_and_governed_acquisition", + }, + provider_adapter_key="mdk_bcp_bathymetry", + source_url="https://www.vlaanderen.be/datavindplaats", + default_crs="EPSG:3812", + default_units="metres", + geographic_coverage={"zone": "Belgian North Sea"}, + usage_policy=_VECTOR_CONTEXT, + ingest_status="not_configured", + known_limitations=( + "A BCP coverage probe is discovery evidence, not a usable bathymetry dataset.", + "Only a successful governed acquisition with an immutable response checksum may create a source snapshot.", + ), + ), + ) +} + + +class SourceRegistryService: + """Fail-closed registry operations for governed importers and validators. + + All methods intentionally flush but do not commit. The caller owns the + dataset/import transaction, so a source snapshot, validation result and + dataset write can be rolled back together. + """ + + @staticmethod + def normalize_source_key(source_key: str) -> str: + normalized = source_key.strip().lower() + if not normalized or len(normalized) > 120: + raise AppError( + code="SOURCE_REGISTRY_KEY_INVALID", + message="Source registry key must be a non-empty value up to 120 characters", + status_code=422, + ) + return normalized + + @classmethod + def definition_for(cls, source_key: str) -> SourceRegistryDefinition: + normalized = cls.normalize_source_key(source_key) + definition = SERVER_OWNED_SOURCE_DEFINITIONS.get(normalized) + if definition is None: + raise AppError( + code="SOURCE_REGISTRY_ENTRY_NOT_FOUND", + message="Source is not registered as a server-owned source", + details={"source_key": normalized}, + status_code=422, + ) + return definition + + @classmethod + def ensure_server_owned_source(cls, db: Session, source_key: str) -> SourceRegistry: + definition = cls.definition_for(source_key) + existing = ( + db.query(SourceRegistry) + .filter(SourceRegistry.source_key == definition.source_key) + .one_or_none() + ) + if existing is not None: + return existing + source = SourceRegistry(**definition.as_model_values()) + db.add(source) + db.flush() + return source + + @staticmethod + def normalize_ingest_key(ingest_key: str) -> str: + normalized = ingest_key.strip() + if not normalized or len(normalized) > 255: + raise AppError( + code="INGEST_KEY_INVALID", + message="Ingest key must be a non-empty value up to 255 characters", + status_code=422, + ) + return normalized + + @classmethod + def find_dataset_by_ingest_key( + cls, db: Session, project_id: UUID, ingest_key: str + ) -> Dataset | None: + normalized = cls.normalize_ingest_key(ingest_key) + return ( + db.query(Dataset) + .filter(Dataset.project_id == project_id, Dataset.ingest_key == normalized) + .one_or_none() + ) + + @classmethod + def find_dataset_version_by_ingest_key( + cls, + db: Session, + dataset_id: UUID, + ingest_key: str, + ) -> DatasetVersion | None: + normalized = cls.normalize_ingest_key(ingest_key) + return ( + db.query(DatasetVersion) + .filter( + DatasetVersion.dataset_id == dataset_id, + DatasetVersion.ingest_key == normalized, + ) + .one_or_none() + ) + + @staticmethod + def _validate_status(value: str, allowed: set[str], field_name: str) -> str: + normalized = value.strip().lower() + if normalized not in allowed: + raise AppError( + code="SOURCE_REGISTRY_STATUS_INVALID", + message=f"Unsupported {field_name}", + details={ + "field": field_name, + "value": value, + "allowed": sorted(allowed), + }, + status_code=422, + ) + return normalized + + @staticmethod + def _validate_checksum(checksum_sha256: str) -> str: + normalized = checksum_sha256.strip().lower() + if not _CHECKSUM.fullmatch(normalized): + raise AppError( + code="SOURCE_SNAPSHOT_CHECKSUM_INVALID", + message="Source snapshot checksum must be a SHA-256 hex digest", + status_code=422, + ) + return normalized + + @classmethod + def record_snapshot( + cls, + db: Session, + *, + source_key: str, + snapshot_key: str, + checksum_sha256: str, + source_version: str | None = None, + snapshot_at: datetime | None = None, + fetched_at: datetime | None = None, + source_url: str | None = None, + crs: str | None = None, + units: str | None = None, + spatial_resolution: dict[str, Any] | None = None, + temporal_coverage: dict[str, Any] | None = None, + geographic_coverage: dict[str, Any] | None = None, + observed_schema: dict[str, Any] | None = None, + freshness_status: str = "unknown", + ingest_status: str = "ingested", + known_limitations: list[str] | None = None, + snapshot_metadata: dict[str, Any] | None = None, + reuse_existing_snapshot: bool = False, + ) -> SourceSnapshot: + """Record immutable source evidence, or reuse an identical snapshot. + + ``fetched_at`` belongs to the immutable source snapshot, whereas a + Dataset's ``imported_at`` records each local ingestion event. A + governed importer may therefore replay an already-known source + snapshot for another project. In that narrow replay mode the existing + ``fetched_at`` is retained; every other evidence field is still + required to be identical and the row is never updated. + """ + normalized_key = snapshot_key.strip() + if not normalized_key or len(normalized_key) > 255: + raise AppError( + code="SOURCE_SNAPSHOT_KEY_INVALID", + message="Snapshot key must be a non-empty value up to 255 characters", + status_code=422, + ) + source = cls.ensure_server_owned_source(db, source_key) + checksum = cls._validate_checksum(checksum_sha256) + normalized_freshness = cls._validate_status( + freshness_status, _FRESHNESS_STATUSES, "freshness_status" + ) + normalized_ingest = cls._validate_status( + ingest_status, _SNAPSHOT_INGEST_STATUSES, "ingest_status" + ) + existing = ( + db.query(SourceSnapshot) + .filter( + SourceSnapshot.source_registry_id == source.id, + SourceSnapshot.snapshot_key == normalized_key, + ) + .one_or_none() + ) + if existing is not None: + immutable_values = { + "checksum_sha256": checksum, + "source_version": source_version.strip() if source_version else None, + "snapshot_at": snapshot_at, + "source_url": source_url.strip() if source_url else None, + "crs": crs.strip() if crs else None, + "units": units.strip() if units else None, + "spatial_resolution_json": ( + dict(spatial_resolution or {"status": "unknown"}) + if spatial_resolution is not None + else None + ), + "temporal_coverage_json": ( + dict(temporal_coverage or {"status": "unknown"}) + if temporal_coverage is not None + else None + ), + "geographic_coverage_json": ( + dict(geographic_coverage or {"status": "unknown"}) + if geographic_coverage is not None + else None + ), + "observed_schema_json": ( + dict(observed_schema or {"status": "unknown"}) + if observed_schema is not None + else None + ), + "known_limitations_json": list(known_limitations) + if known_limitations is not None + else None, + "snapshot_metadata_json": dict(snapshot_metadata) + if snapshot_metadata is not None + else None, + } + if not reuse_existing_snapshot: + immutable_values["fetched_at"] = fetched_at + conflicts = { + field_name: { + "existing": getattr(existing, field_name), + "incoming": incoming, + } + for field_name, incoming in immutable_values.items() + if incoming is not None and getattr(existing, field_name) != incoming + } + if conflicts: + raise AppError( + code="SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT", + message="Existing source snapshot key has different immutable evidence", + details={ + "source_key": source.source_key, + "snapshot_key": normalized_key, + "conflicting_fields": sorted(conflicts), + }, + status_code=409, + ) + return existing + + snapshot = SourceSnapshot( + source_registry_id=source.id, + snapshot_key=normalized_key, + source_version=source_version.strip() if source_version else None, + snapshot_at=snapshot_at, + fetched_at=fetched_at, + source_url=source_url.strip() if source_url else None, + checksum_sha256=checksum, + crs=crs.strip() if crs else None, + units=units.strip() if units else None, + spatial_resolution_json=dict(spatial_resolution or {"status": "unknown"}), + temporal_coverage_json=dict(temporal_coverage or {"status": "unknown"}), + geographic_coverage_json=dict(geographic_coverage or {"status": "unknown"}), + observed_schema_json=dict(observed_schema or {"status": "unknown"}), + freshness_status=normalized_freshness, + ingest_status=normalized_ingest, + known_limitations_json=list(known_limitations or []), + snapshot_metadata_json=dict(snapshot_metadata or {}), + ) + db.add(snapshot) + db.flush() + return snapshot + + @classmethod + def bind_dataset_provenance( + cls, + dataset: Dataset, + *, + source: SourceRegistry, + snapshot: SourceSnapshot, + data_contract_key: str, + data_contract_version: str, + validation_status: str, + provenance_status: str, + lineage_status: str, + ) -> Dataset: + cls._validate_binding( + source=source, + snapshot=snapshot, + data_contract_key=data_contract_key, + data_contract_version=data_contract_version, + validation_status=validation_status, + provenance_status=provenance_status, + lineage_status=lineage_status, + ) + dataset.source_registry_id = source.id + dataset.source_snapshot_id = snapshot.id + dataset.data_contract_key = data_contract_key.strip() + dataset.data_contract_version = data_contract_version.strip() + dataset.validation_status = validation_status.strip().lower() + dataset.provenance_status = provenance_status.strip().lower() + dataset.lineage_status = lineage_status.strip().lower() + return dataset + + @classmethod + def bind_dataset_version_provenance( + cls, + dataset_version: DatasetVersion, + *, + source: SourceRegistry, + snapshot: SourceSnapshot, + data_contract_key: str, + data_contract_version: str, + validation_status: str, + provenance_status: str, + lineage_status: str, + ) -> DatasetVersion: + cls._validate_binding( + source=source, + snapshot=snapshot, + data_contract_key=data_contract_key, + data_contract_version=data_contract_version, + validation_status=validation_status, + provenance_status=provenance_status, + lineage_status=lineage_status, + ) + dataset_version.source_registry_id = source.id + dataset_version.source_snapshot_id = snapshot.id + dataset_version.data_contract_key = data_contract_key.strip() + dataset_version.data_contract_version = data_contract_version.strip() + dataset_version.validation_status = validation_status.strip().lower() + dataset_version.provenance_status = provenance_status.strip().lower() + dataset_version.lineage_status = lineage_status.strip().lower() + return dataset_version + + @classmethod + def _validate_binding( + cls, + *, + source: SourceRegistry, + snapshot: SourceSnapshot, + data_contract_key: str, + data_contract_version: str, + validation_status: str, + provenance_status: str, + lineage_status: str, + ) -> None: + if source.id != snapshot.source_registry_id: + raise AppError( + code="SOURCE_SNAPSHOT_REGISTRY_MISMATCH", + message="Source snapshot does not belong to the selected source registry entry", + status_code=409, + ) + if not data_contract_key.strip() or not data_contract_version.strip(): + raise AppError( + code="DATA_CONTRACT_IDENTITY_REQUIRED", + message="Dataset provenance binding requires a contract key and version", + status_code=422, + ) + cls._validate_status( + validation_status, _VALIDATION_STATUSES, "validation_status" + ) + cls._validate_status( + provenance_status, _PROVENANCE_STATUSES, "provenance_status" + ) + cls._validate_status(lineage_status, _LINEAGE_STATUSES, "lineage_status") + + @staticmethod + def record_lineage_edge( + db: Session, + *, + parent_dataset_id: UUID, + child_dataset_id: UUID, + relation_type: str, + transformation_name: str, + parent_dataset_version_id: UUID | None = None, + child_dataset_version_id: UUID | None = None, + transformation_version: str | None = None, + parameters: dict[str, Any] | None = None, + input_checksum_sha256: str | None = None, + output_checksum_sha256: str | None = None, + ) -> DatasetLineageEdge: + if parent_dataset_id == child_dataset_id: + raise AppError( + code="DATASET_LINEAGE_SELF_REFERENCE", + message="A dataset cannot be its own lineage parent", + status_code=422, + ) + normalized_relation = relation_type.strip() + normalized_transform = transformation_name.strip() + if not normalized_relation or not normalized_transform: + raise AppError( + code="DATASET_LINEAGE_IDENTITY_REQUIRED", + message="Lineage relation type and transformation name are required", + status_code=422, + ) + normalized_input_checksum = SourceRegistryService._optional_checksum( + input_checksum_sha256 + ) + normalized_output_checksum = SourceRegistryService._optional_checksum( + output_checksum_sha256 + ) + existing = ( + db.query(DatasetLineageEdge) + .filter( + DatasetLineageEdge.parent_dataset_id == parent_dataset_id, + DatasetLineageEdge.child_dataset_id == child_dataset_id, + DatasetLineageEdge.relation_type == normalized_relation, + DatasetLineageEdge.transformation_name == normalized_transform, + ) + .one_or_none() + ) + if existing is not None: + if ( + existing.input_checksum_sha256 != normalized_input_checksum + or existing.output_checksum_sha256 != normalized_output_checksum + ): + raise AppError( + code="DATASET_LINEAGE_IMMUTABILITY_CONFLICT", + message="Existing lineage edge has different artifact checksums", + status_code=409, + ) + return existing + + if SourceRegistryService._would_create_lineage_cycle( + db, + parent_dataset_id=parent_dataset_id, + child_dataset_id=child_dataset_id, + ): + raise AppError( + code="DATASET_LINEAGE_CYCLE_DETECTED", + message="The proposed lineage edge would make the dataset lineage graph cyclic", + details={ + "parent_dataset_id": str(parent_dataset_id), + "child_dataset_id": str(child_dataset_id), + }, + status_code=409, + ) + + edge = DatasetLineageEdge( + parent_dataset_id=parent_dataset_id, + child_dataset_id=child_dataset_id, + parent_dataset_version_id=parent_dataset_version_id, + child_dataset_version_id=child_dataset_version_id, + relation_type=normalized_relation, + transformation_name=normalized_transform, + transformation_version=transformation_version.strip() + if transformation_version + else None, + parameters_json=dict(parameters or {}), + input_checksum_sha256=normalized_input_checksum, + output_checksum_sha256=normalized_output_checksum, + ) + db.add(edge) + db.flush() + return edge + + @staticmethod + def _would_create_lineage_cycle( + db: Session, + *, + parent_dataset_id: UUID, + child_dataset_id: UUID, + ) -> bool: + """Return whether ``parent -> child`` would close an existing DAG path. + + A lineage edge is directed from an input/parent Dataset to its derived + child. Adding ``parent -> child`` is unsafe precisely when ``parent`` + is already reachable downstream from ``child``. Querying one indexed + parent frontier at a time avoids loading unrelated lineage history and + the visited set makes a pre-existing corrupt cycle finite to inspect. + The graph-size limit is itself fail-closed: a graph too large to audit + may not receive a new edge until it is investigated. + """ + + frontier = {child_dataset_id} + visited: set[UUID] = set() + while frontier: + if parent_dataset_id in frontier: + return True + current = frontier - visited + if not current: + return False + visited.update(current) + if len(visited) > _MAX_LINEAGE_GRAPH_NODES: + raise AppError( + code="DATASET_LINEAGE_GRAPH_LIMIT_EXCEEDED", + message="Dataset lineage graph exceeds the safe traversal limit", + details={"max_nodes": _MAX_LINEAGE_GRAPH_NODES}, + status_code=409, + ) + edges = ( + db.query(DatasetLineageEdge) + .filter(DatasetLineageEdge.parent_dataset_id.in_(current)) + .all() + ) + frontier = { + edge.child_dataset_id + for edge in edges + if edge.child_dataset_id not in visited + } + return False + + @classmethod + def _lineage_descendant_dataset_ids( + cls, + db: Session, + *, + root_dataset_ids: set[UUID], + ) -> set[UUID]: + """Return every reachable child Dataset, including the supplied roots. + + A quarantine is a lineage safety event, not merely a status update on + the immediately observed asset. Traversal follows the same directed + parent-to-child relation used by the cycle guard and remains bounded so + a corrupt graph cannot make a quarantine operation unobservable. + PostgreSQL enforces the equivalent recursive propagation for all + persisted writes; this application-side traversal keeps the service + fail-closed for normal ORM callers and focused in-memory test doubles. + """ + + affected = set(root_dataset_ids) + frontier = set(root_dataset_ids) + visited: set[UUID] = set() + while frontier: + current = frontier - visited + if not current: + break + visited.update(current) + if len(visited) > _MAX_LINEAGE_GRAPH_NODES: + raise AppError( + code="DATASET_LINEAGE_GRAPH_LIMIT_EXCEEDED", + message="Dataset lineage graph exceeds the safe traversal limit during quarantine propagation", + details={"max_nodes": _MAX_LINEAGE_GRAPH_NODES}, + status_code=409, + ) + edges = ( + db.query(DatasetLineageEdge) + .filter(DatasetLineageEdge.parent_dataset_id.in_(current)) + .all() + ) + frontier = { + edge.child_dataset_id + for edge in edges + if edge.child_dataset_id not in visited + } + affected.update(frontier) + return affected + + @staticmethod + def _mark_dataset_quarantined(dataset: Dataset) -> None: + """Invalidate every Dataset-level consumption gate in one place.""" + + dataset.status = "quarantined" + dataset.quarantine_status = "quarantined" + dataset.validation_status = "failed" + dataset.provenance_status = "incomplete" + dataset.lineage_status = "incomplete" + + @staticmethod + def _mark_dataset_version_quarantined(dataset_version: DatasetVersion) -> None: + """Invalidate a version that belongs to a quarantined Dataset lineage.""" + + dataset_version.validation_status = "failed" + dataset_version.provenance_status = "incomplete" + dataset_version.lineage_status = "incomplete" + + @classmethod + def _propagate_lineage_quarantine( + cls, + db: Session, + *, + root_datasets: tuple[Dataset, ...], + ) -> tuple[Dataset, ...]: + """Quarantine root datasets and all immutable downstream derivatives. + + A source snapshot can be bound by more than one Dataset, so callers + supply every directly affected root. Derived descendants retain their + own source snapshot evidence, but their Dataset and DatasetVersion + state becomes non-consumable until a governed re-ingest establishes a + new valid lineage. + """ + + datasets_by_id: dict[UUID, Dataset] = {} + transient_roots: list[Dataset] = [] + for candidate in root_datasets: + dataset_id = getattr(candidate, "id", None) + if dataset_id is None: + transient_roots.append(candidate) + else: + datasets_by_id[dataset_id] = candidate + + affected_ids = ( + cls._lineage_descendant_dataset_ids( + db, + root_dataset_ids=set(datasets_by_id), + ) + if datasets_by_id + else set() + ) + if affected_ids: + for candidate in ( + db.query(Dataset).filter(Dataset.id.in_(affected_ids)).all() + ): + datasets_by_id[candidate.id] = candidate + + affected_datasets = tuple((*datasets_by_id.values(), *transient_roots)) + for candidate in affected_datasets: + cls._mark_dataset_quarantined(candidate) + + if affected_ids: + for dataset_version in ( + db.query(DatasetVersion) + .filter(DatasetVersion.dataset_id.in_(affected_ids)) + .all() + ): + cls._mark_dataset_version_quarantined(dataset_version) + return affected_datasets + + @classmethod + def quarantine_dataset( + cls, + db: Session, + *, + stage: str, + reason_code: str, + dataset: Dataset | None = None, + dataset_version: DatasetVersion | None = None, + source_snapshot: SourceSnapshot | None = None, + details: dict[str, Any] | None = None, + artifact_path: str | None = None, + artifact_checksum_sha256: str | None = None, + ) -> DatasetQuarantine: + normalized_stage = stage.strip() + normalized_reason = reason_code.strip() + if not normalized_stage or not normalized_reason: + raise AppError( + code="QUARANTINE_REASON_REQUIRED", + message="Quarantine stage and reason code are required", + status_code=422, + ) + if dataset is None and dataset_version is None and source_snapshot is None: + raise AppError( + code="QUARANTINE_TARGET_REQUIRED", + message="Quarantine requires a dataset, dataset version or source snapshot", + status_code=422, + ) + if dataset is None and dataset_version is not None: + dataset = cls._dataset_for_version(db, dataset_version) + if dataset is None: + # A version-only quarantine without the owning Dataset would + # leave the Dataset consumable: every current consumption + # boundary evaluates Dataset, not DatasetVersion. Refuse the + # partial state rather than quietly recording an ineffective + # quarantine. + raise AppError( + code="QUARANTINE_PARENT_DATASET_NOT_FOUND", + message="A version quarantine requires its owning Dataset so the quarantine can propagate.", + details={ + "dataset_version_id": str(dataset_version.id), + "dataset_id": str(dataset_version.dataset_id), + }, + status_code=409, + ) + root_datasets: list[Dataset] = [dataset] if dataset is not None else [] + if source_snapshot is not None: + # A snapshot is immutable shared evidence. Its direct bindings are + # roots too, so invalidating one artifact cannot leave a sibling or + # any downstream derivative consumable through a stale lineage. + root_datasets.extend( + db.query(Dataset) + .filter(Dataset.source_snapshot_id == source_snapshot.id) + .all() + ) + cls._propagate_lineage_quarantine(db, root_datasets=tuple(root_datasets)) + if dataset_version is not None: + cls._mark_dataset_version_quarantined(dataset_version) + if source_snapshot is not None: + # A quarantined source artifact must no longer satisfy the + # authoritative-validation eligibility check for any linked data. + source_snapshot.ingest_status = "quarantined" + + record = DatasetQuarantine( + dataset_id=dataset.id if dataset is not None else None, + dataset_version_id=dataset_version.id + if dataset_version is not None + else None, + source_snapshot_id=source_snapshot.id + if source_snapshot is not None + else None, + stage=normalized_stage, + reason_code=normalized_reason, + details_json=dict(details or {}), + artifact_path=artifact_path, + artifact_checksum_sha256=cls._optional_checksum(artifact_checksum_sha256), + status="quarantined", + ) + db.add(record) + db.flush() + return record + + @staticmethod + def _dataset_for_version( + db: Session, dataset_version: DatasetVersion + ) -> Dataset | None: + """Resolve the Dataset that must share a version's quarantine state.""" + + related = getattr(dataset_version, "dataset", None) + if related is not None: + return related + getter = getattr(db, "get", None) + if callable(getter): + resolved = getter(Dataset, dataset_version.dataset_id) + if resolved is not None: + return resolved + # Use a normal ORM query as a final path for sessions where the + # relationship is deliberately not loaded. This also keeps bounded + # in-memory persistence fixtures representative of production. + return ( + db.query(Dataset) + .filter(Dataset.id == dataset_version.dataset_id) + .one_or_none() + ) + + @staticmethod + def validation_authority_for_task(source: SourceRegistry, task: str) -> str | None: + policy = ( + source.usage_policy_json + if isinstance(source.usage_policy_json, dict) + else {} + ) + authority = policy.get("validation_authority") + if not isinstance(authority, dict): + return None + value = authority.get(task) + return str(value) if value else None + + @classmethod + def is_dataset_eligible_for_authoritative_validation( + cls, + dataset: Dataset, + *, + source: SourceRegistry, + snapshot: SourceSnapshot, + task: str, + ) -> bool: + policy = ( + source.usage_policy_json + if isinstance(source.usage_policy_json, dict) + else {} + ) + return bool( + source.classification == "authoritative" + and policy.get("ground_truth_allowed") is True + and cls.validation_authority_for_task(source, task) == "primary" + and dataset.status == "ready" + and dataset.quarantine_status == "not_quarantined" + and dataset.validation_status == "passed" + and dataset.provenance_status == "complete" + and dataset.lineage_status in {"complete", "not_applicable"} + and dataset.source_registry_id == source.id + and dataset.source_snapshot_id == snapshot.id + and snapshot.source_registry_id == source.id + and snapshot.ingest_status == "ingested" + and bool( + snapshot.checksum_sha256 + and _CHECKSUM.fullmatch(snapshot.checksum_sha256) + ) + ) + + @classmethod + def _optional_checksum(cls, checksum_sha256: str | None) -> str | None: + if checksum_sha256 is None: + return None + return cls._validate_checksum(checksum_sha256) diff --git a/backend/app/services/vector_feature_service.py b/backend/app/services/vector_feature_service.py index a731926e..4e2e7712 100644 --- a/backend/app/services/vector_feature_service.py +++ b/backend/app/services/vector_feature_service.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import math from pathlib import Path from typing import Any, Iterable from uuid import UUID @@ -8,6 +9,7 @@ from uuid import UUID from geoalchemy2.functions import ST_Intersects, ST_MakeEnvelope from geoalchemy2.shape import from_shape from geoalchemy2.shape import to_shape +from pyproj import CRS, Transformer from shapely.geometry import box, mapping, shape from shapely.ops import transform as transform_geometry from shapely.validation import make_valid @@ -268,7 +270,14 @@ class VectorFeatureService: return ("municipality", municipality) if municipality else None @staticmethod - def _feature_row(dataset_id: UUID, feature: dict[str, Any], index: int, feature_class: str | None) -> VectorFeature | None: + def _feature_row( + dataset_id: UUID, + feature: dict[str, Any], + index: int, + feature_class: str | None, + *, + source_crs: str = "EPSG:4326", + ) -> VectorFeature | None: geometry_payload = feature.get("geometry") if geometry_payload is None: return None @@ -282,8 +291,7 @@ class VectorFeatureService: geometry = make_valid(geometry) if geometry.is_empty or not geometry.is_valid: raise AppError(code="INVALID_GEOMETRY", message=f"Invalid feature geometry at index {index}", status_code=400) - if geometry.has_z: - geometry = transform_geometry(lambda x, y, z=None: (x, y), geometry) + geometry = VectorFeatureService._canonical_geometry(geometry, source_crs=source_crs, index=index) properties = feature.get("properties") if isinstance(feature.get("properties"), dict) else {} source_feature_id = feature.get("id") @@ -298,6 +306,109 @@ class VectorFeatureService: geometry=from_shape(geometry, srid=4326), ) + @staticmethod + def _canonical_geometry(geometry: Any, *, source_crs: str, index: int): + """Transform one source geometry to canonical EPSG:4326 safely.""" + if geometry.has_z: + geometry = transform_geometry(lambda x, y, z=None: (x, y), geometry) + + # VectorFeature is deliberately canonical WGS84 storage. Treating + # Lambert or another source CRS as EPSG:4326 produces geometries that + # look syntactically valid but are spatially wrong. All governed + # import callers therefore pass the declared source CRS; the default + # only preserves compatibility for legacy, already-WGS84 call sites. + try: + parsed_source_crs = CRS.from_user_input(source_crs) + target_crs = CRS.from_epsg(4326) + except Exception as exc: + raise AppError( + code="INVALID_DATASET_CRS", + message=f"Invalid source CRS for vector feature at index {index}", + details={"source_crs": source_crs}, + status_code=400, + ) from exc + if not parsed_source_crs.equals(target_crs): + try: + transformer = Transformer.from_crs(parsed_source_crs, target_crs, always_xy=True) + geometry = transform_geometry(transformer.transform, geometry) + except Exception as exc: + raise AppError( + code="VECTOR_CRS_TRANSFORMATION_FAILED", + message=f"Could not transform vector feature at index {index} to EPSG:4326", + details={"source_crs": source_crs}, + status_code=400, + ) from exc + if geometry.is_empty or not geometry.is_valid: + geometry = make_valid(geometry) + if geometry.is_empty or not geometry.is_valid: + raise AppError( + code="INVALID_GEOMETRY", + message=f"Invalid transformed feature geometry at index {index}", + status_code=400, + ) + min_x, min_y, max_x, max_y = geometry.bounds + if ( + not all(math.isfinite(value) for value in (min_x, min_y, max_x, max_y)) + or min_x < -180 + or max_x > 180 + or min_y < -90 + or max_y > 90 + ): + raise AppError( + code="VECTOR_GEOMETRY_OUTSIDE_EPSG4326", + message=f"Transformed feature geometry at index {index} is outside EPSG:4326 bounds", + details={"source_crs": source_crs, "bounds": [min_x, min_y, max_x, max_y]}, + status_code=400, + ) + return geometry + + @staticmethod + def canonicalize_geojson_payload(payload: dict[str, Any], *, source_crs: str) -> dict[str, Any]: + """Return a canonical-WGS84 feature collection without losing source attributes. + + Callers use this payload for validation, spatial indexing and + map-safe consumption storage. A non-canonical source file, when + retained, belongs to explicit provenance evidence rather than the + Dataset consumption path; no implicit CRS assumption is recorded. + """ + features = payload.get("features") + if payload.get("type") != "FeatureCollection" or not isinstance(features, list): + raise AppError(code="INVALID_GEOJSON", message="GeoJSON payload must be a FeatureCollection", status_code=400) + canonical_features: list[dict[str, Any]] = [] + for index, feature in enumerate(features): + if not isinstance(feature, dict): + raise AppError(code="INVALID_GEOJSON", message=f"Feature {index} must be an object", status_code=400) + canonical_feature = dict(feature) + geometry_payload = feature.get("geometry") + if geometry_payload is not None: + try: + geometry = shape(geometry_payload) + except Exception as exc: + raise AppError( + code="INVALID_GEOJSON", + message=f"Invalid feature geometry at index {index}", + status_code=400, + ) from exc + if not geometry.is_empty: + if not geometry.is_valid: + geometry = make_valid(geometry) + if geometry.is_empty or not geometry.is_valid: + raise AppError( + code="INVALID_GEOMETRY", + message=f"Invalid feature geometry at index {index}", + status_code=400, + ) + canonical_feature["geometry"] = mapping( + VectorFeatureService._canonical_geometry(geometry, source_crs=source_crs, index=index) + ) + canonical_features.append(canonical_feature) + return { + **{key: value for key, value in payload.items() if key not in {"crs", "features"}}, + "type": "FeatureCollection", + "crs": {"type": "name", "properties": {"name": "EPSG:4326"}}, + "features": canonical_features, + } + @staticmethod def _normalize_selection_bbox(bbox: dict[str, Any]) -> dict[str, float | str]: try: @@ -882,6 +993,7 @@ class VectorFeatureService: feature_class: str | None = None, *, commit: bool = True, + source_crs: str = "EPSG:4326", ) -> list[VectorFeature]: features = payload.get("features") if payload.get("type") != "FeatureCollection" or not isinstance(features, list): @@ -891,7 +1003,13 @@ class VectorFeatureService: for index, feature in enumerate(features): if not isinstance(feature, dict): raise AppError(code="INVALID_GEOJSON", message=f"Feature {index} must be an object", status_code=400) - row = VectorFeatureService._feature_row(dataset_id, feature, index, feature_class) + row = VectorFeatureService._feature_row( + dataset_id, + feature, + index, + feature_class, + source_crs=source_crs, + ) if row is None: continue db.add(row) @@ -910,6 +1028,7 @@ class VectorFeatureService: feature_class: str | None = None, *, batch_size: int = 1000, + source_crs: str = "EPSG:4326", ) -> int: if batch_size <= 0: raise ValueError("batch_size must be positive") @@ -942,7 +1061,13 @@ class VectorFeatureService: message=f"Feature {index} in {path.name} must be an object", status_code=400, ) - row = VectorFeatureService._feature_row(dataset_id, feature, index, feature_class) + row = VectorFeatureService._feature_row( + dataset_id, + feature, + index, + feature_class, + source_crs=source_crs, + ) if row is None: continue if row.source_feature_id: diff --git a/backend/app/services/vector_operations_service.py b/backend/app/services/vector_operations_service.py index 372303bb..2efa255b 100644 --- a/backend/app/services/vector_operations_service.py +++ b/backend/app/services/vector_operations_service.py @@ -3,13 +3,17 @@ from __future__ import annotations import json import uuid from datetime import datetime, timezone +from hashlib import sha256 from pathlib import Path +import re from typing import Any from geoalchemy2.shape import to_shape +from pyproj import CRS, Transformer from shapely.geometry import GeometryCollection, MultiPolygon, shape from shapely.geometry.base import BaseGeometry from shapely.geometry import mapping +from shapely.ops import transform as shapely_transform from shapely.ops import unary_union from shapely.validation import make_valid from sqlalchemy.orm import Session @@ -18,12 +22,16 @@ from app.core.errors import AppError from app.models import Area, Dataset, DatasetVersion from app.schemas.dataset import DatasetCreateResponse from app.schemas.operations import VectorOperationResult +from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService from app.services.geojson_service import parse_geojson_payload from app.services.storage_service import StorageService from app.services.vector_feature_service import VectorFeatureService class VectorOperationsService: + CANONICAL_VECTOR_CRS = "EPSG:4326" + _CHECKSUM_SHA256 = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE) + @staticmethod def _require_vector_dataset(dataset: Dataset) -> None: if dataset.dataset_type not in {"vector", "geojson"}: @@ -37,7 +45,8 @@ class VectorOperationsService: if not path.exists(): raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404) try: - payload = json.loads(path.read_text(encoding="utf-8")) + stored_bytes = path.read_bytes() + payload = json.loads(stored_bytes.decode("utf-8")) except Exception as exc: raise AppError(code="INVALID_GEOJSON", message="Stored dataset is not valid JSON", status_code=400) from exc @@ -47,6 +56,55 @@ class VectorOperationsService: features = payload.get("features") if not isinstance(features, list): raise AppError(code="INVALID_GEOJSON", message="Dataset payload is missing features", status_code=400) + + # The normal Dataset storage path is a consumption artifact, not a + # provenance source archive. Refuse projected/original source bytes + # here rather than letting a spatial operation interpret them as + # canonical map coordinates. + raw_crs = payload.get("crs") + if isinstance(raw_crs, dict): + crs_properties = raw_crs.get("properties") + raw_crs = crs_properties.get("name") if isinstance(crs_properties, dict) else None + stored_crs = str(raw_crs or VectorOperationsService.CANONICAL_VECTOR_CRS).strip().upper() + dataset_crs = str(dataset.crs or "").strip().upper() + if stored_crs != VectorOperationsService.CANONICAL_VECTOR_CRS or ( + dataset_crs and dataset_crs != VectorOperationsService.CANONICAL_VECTOR_CRS + ): + raise AppError( + code="DATASET_STORAGE_CRS_MISMATCH", + message="Vector operations require canonical EPSG:4326 dataset storage.", + details={ + "stored_crs": raw_crs or VectorOperationsService.CANONICAL_VECTOR_CRS, + "dataset_crs": dataset.crs, + "expected_crs": VectorOperationsService.CANONICAL_VECTOR_CRS, + }, + status_code=409, + ) + + expected_checksum = str(dataset.checksum_sha256 or "").strip().lower() + actual_checksum = sha256(stored_bytes).hexdigest() + governed_artifact = bool( + getattr(dataset, "data_contract_key", None) + or ( + isinstance(getattr(dataset, "metadata_json", None), dict) + and dataset.metadata_json.get("canonical_storage_crs") + ) + ) + if expected_checksum and VectorOperationsService._CHECKSUM_SHA256.fullmatch(expected_checksum): + if expected_checksum != actual_checksum: + raise AppError( + code="DATASET_STORAGE_CHECKSUM_MISMATCH", + message="Vector dataset storage no longer matches its validated checksum.", + details={"expected_checksum_sha256": expected_checksum, "actual_checksum_sha256": actual_checksum}, + status_code=409, + ) + elif governed_artifact: + raise AppError( + code="DATASET_STORAGE_CHECKSUM_UNVERIFIABLE", + message="Governed vector storage requires a valid SHA-256 checksum before use.", + details={"checksum_sha256": dataset.checksum_sha256}, + status_code=409, + ) return payload, [feature for feature in features if isinstance(feature, dict)] @staticmethod @@ -73,6 +131,25 @@ class VectorOperationsService: raise AppError(code="EMPTY_VECTOR_DATASET", message="Dataset has no parseable geometries", status_code=422) return geometries + @staticmethod + def _buffer_in_metres(geometry: BaseGeometry, distance_m: float, source_crs: str) -> BaseGeometry: + """Buffer in a Belgian projected CRS, never in angular degrees.""" + + try: + input_crs = CRS.from_user_input(source_crs) + metric_crs = CRS.from_epsg(31370) + if input_crs == metric_crs: + return geometry.buffer(distance_m) + forward = Transformer.from_crs(input_crs, metric_crs, always_xy=True) + backward = Transformer.from_crs(metric_crs, input_crs, always_xy=True) + return shapely_transform(backward.transform, shapely_transform(forward.transform, geometry).buffer(distance_m)) + except Exception as exc: + raise AppError( + code="INVALID_CRS", + message="A valid explicit CRS is required for metre-based vector buffering.", + status_code=400, + ) from exc + @staticmethod def inspect(db: Session, dataset_id: uuid.UUID) -> VectorOperationResult: dataset = db.get(Dataset, dataset_id) @@ -94,7 +171,7 @@ class VectorOperationsService: feature_count=len(geometries), geometry_type_summary=geometry_type_summary, bounds_json={"min_x": float(bounds[0]), "min_y": float(bounds[1]), "max_x": float(bounds[2]), "max_y": float(bounds[3])}, - crs=payload.get("crs") if isinstance(payload.get("crs"), str) else dataset.crs, + crs=VectorOperationsService.CANONICAL_VECTOR_CRS, ) @staticmethod @@ -184,9 +261,13 @@ class VectorOperationsService: if distance_m <= 0: raise AppError(code="INVALID_PARAMETERS", message="distance_m must be greater than 0", status_code=400) - _, features = VectorOperationsService._load_dataset_payload(source_dataset) + payload, features = VectorOperationsService._load_dataset_payload(source_dataset) + source_crs = VectorOperationsService.CANONICAL_VECTOR_CRS geometries = VectorOperationsService._extract_geometries(features) - buffered_features = [(feature, geometry.buffer(distance_m)) for feature, geometry in geometries] + buffered_features = [ + (feature, VectorOperationsService._buffer_in_metres(geometry, distance_m, source_crs)) + for feature, geometry in geometries + ] output_features: list[dict[str, Any]] = [] for feature, geometry in buffered_features: @@ -431,7 +512,14 @@ class VectorOperationsService: if not output_name_value.strip(): output_name_value = f"{default_name}.geojson" - stored = json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + # All current governed vector storage is EPSG:4326. A legacy source + # with another CRS is not relabelled here: the derived contract will + # quarantine the result instead of placing non-WGS84 coordinates on + # the map as if they were WGS84. + output_crs = VectorOperationsService.CANONICAL_VECTOR_CRS + output_feature_collection = dict(feature_collection) + output_feature_collection["crs"] = output_crs + stored = json.dumps(output_feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8") storage_info = StorageService.persist_dataset_file( project_id=str(source_dataset.project_id), dataset_id=str(derived_id), @@ -441,7 +529,7 @@ class VectorOperationsService: content_type="application/geo+json", ) - metadata = parse_geojson_payload(json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":"))) + metadata = parse_geojson_payload(json.dumps(output_feature_collection, ensure_ascii=False, separators=(",", ":"))) if metadata_extra: metadata.update(metadata_extra) derived_dataset = Dataset( @@ -452,7 +540,7 @@ class VectorOperationsService: dataset_type="vector", source=f"operation:{operation}", dataset_role=dataset_role, - source_name=source_name, + source_name=source_name or "derived", source_metadata=source_metadata, provenance_metadata=provenance_metadata, imported_at=datetime.now(timezone.utc), @@ -478,29 +566,44 @@ class VectorOperationsService: resolution_json=metadata.get("resolution_json"), bands_json=metadata.get("bands_json"), metadata_json=metadata, - status="ready", + status="validating", ) db.add(derived_dataset) - db.add( - DatasetVersion( - dataset_id=derived_dataset.id, - version=1, - storage_path=derived_dataset.storage_path, - source_version=derived_dataset.source_version, - observed_at=derived_dataset.observed_at, - valid_from=derived_dataset.valid_from, - valid_to=derived_dataset.valid_to, - checksum_sha256=derived_dataset.checksum_sha256, - source_metadata=derived_dataset.source_metadata, - provenance_metadata=derived_dataset.provenance_metadata, - ) + dataset_version = DatasetVersion( + dataset_id=derived_dataset.id, + version=1, + storage_path=derived_dataset.storage_path, + source_version=derived_dataset.source_version, + observed_at=derived_dataset.observed_at, + valid_from=derived_dataset.valid_from, + valid_to=derived_dataset.valid_to, + checksum_sha256=derived_dataset.checksum_sha256, + source_metadata=derived_dataset.source_metadata, + provenance_metadata=derived_dataset.provenance_metadata, ) - db.commit() - db.refresh(derived_dataset) - if persist_vector_features: + db.add(dataset_version) + derived_source_key = "map_selection" if source_name == "map_selection" else "derived" + is_ready = DerivedDatasetGovernanceService.govern_vector( + db, + dataset=derived_dataset, + dataset_version=dataset_version, + feature_collection=output_feature_collection, + source_key=derived_source_key, + operation=f"vector.{operation}", + parent_dataset=source_dataset, + operation_parameters={ + "operation": operation, + "output_name": output_name_value, + **(metadata_extra or {}), + }, + ) + if persist_vector_features and is_ready: VectorFeatureService.persist_geojson_features( db=db, dataset_id=derived_dataset.id, - payload=feature_collection, + payload=output_feature_collection, + commit=False, ) + db.commit() + db.refresh(derived_dataset) return derived_id diff --git a/backend/app/services/yolo_preflight_service.py b/backend/app/services/yolo_preflight_service.py index b8a3bd47..2bcc8378 100644 --- a/backend/app/services/yolo_preflight_service.py +++ b/backend/app/services/yolo_preflight_service.py @@ -9,6 +9,7 @@ from app.core.config import Settings, get_settings from app.core.errors import AppError from app.services.detection_service import DetectionService from app.services.model_asset_catalog_service import ModelAssetCatalogService +from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService from app.services.yolo_adapter import YoloDetectionAdapter @@ -41,6 +42,8 @@ class YoloPreflightService: "accelerator_ready": None, "model_path_set": None, "model_file_exists": None, + "model_provenance_manifest_path": None, + "model_provenance_valid": None, "model_load_requested": check_model_load, "model_load_ok": None, "manifest_path_set": None, @@ -98,6 +101,29 @@ class YoloPreflightService: result["message"] = "YOLO_MODEL_PATH does not point to an existing local model file." return result + result["checks"]["model_provenance_manifest_path"] = str( + RuntimeModelProvenanceService.manifest_path_for_model(model_path) + ) + try: + RuntimeModelProvenanceService.validate_for_runtime( + model_path=model_path, + model_id=resolved_settings.yolo_model_id, + task_type="object_detection", + expected_model_version=resolved_settings.yolo_model_version, + allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"), + ) + except AppError as exc: + result["checks"]["model_provenance_valid"] = False + result["status"] = "contract_incomplete" + result["message"] = ( + "Configured YOLO weights are not runnable until their immutable runtime provenance sidecar validates: " + f"{exc.message}" + ) + result["error_code"] = exc.code + result["details"] = exc.details + return result + result["checks"]["model_provenance_valid"] = True + if check_model_load: try: yolo_adapter_class(resolved_settings).load_model(model_path) diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 00000000..05b47f10 --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1,19 @@ +"""Canonical backend-test import boundary. + +Pytest is intentionally runnable from ``backend/`` because that is the CI +entrypoint. Some contract tests exercise repository-level deterministic +scripts; put the canonical repository root ahead of the legacy +``backend/scripts`` helper directory so those imports resolve to the code that +is actually shipped by the root Docker build. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + + +REPOSITORY_ROOT = Path(__file__).resolve().parents[2] +repository_root_text = str(REPOSITORY_ROOT) +if repository_root_text not in sys.path: + sys.path.insert(0, repository_root_text) diff --git a/backend/tests/test_accuracy_phase2_foundation_audit.py b/backend/tests/test_accuracy_phase2_foundation_audit.py new file mode 100644 index 00000000..e68d323c --- /dev/null +++ b/backend/tests/test_accuracy_phase2_foundation_audit.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "run_accuracy_phase2_foundation_audit.py" +SPEC = importlib.util.spec_from_file_location("accuracy_phase2_foundation_audit", SCRIPT) +assert SPEC is not None and SPEC.loader is not None +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def test_phase2_foundation_audit_enumerates_exact_source_and_contract_policies() -> None: + payload = MODULE.collect() + + assert payload["phase"] == "P2" + assert payload["migration_revision"] == "202608010001" + assert payload["source_registry"]["definition_count"] >= 40 + assert payload["source_registry"]["required_building_policy"] == { + "grb_primary_building_validation": "primary", + "buildings_register_classification": "authoritative", + "sentinel_2_classification": "contextual", + "dhmv_classification": "authoritative", + "osm_ground_truth_allowed": False, + } + assert {(item["key"], item["version"]) for item in payload["data_contracts"]} == { + ("geointel.vector.geojson", "1.0.0"), + ("geointel.raster.geotiff", "1.0.0"), + ("geointel.label.yolo", "1.0.0"), + ("geointel.label.yolo", "1.1.0"), + ("geointel.model.pytorch", "1.0.0"), + } diff --git a/backend/tests/test_accuracy_phase2_source_registry.py b/backend/tests/test_accuracy_phase2_source_registry.py new file mode 100644 index 00000000..f082dde9 --- /dev/null +++ b/backend/tests/test_accuracy_phase2_source_registry.py @@ -0,0 +1,772 @@ +from __future__ import annotations + +from datetime import timedelta +from importlib.util import module_from_spec, spec_from_file_location +from pathlib import Path +from uuid import uuid4 + +import pytest +from sqlalchemy import CheckConstraint + +from app.core.errors import AppError +from app.models import ( + Dataset, + DatasetLineageEdge, + DatasetQuarantine, + DatasetVersion, + SourceRegistry, + SourceSnapshot, +) +from app.services.coverage_registry_service import SOURCE_DEFINITIONS +from app.services.dataset_consumption_gate_service import DatasetConsumptionGate +from app.services.dataset_service import DatasetService +from app.services.source_registry_service import ( + SERVER_OWNED_SOURCE_DEFINITIONS, + SourceRegistryService, +) + + +class _Query: + def __init__(self, session: "InMemorySession", model: type) -> None: + self.session = session + self.model = model + self.predicates = [] + + def filter(self, *predicates): + self.predicates.extend(predicates) + return self + + def one_or_none(self): + matches = self._matches() + if len(matches) > 1: + raise AssertionError( + f"Expected one {self.model.__name__}, found {len(matches)}" + ) + return matches[0] if matches else None + + def all(self): + return self._matches() + + def _matches(self): + matches = list(self.session.objects.get(self.model, [])) + for predicate in self.predicates: + field_name = predicate.left.key + expected = predicate.right.value + operator_name = getattr(predicate.operator, "__name__", "") + if operator_name == "in_op": + matches = [ + item for item in matches if getattr(item, field_name) in expected + ] + else: + matches = [ + item for item in matches if getattr(item, field_name) == expected + ] + return matches + + +class InMemorySession: + def __init__(self, *objects: object) -> None: + self.objects: dict[type, list[object]] = {} + self.added: list[object] = [] + self.flushes = 0 + for item in objects: + self._store(item) + + def query(self, model: type) -> _Query: + return _Query(self, model) + + def add(self, item: object) -> None: + if getattr(item, "id", None) is None: + setattr(item, "id", uuid4()) + self._store(item) + self.added.append(item) + + def flush(self) -> None: + self.flushes += 1 + + def _store(self, item: object) -> None: + self.objects.setdefault(type(item), []).append(item) + + +def _registry(source_key: str) -> SourceRegistry: + definition = SERVER_OWNED_SOURCE_DEFINITIONS[source_key] + return SourceRegistry(id=uuid4(), **definition.as_model_values()) + + +def _dataset() -> Dataset: + return Dataset( + id=uuid4(), + project_id=uuid4(), + name="candidate.tif", + dataset_type="raster", + source="governed", + status="ready", + validation_status="not_validated", + provenance_status="incomplete", + lineage_status="incomplete", + quarantine_status="not_quarantined", + ) + + +def test_server_owned_definitions_encode_building_authority_and_non_ground_truth_sources() -> ( + None +): + grb = SERVER_OWNED_SOURCE_DEFINITIONS["grb"] + buildings_register = SERVER_OWNED_SOURCE_DEFINITIONS[ + "digitaal_vlaanderen_buildings_addresses_register" + ] + sentinel = SERVER_OWNED_SOURCE_DEFINITIONS["sentinel_2"] + dhmv = SERVER_OWNED_SOURCE_DEFINITIONS["digitaal_vlaanderen_dhmv"] + osm = SERVER_OWNED_SOURCE_DEFINITIONS["osm"] + + assert grb.classification == "authoritative" + assert grb.usage_policy["ground_truth_allowed"] is True + assert grb.usage_policy["validation_authority"]["building_validation"] == "primary" + assert buildings_register.classification == "authoritative" + assert buildings_register.usage_policy["ground_truth_allowed"] is False + assert ( + buildings_register.usage_policy["validation_authority"]["building_validation"] + == "corroborative" + ) + assert ( + buildings_register.usage_policy["validation_authority"][ + "building_register_validation" + ] + == "primary" + ) + assert sentinel.classification == "contextual" + assert dhmv.classification == "authoritative" + assert dhmv.usage_policy["ground_truth_allowed"] is False + assert ( + dhmv.usage_policy["validation_authority"]["building_validation"] + == "corroborative" + ) + assert ( + dhmv.usage_policy["validation_authority"]["elevation_validation"] == "primary" + ) + assert osm.classification == "contextual" + assert osm.usage_policy["ground_truth_allowed"] is False + assert osm.usage_policy["automatic_ground_truth"] is False + assert osm.usage_policy["training_allowed"] is False + assert { + "ngi_adminvector", + "rbins_marine_reporting_units", + "rbins_msp_2026", + "grb", + "digitaal_vlaanderen", + "vrbg", + "digitaal_vlaanderen_buildings_addresses_register", + "digitaal_vlaanderen_orthophoto", + "spw_orthophoto", + "urbis_orthophoto", + "digitaal_vlaanderen_dhmv", + "spw_terrain", + "spw_walous_land_cover", + "spw_geoportail", + "spw_picc", + "urbis", + "vmm_flood_hazard", + "vmm_vha_bathymetry_profiles", + "dov_soil_map", + "statbel", + "waterinfo", + "agentschap_landbouw_zeevisserij_agricultural_parcels", + "sentinel_2", + "osm", + "manual", + "fixture", + "map_selection", + "derived", + "training_label", + "model", + "experimental", + "mdk_bcp_bathymetry", + }.issubset(SERVER_OWNED_SOURCE_DEFINITIONS) + + for umbrella_key in ("digitaal_vlaanderen", "spw_geoportail"): + definition = SERVER_OWNED_SOURCE_DEFINITIONS[umbrella_key] + assert definition.classification == "authoritative" + assert definition.usage_policy["ground_truth_allowed"] is False + assert definition.usage_policy["automatic_ground_truth"] is False + + assert ( + SERVER_OWNED_SOURCE_DEFINITIONS["mdk_bcp_bathymetry"].ingest_status + == "not_configured" + ) + + +def test_coverage_and_direct_adapter_source_keys_are_registry_backed() -> None: + coverage_source_keys = { + definition.contract.source_name for definition in SOURCE_DEFINITIONS + } + coverage_materialization_keys = { + source_key + for definition in SOURCE_DEFINITIONS + for source_key in definition.materialized_source_names + } + direct_adapter_source_keys = { + "digitaal_vlaanderen", + "spw_geoportail", + "mdk_bcp_bathymetry", + } + + assert ( + coverage_source_keys + | coverage_materialization_keys + | direct_adapter_source_keys + <= set(SERVER_OWNED_SOURCE_DEFINITIONS) + ) + + +def test_new_adapter_source_seed_rows_match_server_owned_registry_semantics() -> None: + migration_path = ( + Path(__file__).resolve().parents[1] + / "alembic" + / "versions" + / "202608010001_source_registry_provenance.py" + ) + spec = spec_from_file_location("phase2_source_registry_migration", migration_path) + assert spec is not None and spec.loader is not None + migration = module_from_spec(spec) + spec.loader.exec_module(migration) + seed_rows = {row["source_key"]: row for row in migration._seed_rows()} + + for source_key in ("digitaal_vlaanderen", "spw_geoportail", "mdk_bcp_bathymetry"): + expected = SERVER_OWNED_SOURCE_DEFINITIONS[source_key].as_model_values() + observed = seed_rows[source_key] + for field in ( + "source_key", + "display_name", + "classification", + "authority_name", + "authority_scope_json", + "provider_adapter_key", + "source_url", + "default_crs", + "default_units", + "geographic_coverage_json", + "usage_policy_json", + "freshness_status", + "ingest_status", + "known_limitations_json", + ): + assert observed[field] == expected[field] + + +def test_ensure_source_is_idempotent_and_rejects_caller_owned_unknown_sources() -> None: + grb = _registry("grb") + session = InMemorySession(grb) + + assert SourceRegistryService.ensure_server_owned_source(session, "GRB") is grb + assert session.added == [] + + with pytest.raises(AppError) as exc_info: + SourceRegistryService.ensure_server_owned_source(session, "caller_claimed_grb") + + assert exc_info.value.code == "SOURCE_REGISTRY_ENTRY_NOT_FOUND" + + +def test_snapshot_is_checksum_bound_and_idempotent() -> None: + grb = _registry("grb") + session = InMemorySession(grb) + checksum = "a" * 64 + + snapshot = SourceRegistryService.record_snapshot( + session, + source_key="grb", + snapshot_key="2026-08-01-gbg", + checksum_sha256=checksum, + source_version="2026-08-01", + crs="EPSG:31370", + units="metres", + ) + + assert snapshot.source_registry_id == grb.id + assert snapshot.checksum_sha256 == checksum + assert snapshot.ingest_status == "ingested" + assert ( + SourceRegistryService.record_snapshot( + session, + source_key="grb", + snapshot_key="2026-08-01-gbg", + checksum_sha256=checksum, + ) + is snapshot + ) + + with pytest.raises(AppError) as exc_info: + SourceRegistryService.record_snapshot( + session, + source_key="grb", + snapshot_key="2026-08-01-gbg", + checksum_sha256="b" * 64, + ) + assert exc_info.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT" + + with pytest.raises(AppError) as version_conflict: + SourceRegistryService.record_snapshot( + session, + source_key="grb", + snapshot_key="2026-08-01-gbg", + checksum_sha256=checksum, + source_version="2026-08-02", + ) + assert version_conflict.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT" + + with pytest.raises(AppError) as invalid_checksum: + SourceRegistryService.record_snapshot( + session, + source_key="grb", + snapshot_key="bad-checksum", + checksum_sha256="not-a-checksum", + ) + assert invalid_checksum.value.code == "SOURCE_SNAPSHOT_CHECKSUM_INVALID" + + +def test_governed_import_reuses_an_identical_snapshot_without_rewriting_fetched_at() -> None: + """A second project may bind the same immutable source snapshot safely.""" + + grb = _registry("grb") + session = InMemorySession(grb) + checksum = "a" * 64 + observed_at = None + metadata = { + "dataset_type": "vector", + "bounds_json": {"min_x": 4.0, "min_y": 50.0, "max_x": 4.1, "max_y": 50.1}, + } + source_metadata = {"source_url": "https://example.test/grb", "units": "metres"} + + # Dataset ingest keys are project-scoped, while a source snapshot is + # globally keyed by immutable source evidence. This represents the same + # source file arriving through two independently resumable imports. + project_one, project_two = uuid4(), uuid4() + assert ( + DatasetService._ingest_key( + project_id=project_one, + source_key="grb", + checksum_sha256=checksum, + dataset_type="vector", + dataset_role="source", + area_id=None, + reference_layer_name=None, + source_version="2026-08-01", + ) + != DatasetService._ingest_key( + project_id=project_two, + source_key="grb", + checksum_sha256=checksum, + dataset_type="vector", + dataset_role="source", + area_id=None, + reference_layer_name=None, + source_version="2026-08-01", + ) + ) + + first_source, first_snapshot = DatasetService._record_snapshot( + session, + source_key="grb", + checksum_sha256=checksum, + source_version="2026-08-01", + observed_at=observed_at, + valid_from=None, + valid_to=None, + source_crs="EPSG:31370", + source_metadata=source_metadata, + metadata=metadata, + ) + original_fetched_at = first_snapshot.fetched_at + + replay_source, replay_snapshot = DatasetService._record_snapshot( + session, + source_key="grb", + checksum_sha256=checksum, + source_version="2026-08-01", + observed_at=observed_at, + valid_from=None, + valid_to=None, + source_crs="EPSG:31370", + source_metadata=source_metadata, + metadata=metadata, + ) + + assert replay_source is first_source + assert replay_snapshot is first_snapshot + assert replay_snapshot.fetched_at == original_fetched_at + assert session.objects[SourceSnapshot] == [first_snapshot] + + # Outside the governed replay path, a contradictory acquisition timestamp + # remains immutable evidence and is still rejected. + with pytest.raises(AppError) as fetched_at_conflict: + SourceRegistryService.record_snapshot( + session, + source_key="grb", + snapshot_key=first_snapshot.snapshot_key, + checksum_sha256=checksum, + fetched_at=original_fetched_at + timedelta(seconds=1), + ) + assert fetched_at_conflict.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT" + + # Replay mode is narrow: a changed immutable evidence field still fails. + with pytest.raises(AppError) as conflict: + SourceRegistryService.record_snapshot( + session, + source_key="grb", + snapshot_key=first_snapshot.snapshot_key, + checksum_sha256=checksum, + crs="EPSG:4326", + reuse_existing_snapshot=True, + ) + assert conflict.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT" + + +def test_snapshot_schema_requires_a_canonical_sha256() -> None: + constraints = { + constraint.name: str(constraint.sqltext) + for constraint in SourceSnapshot.__table__.constraints + if isinstance(constraint, CheckConstraint) + } + + assert SourceSnapshot.__table__.c.checksum_sha256.nullable is False + assert "ck_source_snapshots_checksum_sha256" in constraints + assert ( + "lower(checksum_sha256)" in constraints["ck_source_snapshots_checksum_sha256"] + ) + + +def test_complete_provenance_binding_is_required_before_authoritative_validation() -> ( + None +): + grb = _registry("grb") + snapshot = SourceSnapshot( + id=uuid4(), + source_registry_id=grb.id, + snapshot_key="governed-grb", + checksum_sha256="c" * 64, + ingest_status="ingested", + ) + dataset = _dataset() + + SourceRegistryService.bind_dataset_provenance( + dataset, + source=grb, + snapshot=snapshot, + data_contract_key="vector.grb.buildings", + data_contract_version="1.0.0", + validation_status="passed", + provenance_status="complete", + lineage_status="complete", + ) + + assert SourceRegistryService.is_dataset_eligible_for_authoritative_validation( + dataset, + source=grb, + snapshot=snapshot, + task="building_validation", + ) + + osm = _registry("osm") + dataset.source_registry_id = osm.id + snapshot.source_registry_id = osm.id + assert not SourceRegistryService.is_dataset_eligible_for_authoritative_validation( + dataset, + source=osm, + snapshot=snapshot, + task="building_validation", + ) + + +def test_lineage_and_quarantine_are_fail_closed_and_observable() -> None: + session = InMemorySession() + parent_id = uuid4() + child_id = uuid4() + edge = SourceRegistryService.record_lineage_edge( + session, + parent_dataset_id=parent_id, + child_dataset_id=child_id, + relation_type="derived_from", + transformation_name="vector_clip", + input_checksum_sha256="d" * 64, + output_checksum_sha256="e" * 64, + ) + + assert isinstance(edge, DatasetLineageEdge) + assert ( + SourceRegistryService.record_lineage_edge( + session, + parent_dataset_id=parent_id, + child_dataset_id=child_id, + relation_type="derived_from", + transformation_name="vector_clip", + input_checksum_sha256="d" * 64, + output_checksum_sha256="e" * 64, + ) + is edge + ) + + with pytest.raises(AppError) as self_reference: + SourceRegistryService.record_lineage_edge( + session, + parent_dataset_id=parent_id, + child_dataset_id=parent_id, + relation_type="derived_from", + transformation_name="vector_clip", + ) + assert self_reference.value.code == "DATASET_LINEAGE_SELF_REFERENCE" + + grandchild_id = uuid4() + SourceRegistryService.record_lineage_edge( + session, + parent_dataset_id=child_id, + child_dataset_id=grandchild_id, + relation_type="derived_from", + transformation_name="vector_buffer", + ) + with pytest.raises(AppError) as cycle: + SourceRegistryService.record_lineage_edge( + session, + parent_dataset_id=grandchild_id, + child_dataset_id=parent_id, + relation_type="derived_from", + transformation_name="vector_union", + ) + assert cycle.value.code == "DATASET_LINEAGE_CYCLE_DETECTED" + + dataset = _dataset() + record = SourceRegistryService.quarantine_dataset( + session, + dataset=dataset, + stage="vector_ingest", + reason_code="CRS_UNVERIFIED", + details={"observed_crs": None}, + ) + assert isinstance(record, DatasetQuarantine) + assert dataset.status == "quarantined" + assert dataset.quarantine_status == "quarantined" + assert dataset.validation_status == "failed" + + version_parent = _dataset() + version = DatasetVersion(id=uuid4(), dataset_id=version_parent.id, version=1) + version_session = InMemorySession(version_parent, version) + version_record = SourceRegistryService.quarantine_dataset( + version_session, + dataset_version=version, + stage="dataset_version_validation", + reason_code="CHECKSUM_MISMATCH", + ) + assert version_record.dataset_id == version_parent.id + assert version_record.dataset_version_id == version.id + assert version_parent.status == "quarantined" + assert version_parent.quarantine_status == "quarantined" + assert version_parent.validation_status == "failed" + assert version_parent.provenance_status == "incomplete" + assert version_parent.lineage_status == "incomplete" + assert version.validation_status == "failed" + assert version.provenance_status == "incomplete" + + snapshot = SourceSnapshot( + id=uuid4(), + source_registry_id=uuid4(), + snapshot_key="quarantined-source", + checksum_sha256="f" * 64, + ingest_status="ingested", + ) + SourceRegistryService.quarantine_dataset( + session, + source_snapshot=snapshot, + stage="source_snapshot_validation", + reason_code="CHECKSUM_MISMATCH", + ) + assert snapshot.ingest_status == "quarantined" + + +def test_quarantine_propagates_transitively_to_descendant_dataset_and_version_consumption_gates() -> ( + None +): + """A -> B -> C must fail closed when the governing A artifact is rejected.""" + + source = _registry("grb") + snapshot = SourceSnapshot( + id=uuid4(), + source_registry_id=source.id, + snapshot_key="transitive-quarantine-source", + checksum_sha256="a" * 64, + freshness_status="current", + ingest_status="ingested", + ) + + def governed_dataset(name: str) -> Dataset: + dataset = _dataset() + dataset.name = name + dataset.source = "grb" + dataset.source_name = "grb" + dataset.dataset_role = "source" + dataset.checksum_sha256 = snapshot.checksum_sha256 + dataset.source_registry_id = source.id + dataset.source_snapshot_id = snapshot.id + dataset.data_contract_key = "geointel.raster.geotiff" + dataset.data_contract_version = "1.0.0" + dataset.validation_status = "passed" + dataset.provenance_status = "complete" + dataset.lineage_status = "complete" + dataset.quarantine_status = "not_quarantined" + dataset.status = "ready" + dataset.source_registry = source + dataset.source_snapshot = snapshot + return dataset + + parent = governed_dataset("parent.tif") + child = governed_dataset("child.tif") + grandchild = governed_dataset("grandchild.tif") + parent_version = DatasetVersion( + id=uuid4(), + dataset_id=parent.id, + version=1, + validation_status="passed", + provenance_status="complete", + lineage_status="complete", + ) + child_version = DatasetVersion( + id=uuid4(), + dataset_id=child.id, + version=1, + validation_status="passed", + provenance_status="complete", + lineage_status="complete", + ) + grandchild_version = DatasetVersion( + id=uuid4(), + dataset_id=grandchild.id, + version=1, + validation_status="passed", + provenance_status="complete", + lineage_status="complete", + ) + session = InMemorySession( + parent, + child, + grandchild, + parent_version, + child_version, + grandchild_version, + ) + SourceRegistryService.record_lineage_edge( + session, + parent_dataset_id=parent.id, + child_dataset_id=child.id, + relation_type="derived_from", + transformation_name="clip", + ) + SourceRegistryService.record_lineage_edge( + session, + parent_dataset_id=child.id, + child_dataset_id=grandchild.id, + relation_type="derived_from", + transformation_name="buffer", + ) + + assert ( + DatasetConsumptionGate.evaluate(child, purpose="production_inference").eligible + is True + ) + assert ( + DatasetConsumptionGate.evaluate( + grandchild, purpose="production_inference" + ).eligible + is True + ) + + SourceRegistryService.quarantine_dataset( + session, + dataset=parent, + stage="contract_validation", + reason_code="CHECKSUM_MISMATCH", + ) + + for dataset in (parent, child, grandchild): + decision = DatasetConsumptionGate.evaluate( + dataset, purpose="production_inference" + ) + assert dataset.status == "quarantined" + assert dataset.quarantine_status == "quarantined" + assert dataset.validation_status == "failed" + assert dataset.provenance_status == "incomplete" + assert dataset.lineage_status == "incomplete" + assert decision.eligible is False + assert "dataset_quarantined" in decision.reasons + for dataset_version in (parent_version, child_version, grandchild_version): + assert dataset_version.validation_status == "failed" + assert dataset_version.provenance_status == "incomplete" + assert dataset_version.lineage_status == "incomplete" + + +def test_ingest_keys_are_scoped_and_migration_keeps_unknown_legacy_unbound() -> None: + project_id = uuid4() + dataset = _dataset() + dataset.project_id = project_id + dataset.ingest_key = "grb:2026-08-01:gbg:area-sha" + version = DatasetVersion( + id=uuid4(), + dataset_id=dataset.id, + ingest_key=dataset.ingest_key, + validation_status="not_validated", + provenance_status="incomplete", + lineage_status="incomplete", + ) + session = InMemorySession(dataset, version) + + assert ( + SourceRegistryService.find_dataset_by_ingest_key( + session, project_id, dataset.ingest_key + ) + is dataset + ) + assert ( + SourceRegistryService.find_dataset_version_by_ingest_key( + session, dataset.id, dataset.ingest_key + ) + is version + ) + with pytest.raises(AppError) as invalid_key: + SourceRegistryService.find_dataset_by_ingest_key(session, project_id, " ") + assert invalid_key.value.code == "INGEST_KEY_INVALID" + + migration = ( + Path(__file__).resolve().parents[1] + / "alembic" + / "versions" + / "202608010001_source_registry_provenance.py" + ).read_text(encoding="utf-8") + assert "uuid_generate_v5" not in migration + assert "__unregistered_legacy_source__" in migration + assert "uq_datasets_project_ingest_key" in migration + assert "uq_dataset_versions_dataset_ingest_key" in migration + + +def test_migration_contains_database_guards_for_snapshot_pairing_contract_lineage_and_quarantine() -> ( + None +): + migration = ( + Path(__file__).resolve().parents[1] + / "alembic" + / "versions" + / "202608010001_source_registry_provenance.py" + ).read_text(encoding="utf-8") + + assert "trg_datasets_snapshot_registry_guard" in migration + assert "trg_dataset_versions_snapshot_registry_guard" in migration + assert "trg_source_registry_write_guard" in migration + assert "trg_source_snapshots_evidence_immutable" in migration + assert "geointel_phase2_contract_report_guard" in migration + assert "trg_datasets_contract_report_guard" in migration + assert "trg_dataset_versions_contract_report_guard" in migration + assert "matching complete validation report" in migration + assert "geointel_phase2_lineage_cycle_guard" in migration + assert "trg_dataset_lineage_edges_cycle_guard" in migration + assert "geointel_phase2_lineage_edge_immutable_guard" in migration + assert "trg_dataset_lineage_edges_immutable" in migration + assert "WITH RECURSIVE descendants" in migration + assert "geointel_phase2_quarantine_lineage_descendants" in migration + assert "geointel_phase2_quarantine_state_guard" in migration + assert "trg_dataset_quarantines_state_guard" in migration + assert "accepted dataset artifact and provenance evidence is immutable" in migration diff --git a/backend/tests/test_auth.py b/backend/tests/test_auth.py index 374b3352..7e28b893 100644 --- a/backend/tests/test_auth.py +++ b/backend/tests/test_auth.py @@ -154,6 +154,7 @@ def test_guest_login_exposes_models_but_rejects_management_and_cross_project_req other_project = client.get("/api/v1/projects/00000000-0000-0000-0000-000000000999") detection_models = client.get("/api/v1/detection/models") segmentation_models = client.get("/api/v1/segmentation/models") + global_source_registry = client.get("/api/v1/source-registry/grb") cross_project_runs = client.get( "/api/v1/detection/runs?project_id=00000000-0000-0000-0000-000000000999" ) @@ -179,6 +180,8 @@ def test_guest_login_exposes_models_but_rejects_management_and_cross_project_req assert detection_models.status_code == 200 assert detection_models.json()["data"]["models"] assert segmentation_models.status_code == 200 + assert global_source_registry.status_code == 403 + assert global_source_registry.json()["error"] == "GUEST_ROUTE_NOT_AVAILABLE" assert cross_project_runs.status_code == 403 assert cross_project_runs.json()["error"] == "GUEST_PROJECT_SCOPE_REQUIRED" assert cross_project_coverage.status_code == 403 @@ -186,7 +189,7 @@ def test_guest_login_exposes_models_but_rejects_management_and_cross_project_req def test_guest_session_tokens_fail_closed_without_a_project_scope(monkeypatch) -> None: - client = auth_client(monkeypatch, guest_access=True) + auth_client(monkeypatch, guest_access=True) settings = get_settings() try: diff --git a/backend/tests/test_belgium_training_loop.py b/backend/tests/test_belgium_training_loop.py index 9f5b43d8..65d5dca1 100644 --- a/backend/tests/test_belgium_training_loop.py +++ b/backend/tests/test_belgium_training_loop.py @@ -1,6 +1,7 @@ from __future__ import annotations import importlib.util +import hashlib import json import subprocess import sys @@ -13,6 +14,94 @@ assert SPEC and SPEC.loader MODULE = importlib.util.module_from_spec(SPEC) SPEC.loader.exec_module(MODULE) +from training_release_manifest import create_training_release_manifest # noqa: E402 + + +def write_fixture_manifest(path: Path) -> None: + policy = "geointel-training-source-eligibility/v1" + def eligible(sample_slug: str) -> dict[str, object]: + return { + "policy_version": policy, + "eligible": True, + "fixture_mode": True, + "raster": { + "eligible": True, + "reasons": [], + "evidence": { + "dataset_id": f"raster:{sample_slug}", + "checksum_sha256": "a" * 64, + "source_registry_id": "fixture-raster", + "source_snapshot_id": "fixture-raster-snapshot", + }, + }, + "reference": { + "eligible": True, + "reasons": [], + "evidence": { + "dataset_id": f"reference:{sample_slug}", + "checksum_sha256": "b" * 64, + "source_registry_id": "fixture-reference", + "source_snapshot_id": "fixture-reference-snapshot", + }, + }, + } + path.write_text( + json.dumps( + { + "training_eligibility": { + "policy_version": policy, + "status": "eligible", + "fixture_mode": True, + }, + "samples": [ + { + "sample_slug": sample_slug, + "split": split, + "raster_dataset_id": f"raster:{sample_slug}", + "reference_dataset_id": f"reference:{sample_slug}", + "training_eligibility": eligible(sample_slug), + } + for sample_slug, split in (("fixture-train", "train"), ("fixture-val", "val")) + ], + } + ), + encoding="utf-8", + ) + (path.parent / "corpus-freeze.json").write_text( + json.dumps( + { + "schema_version": 2, + "manifest_sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + "immutable": True, + "training_eligibility_policy": policy, + "fixture_mode": True, + } + ), + encoding="utf-8", + ) + + +def write_fixture_training_release(tmp_path: Path, manifest: Path) -> Path: + dataset_dir = tmp_path / "fixture-dataset" + for split, sample_slug in (("train", "fixture-train"), ("val", "fixture-val")): + image = dataset_dir / "images" / split / f"{sample_slug}.png" + label = dataset_dir / "labels" / split / f"{sample_slug}.txt" + image.parent.mkdir(parents=True, exist_ok=True) + label.parent.mkdir(parents=True, exist_ok=True) + image.write_bytes(split.encode("utf-8")) + label.write_text("0 0.5 0.5 0.2 0.2\n", encoding="utf-8") + yaml_path = dataset_dir / "dataset.yaml" + yaml_path.write_text( + f"path: {dataset_dir}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n", + encoding="utf-8", + ) + create_training_release_manifest( + train_yaml=yaml_path, + corpus_manifest=manifest, + fixture_mode=True, + ) + return yaml_path + def test_training_command_is_cuda_deterministic_and_bound_to_frozen_inputs(tmp_path: Path) -> None: command = MODULE.training_command( @@ -78,6 +167,7 @@ def test_failed_iteration_builds_train_only_sampling_for_next_checkpoint(tmp_pat corpus_manifest=tmp_path / "manifest.json", assessment=tmp_path / "assessment.json", output_dir=tmp_path / "iteration-001" / "failure-driven-training", + review_audit=tmp_path / "review-audit.json", ) assert command[1].endswith("build_failure_driven_yolo_sampling.py") assert command[command.index("--summary") + 1].endswith("train-summary.json") @@ -103,20 +193,23 @@ def test_dry_run_can_gate_existing_checkpoint_without_training(tmp_path: Path) - "status": "ok", "low_variance_positive_tile_count": 0, "label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0}, })) + manifest = tmp_path / "manifest.json" + write_fixture_manifest(manifest) + train_yaml = write_fixture_training_release(tmp_path, manifest) result = subprocess.run( [ sys.executable, str(SCRIPT), "--initial-model", str(tmp_path / "candidate.pt"), - "--train-yaml", str(tmp_path / "dataset.yaml"), + "--train-yaml", str(train_yaml), "--train-summary", str(tmp_path / "train-summary.json"), "--dataset-audit", str(audit), "--train-quality-audit", str(quality), "--calibration-summary", str(tmp_path / "cal.json"), "--test-summary", str(tmp_path / "test.json"), "--background-summary", str(tmp_path / "background.json"), - "--corpus-manifest", str(tmp_path / "manifest.json"), + "--corpus-manifest", str(manifest), "--output-dir", str(tmp_path / "output"), - "--evaluate-initial-model", "--dry-run", + "--evaluate-initial-model", "--fixture-mode", "--dry-run", ], capture_output=True, text=True, check=False, ) assert result.returncode == 0 @@ -139,6 +232,9 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None: "status": "ok", "low_variance_positive_tile_count": 0, "label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0}, })) + manifest = tmp_path / "manifest.json" + write_fixture_manifest(manifest) + train_yaml = write_fixture_training_release(tmp_path, manifest) result = subprocess.run( [ sys.executable, @@ -146,7 +242,7 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None: "--initial-model", str(tmp_path / "base.pt"), "--train-yaml", - str(tmp_path / "dataset.yaml"), + str(train_yaml), "--train-summary", str(tmp_path / "train-summary.json"), "--dataset-audit", @@ -160,9 +256,10 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None: "--background-summary", str(tmp_path / "background.json"), "--corpus-manifest", - str(tmp_path / "manifest.json"), + str(manifest), "--output-dir", str(tmp_path / "output"), + "--fixture-mode", ], capture_output=True, text=True, @@ -172,7 +269,45 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None: assert "Dataset audit is not eligible for training" in result.stderr -def test_pending_human_review_does_not_block_objective_training() -> None: +def test_loop_rejects_manifest_without_source_eligibility_before_cuda_training(tmp_path: Path) -> None: + audit = tmp_path / "audit.json" + audit.write_text(json.dumps({ + "status": "needs_human_review", "failures": [], + "manifest_immutable": True, "spatial_leakage_status": "ok", + })) + quality = tmp_path / "quality.json" + quality.write_text(json.dumps({ + "status": "ok", "low_variance_positive_tile_count": 0, + "label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0}, + })) + manifest = tmp_path / "manifest.json" + manifest.write_text(json.dumps({"samples": [{"sample_slug": "unproven"}]}), encoding="utf-8") + + result = subprocess.run( + [ + sys.executable, str(SCRIPT), + "--initial-model", str(tmp_path / "candidate.pt"), + "--train-yaml", str(tmp_path / "dataset.yaml"), + "--train-summary", str(tmp_path / "train-summary.json"), + "--dataset-audit", str(audit), + "--train-quality-audit", str(quality), + "--calibration-summary", str(tmp_path / "cal.json"), + "--test-summary", str(tmp_path / "test.json"), + "--background-summary", str(tmp_path / "background.json"), + "--corpus-manifest", str(manifest), + "--output-dir", str(tmp_path / "output"), + "--evaluate-initial-model", "--dry-run", + ], + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode != 0 + assert "manifest_training_eligibility_missing" in result.stderr + + +def test_pending_human_review_blocks_operational_training() -> None: audit = { "status": "needs_human_review", "failures": [], @@ -185,7 +320,58 @@ def test_pending_human_review_does_not_block_objective_training() -> None: "status": "ok", "low_variance_positive_tile_count": 0, "label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0}, } - assert MODULE.dataset_audit_failures(audit, quality) == [] + failures = MODULE.dataset_audit_failures(audit, quality) + assert "unsupported audit status: needs_human_review" in failures + assert "review_complete_not_true" in failures + assert "accepted_human_review_evidence_missing" in failures + + +def test_fixture_mode_can_relax_review_only_after_fixture_manifest_gate() -> None: + audit = { + "status": "needs_human_review", + "failures": [], + "manifest_immutable": True, + "spatial_leakage_status": "ok", + "review_complete": False, + } + quality = { + "status": "ok", "low_variance_positive_tile_count": 0, + "label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0}, + } + assert MODULE.dataset_audit_failures(audit, quality, fixture_mode=True) == [] + + +def test_operational_dataset_audit_must_be_the_one_bound_into_the_release(tmp_path: Path) -> None: + bound = tmp_path / "bound-audit.json" + other = tmp_path / "other-audit.json" + bound.write_text("{}", encoding="utf-8") + other.write_text("{}", encoding="utf-8") + release = {"human_review": {"audit_path": str(bound.resolve())}} + + MODULE.assert_dataset_audit_bound_to_release( + release=release, + dataset_audit=bound, + fixture_mode=False, + ) + try: + MODULE.assert_dataset_audit_bound_to_release( + release=release, + dataset_audit=other, + fixture_mode=False, + ) + except MODULE.TrainingReleaseError as exc: + assert "does not match" in str(exc) + else: + raise AssertionError("unbound dataset audit was accepted") + + +def test_protected_assessment_feedback_is_terminal_and_cannot_seed_another_yaml() -> None: + assert MODULE.protected_feedback_roles( + {"status": "continue_training_loop", "test": {"aggregate": {}}, "background": None} + ) == ["test"] + assert MODULE.protected_feedback_roles( + {"status": "continue_training_loop", "test": None, "background": {"aggregate": {}}} + ) == ["background"] def test_training_audit_still_fails_closed_on_automated_integrity_gates() -> None: diff --git a/backend/tests/test_data_contract_validation.py b/backend/tests/test_data_contract_validation.py new file mode 100644 index 00000000..805fd4b8 --- /dev/null +++ b/backend/tests/test_data_contract_validation.py @@ -0,0 +1,449 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +from hashlib import sha256 +import json +from pathlib import Path + +import pytest +from shapely.geometry import box + +from app.core.errors import AppError +from app.services.data_contract_validation import ( + AttributeRule, + BoundingBox, + ContractKind, + DataAssetValidationInput, + DataContract, + DataContractRegistry, + DataContractValidator, + FreshnessRules, + GeometryRecord, + GeometryRules, + LineageEvidence, + LineageRules, + RasterRules, + RequirementLevel, + Resolution, + ResolutionRules, + TransformationEvidence, + ValidationStatus, + build_default_data_contract_registry, + build_label_validation_input, + build_model_validation_input, + build_raster_ingest_input, + build_vector_ingest_input, + validate_registered_asset, +) +from app.services.data_quarantine_service import AssetUse, DataQuarantineService + + +FIXTURE_ROOT = Path(__file__).resolve().parents[2] / "tests" / "fixtures" / "data-contracts" +NOW = datetime(2026, 8, 1, 12, 0, tzinfo=timezone.utc) +CHECKSUM_A = "a" * 64 + + +def _fixture_json(name: str) -> tuple[bytes, object]: + raw = (FIXTURE_ROOT / name).read_bytes() + return raw, json.loads(raw) + + +def _checksum(raw: bytes) -> str: + return sha256(raw).hexdigest() + + +def _lineage_with_transform() -> LineageEvidence: + return LineageEvidence( + transformations=( + TransformationEvidence( + name="epsg31370-to-epsg4326", + version="1.0.0", + checksum_sha256=CHECKSUM_A, + ), + ), + ) + + +def _vector_input_from_fixture(name: str, *, source_crs: str = "EPSG:31370", storage_crs: str = "EPSG:4326") -> DataAssetValidationInput: + raw, payload = _fixture_json(name) + assert isinstance(payload, dict) + return build_vector_ingest_input( + asset_id=f"fixture:{name}", + source_crs=source_crs, + storage_crs=storage_crs, + feature_collection=payload, + checksum_sha256=_checksum(raw), + computed_checksum_sha256=_checksum(raw), + content=raw, + source_registry_id="source:digitaal-vlaanderen:grb", + source_snapshot_id="snapshot:grb:2026-07-31", + imported_at=NOW, + metadata={"license": "Open Data Licence", "provider": "Digitaal Vlaanderen"}, + observed_at=NOW - timedelta(days=1), + source_version="2026.07.31", + lineage=_lineage_with_transform() if source_crs != storage_crs else LineageEvidence(), + ) + + +def _issue_codes(report) -> set[str]: + return {issue.code for issue in report.issues} + + +def test_default_vector_contract_accepts_transformed_geojson_with_complete_provenance() -> None: + report = validate_registered_asset(_vector_input_from_fixture("vector-building-valid.geojson"), now=NOW) + + assert report.validation_status == ValidationStatus.PASSED + assert report.quarantine_status == "not_quarantined" + assert report.provenance_status == "complete" + assert report.lineage_status == "complete" + persisted = report.persistence_fields() + assert persisted["data_contract_key"] == "geointel.vector.geojson" + assert persisted["data_contract_version"] == "1.0.0" + assert persisted["validation_report_json"]["report_sha256"] == report.report_sha256 + + +def test_default_vector_contract_quarantines_lambert_coordinates_mislabelled_as_epsg4326() -> None: + report = validate_registered_asset( + _vector_input_from_fixture( + "vector-lambert-mislabelled-as-4326.geojson", + source_crs="EPSG:4326", + storage_crs="EPSG:4326", + ), + now=NOW, + ) + + assert report.validation_status == ValidationStatus.FAILED + assert report.quarantine_status == "quarantined" + assert "CRS_COORDINATE_DOMAIN_VIOLATION" in _issue_codes(report) + + +def test_vector_contract_checks_geometry_attributes_bounds_and_topology_fail_closed() -> None: + contract = DataContract( + key="test.vector.buildings", + version="1.0.0", + kind=ContractKind.VECTOR, + accepted_source_crs=frozenset({"EPSG:4326"}), + canonical_storage_crs="EPSG:4326", + spatial_domain=BoundingBox(2.0, 49.0, 7.0, 52.0), + require_bounds=True, + geometry_rules=GeometryRules( + allowed_geometry_types=frozenset({"Polygon"}), + attribute_rules=(AttributeRule("native_id", accepted_types=("integer",)),), + forbid_shared_area=True, + ), + ) + raw = b"overlapping-vector" + asset = DataAssetValidationInput( + asset_id="vector:bad", + data_contract_key=contract.key, + data_contract_version=contract.version, + kind=ContractKind.VECTOR, + source_crs="EPSG:4326", + storage_crs="EPSG:4326", + bounds=BoundingBox(4.0, 51.0, 4.1, 51.1), + checksum_sha256=_checksum(raw), + computed_checksum_sha256=_checksum(raw), + content=raw, + geometry_records=( + GeometryRecord(box(4.0, 51.0, 4.05, 51.05), {"native_id": "wrong-type"}), + GeometryRecord(box(4.025, 51.025, 4.075, 51.075), {}), + ), + source_registry_id="source:test", + source_snapshot_id="snapshot:test", + imported_at=NOW, + ) + + report = DataContractValidator.validate(contract, asset, now=NOW) + + assert report.validation_status == ValidationStatus.FAILED + assert {"ATTRIBUTE_TYPE_INVALID", "ATTRIBUTE_REQUIRED", "TOPOLOGY_SHARED_AREA"} <= _issue_codes(report) + assert "BOUNDS_GEOMETRY_MISMATCH" in _issue_codes(report) + + +def test_default_vector_contract_validates_replayable_large_partition_stream_without_materialising_geometry_list() -> None: + """Regional imports may be large but remain fully schema/domain checked. + + The default contract has no source-specific shared-area rule, so the + validator must make its bounds/schema passes over a replayable stream + without accumulating every Shapely geometry in memory. A stricter + source-specific contract can still opt into a bounded topology batch. + """ + + class ReplayableRecords: + def __init__(self, count: int) -> None: + self.count = count + self.iterations = 0 + + def __iter__(self): + self.iterations += 1 + for index in range(self.count): + yield GeometryRecord( + box(4.69, 51.09, 4.70, 51.10), + {"partition_feature": index}, + ) + + raw = b"partitioned-vector-stream" + records = ReplayableRecords(12_000) + asset = DataAssetValidationInput( + asset_id="vector:partitioned-stream", + data_contract_key="geointel.vector.geojson", + data_contract_version="1.0.0", + kind=ContractKind.VECTOR, + source_crs="EPSG:4326", + storage_crs="EPSG:4326", + bounds=BoundingBox(4.69, 51.09, 4.70, 51.10), + checksum_sha256=_checksum(raw), + computed_checksum_sha256=_checksum(raw), + content=raw, + metadata={"license": "Open Data"}, + geometry_records=records, + source_registry_id="source:grb", + source_snapshot_id="snapshot:grb:partitioned", + imported_at=NOW, + observed_at=NOW, + source_version="2026-08-01", + ) + + report = validate_registered_asset(asset, now=NOW) + + assert report.validation_status == ValidationStatus.PASSED + assert records.iterations >= 2 + + +def test_raster_contract_accepts_explicit_units_and_quarantines_stale_bad_profile() -> None: + raw = b"raster-stage" + valid = build_raster_ingest_input( + asset_id="raster:valid", + source_crs="EPSG:31370", + storage_crs="EPSG:31370", + raster_profile={"width": 512, "height": 512, "band_count": 3, "dtype": ["uint8"]}, + bounds=BoundingBox(193_277.5, 205_708.3, 193_777.5, 206_208.3), + resolution=Resolution(0.9765625, 0.9765625, "m"), + checksum_sha256=_checksum(raw), + computed_checksum_sha256=_checksum(raw), + content=raw, + source_registry_id="source:orthophoto", + source_snapshot_id="snapshot:orthophoto:2026.01", + imported_at=NOW, + metadata={"license": "Open Data"}, + observed_at=None, + temporal_unknown_reason="latest mosaic has no per-pixel observation date", + source_version=None, + source_version_unknown_reason="provider did not publish an edition", + ) + assert validate_registered_asset(valid, now=NOW).validation_status == ValidationStatus.PASSED + + strict = DataContract( + key="test.raster.strict", + version="1.0.0", + kind=ContractKind.RASTER, + accepted_source_crs=frozenset({"EPSG:31370"}), + require_bounds=True, + raster_rules=RasterRules(allowed_band_counts=frozenset({3}), allowed_dtypes=frozenset({"uint8"})), + resolution_rules=ResolutionRules(allowed_units=frozenset({"m"}), min_x=0.2, max_x=1.0, min_y=0.2, max_y=1.0), + freshness_rules=FreshnessRules(observed_at=RequirementLevel.REQUIRED, max_age=timedelta(days=30)), + lineage_rules=LineageRules(require_transformation_when_crs_changes=False), + ) + invalid = DataAssetValidationInput( + asset_id="raster:bad", + data_contract_key=strict.key, + data_contract_version=strict.version, + kind=ContractKind.RASTER, + source_crs="EPSG:31370", + storage_crs="EPSG:31370", + bounds=BoundingBox(100.0, 100.0, 200.0, 200.0), + checksum_sha256=_checksum(raw), + computed_checksum_sha256=_checksum(raw), + content=raw, + raster_profile={"width": 0, "height": 10, "band_count": 2, "dtype": ["float32"]}, + resolution=Resolution(2.0, 0.1, "degree"), + source_registry_id="source:raster", + source_snapshot_id="snapshot:raster", + imported_at=NOW, + observed_at=NOW - timedelta(days=31), + ) + report = DataContractValidator.validate(strict, invalid, now=NOW) + + assert report.validation_status == ValidationStatus.FAILED + assert { + "RASTER_PROFILE_VALUE_INVALID", + "RASTER_BAND_COUNT_NOT_ALLOWED", + "RASTER_DTYPE_NOT_ALLOWED", + "RESOLUTION_UNIT_NOT_ALLOWED", + "RESOLUTION_OUT_OF_RANGE", + "FRESHNESS_EXCEEDED", + } <= _issue_codes(report) + + +def test_default_label_and_model_contracts_validate_good_and_bad_fixtures() -> None: + valid_raw, valid_labels = _fixture_json("labels-valid.json") + invalid_raw, invalid_labels = _fixture_json("labels-invalid.json") + assert isinstance(valid_labels, list) + assert isinstance(invalid_labels, list) + lineage = LineageEvidence(upstream_asset_ids=("image:1",), upstream_checksums_sha256=(CHECKSUM_A,)) + valid_label = build_label_validation_input( + asset_id="label:valid", + label_records=valid_labels, + checksum_sha256=_checksum(valid_raw), + computed_checksum_sha256=_checksum(valid_raw), + content=valid_raw, + source_registry_id="source:labels", + source_snapshot_id="snapshot:labels:1", + imported_at=NOW, + metadata={ + "image_checksum_sha256": CHECKSUM_A, + "class_ontology_version": "buildings-v1", + "source_corpus_manifest_sha256": CHECKSUM_A, + }, + temporal_unknown_reason="labels inherit image observation handling", + source_version_unknown_reason="label release is represented by its snapshot", + lineage=lineage, + ) + valid_report = validate_registered_asset(valid_label, now=NOW) + assert valid_report.validation_status == ValidationStatus.PASSED + + invalid_label = build_label_validation_input( + asset_id="label:invalid", + label_records=invalid_labels, + checksum_sha256=_checksum(invalid_raw), + computed_checksum_sha256=_checksum(invalid_raw), + content=invalid_raw, + source_registry_id="source:labels", + source_snapshot_id="snapshot:labels:1", + imported_at=NOW, + metadata={ + "image_checksum_sha256": "not-a-sha256", + "class_ontology_version": "buildings-v1", + "source_corpus_manifest_sha256": CHECKSUM_A, + }, + temporal_unknown_reason="labels inherit image observation handling", + source_version_unknown_reason="label release is represented by its snapshot", + lineage=lineage, + ) + invalid_report = validate_registered_asset(invalid_label, now=NOW) + assert invalid_report.validation_status == ValidationStatus.FAILED + assert { + "LABEL_CLASS_ID_NOT_ALLOWED", + "LABEL_NORMALIZED_COORDINATE_INVALID", + "METADATA_CHECKSUM_INVALID", + } <= _issue_codes(invalid_report) + + pure_background_raw = b"" + pure_background_metadata = { + "image_checksum_sha256": CHECKSUM_A, + "class_ontology_version": "buildings-v1", + "source_corpus_manifest_sha256": CHECKSUM_A, + "label_mode": "pure_background", + "sample_slug": "forest-background-aoi", + "split": "train", + "raster_dataset_id": "dataset:raster:1", + "reference_dataset_id": "dataset:reference:1", + "review_decision": "accepted", + "reviewer_id": "reviewer@example.test", + "reviewed_at": "2026-08-01T11:00:00+00:00", + "review_artifact_sha256": CHECKSUM_A, + } + pure_background = build_label_validation_input( + asset_id="label:pure-background", + label_records=(), + label_mode="pure_background", + checksum_sha256=_checksum(pure_background_raw), + computed_checksum_sha256=_checksum(pure_background_raw), + content=pure_background_raw, + source_registry_id="source:labels", + source_snapshot_id="snapshot:labels:1", + imported_at=NOW, + metadata=pure_background_metadata, + temporal_unknown_reason="labels inherit image observation handling", + source_version_unknown_reason="label release is represented by its snapshot", + lineage=LineageEvidence( + upstream_asset_ids=("dataset:raster:1", "dataset:reference:1"), + upstream_checksums_sha256=(CHECKSUM_A, CHECKSUM_A), + ), + ) + assert validate_registered_asset(pure_background, now=NOW).validation_status == ValidationStatus.PASSED + + unmarked_empty = build_label_validation_input( + asset_id="label:unmarked-empty", + label_records=(), + checksum_sha256=_checksum(pure_background_raw), + computed_checksum_sha256=_checksum(pure_background_raw), + content=pure_background_raw, + source_registry_id="source:labels", + source_snapshot_id="snapshot:labels:1", + imported_at=NOW, + metadata={ + "image_checksum_sha256": CHECKSUM_A, + "class_ontology_version": "buildings-v1", + "source_corpus_manifest_sha256": CHECKSUM_A, + }, + temporal_unknown_reason="labels inherit image observation handling", + source_version_unknown_reason="label release is represented by its snapshot", + lineage=LineageEvidence( + upstream_asset_ids=("dataset:raster:1", "dataset:reference:1"), + upstream_checksums_sha256=(CHECKSUM_A, CHECKSUM_A), + ), + ) + assert "PURE_BACKGROUND_MODE_REQUIRED" in _issue_codes(validate_registered_asset(unmarked_empty, now=NOW)) + + model_raw = b"model-asset" + model = build_model_validation_input( + asset_id="model:valid", + model_metadata={"model_format": "pytorch", "framework": "torch", "class_mapping": {"0": "building"}}, + checksum_sha256=_checksum(model_raw), + computed_checksum_sha256=_checksum(model_raw), + content=model_raw, + source_registry_id="source:model-registry", + source_snapshot_id="snapshot:model:1", + imported_at=NOW, + source_version="candidate-1", + metadata={"training_manifest_sha256": CHECKSUM_A, "runtime_manifest_sha256": CHECKSUM_A}, + lineage=lineage, + ) + assert validate_registered_asset(model, now=NOW).validation_status == ValidationStatus.PASSED + + +def test_unknown_contract_and_quarantine_gate_are_deterministic_and_fail_closed() -> None: + unknown = DataAssetValidationInput( + asset_id="asset:unknown", + data_contract_key="does.not.exist", + data_contract_version="9.9.9", + kind=ContractKind.VECTOR, + ) + report = DataContractRegistry().validate(unknown, now=NOW) + assert report.validation_status == ValidationStatus.FAILED + assert report.quarantine_status == "quarantined" + assert _issue_codes(report) == {"DATA_CONTRACT_UNKNOWN"} + + first = DataQuarantineService.decide(report) + second = DataQuarantineService.decide(report) + assert first.idempotency_key == second.idempotency_key + assert first.reason_codes == ("DATA_CONTRACT_UNKNOWN",) + with pytest.raises(AppError, match="cannot enter this pipeline") as exc_info: + DataQuarantineService.require_eligible(first, use=AssetUse.PRODUCTION_INFERENCE) + assert exc_info.value.code == "DATASET_QUARANTINED" + assert exc_info.value.details["use"] == "production_inference" + + clean_report = validate_registered_asset(_vector_input_from_fixture("vector-building-valid.geojson"), now=NOW) + release_request = DataQuarantineService.decide(clean_report, previous=first) + assert release_request.quarantine_status == "quarantined" + assert release_request.requires_explicit_release is True + assert release_request.reason_codes == ("QUARANTINE_RELEASE_REQUIRES_EXPLICIT_PERSISTENCE",) + + +def test_registry_requires_exact_contract_version_and_fingerprints_schema() -> None: + registry = build_default_data_contract_registry() + version_mismatch = _vector_input_from_fixture("vector-building-valid.geojson") + mismatched = DataAssetValidationInput( + **{**version_mismatch.__dict__, "data_contract_version": "2.0.0"}, + ) + + report = registry.validate(mismatched, now=NOW) + assert report.validation_status == ValidationStatus.FAILED + assert "DATA_CONTRACT_UNKNOWN" in _issue_codes(report) + + contract = registry.resolve("geointel.vector.geojson", "1.0.0") + assert contract is not None + direct_report = DataContractValidator.validate(contract, mismatched, now=NOW) + assert direct_report.validation_status == ValidationStatus.FAILED + assert "DATA_CONTRACT_IDENTITY_MISMATCH" in _issue_codes(direct_report) diff --git a/backend/tests/test_dataset_consumption_gate.py b/backend/tests/test_dataset_consumption_gate.py new file mode 100644 index 00000000..6862ffa7 --- /dev/null +++ b/backend/tests/test_dataset_consumption_gate.py @@ -0,0 +1,316 @@ +from __future__ import annotations + +from types import SimpleNamespace +from uuid import uuid4 + +import pytest +from shapely.geometry import box + +from app.core.errors import AppError +from app.models import Dataset, SourceRegistry, SourceSnapshot +from app.services.coverage_registry_service import CoverageRegistryService, SOURCE_DEFINITIONS +import app.services.dataset_consumption_gate_service as gate_module +from app.services.dataset_consumption_gate_service import DatasetConsumptionGate +from app.services.export_service import ExportService + + +def _governed_dataset( + *, + source_key: str = "grb", + classification: str = "authoritative", + snapshot_freshness_status: str = "current", +) -> Dataset: + source_id = uuid4() + snapshot_id = uuid4() + checksum = "a" * 64 + source = SourceRegistry( + id=source_id, + source_key=source_key, + display_name=f"{source_key} test source", + classification=classification, + authority_name="GeoIntel test authority", + authority_scope_json={"scope": "test"}, + usage_policy_json={ + "ground_truth_allowed": classification == "authoritative", + "validation_authority": {"building_validation": "primary"} + if classification == "authoritative" + else {}, + }, + ) + snapshot = SourceSnapshot( + id=snapshot_id, + source_registry_id=source_id, + snapshot_key="test-snapshot", + checksum_sha256=checksum, + freshness_status=snapshot_freshness_status, + ingest_status="ingested", + ) + dataset = Dataset( + id=uuid4(), + project_id=uuid4(), + name="governed.tif", + dataset_type="raster", + source=source_key, + source_name=source_key, + dataset_role="source", + checksum_sha256=checksum, + source_registry_id=source_id, + source_snapshot_id=snapshot_id, + data_contract_key="geointel.raster.geotiff", + data_contract_version="1.0.0", + validation_status="passed", + provenance_status="complete", + lineage_status="not_applicable", + quarantine_status="not_quarantined", + status="ready", + ) + dataset.source_registry = source + dataset.source_snapshot = snapshot + return dataset + + +def test_governed_dataset_passes_production_inference_and_authoritative_coverage() -> None: + dataset = _governed_dataset() + + inference = DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference") + coverage = DatasetConsumptionGate.assert_eligible(dataset, purpose="authoritative_coverage") + + assert inference.eligible is True + assert coverage.eligible is True + + +@pytest.mark.parametrize( + ("field", "value", "error_code"), + ( + ("provenance_status", "incomplete", "DATASET_PROVENANCE_INCOMPLETE"), + ("validation_status", "failed", "DATASET_QUARANTINED"), + ("quarantine_status", "quarantined", "DATASET_QUARANTINED"), + ), +) +def test_explicit_unsafe_states_can_never_be_relaxed(field: str, value: str, error_code: str) -> None: + dataset = _governed_dataset() + setattr(dataset, field, value) + + with pytest.raises(AppError) as exc_info: + DatasetConsumptionGate.assert_eligible( + dataset, + purpose="production_inference", + fixture_mode=True, + ) + + assert exc_info.value.code == error_code + assert field.replace("_status", "") in " ".join(exc_info.value.details["reasons"]) + + +def test_legacy_fixture_can_support_fixture_qa_but_never_authoritative_coverage() -> None: + fixture = Dataset( + id=uuid4(), + project_id=uuid4(), + name="fixture.tif", + dataset_type="raster", + source="fixture", + ) + + qa = DatasetConsumptionGate.assert_eligible(fixture, purpose="quality_assessment") + with pytest.raises(AppError) as inference_error: + DatasetConsumptionGate.assert_eligible( + fixture, + purpose="production_inference", + fixture_mode=True, + ) + with pytest.raises(AppError) as export_error: + DatasetConsumptionGate.assert_eligible(fixture, purpose="export") + with pytest.raises(AppError) as fixture_export_error: + DatasetConsumptionGate.assert_eligible(fixture, purpose="export", fixture_mode=True) + coverage = DatasetConsumptionGate.evaluate(fixture, purpose="authoritative_coverage") + + assert qa.fixture_legacy_exception is True + assert inference_error.value.code == "DATASET_PROVENANCE_INCOMPLETE" + assert "fixture_qa_only" in inference_error.value.details["reasons"] + assert export_error.value.code == "DATASET_PROVENANCE_INCOMPLETE" + assert fixture_export_error.value.code == "DATASET_PROVENANCE_INCOMPLETE" + assert "fixture_qa_only" in fixture_export_error.value.details["reasons"] + assert coverage.eligible is False + assert "fixture_not_authoritative_coverage" in coverage.reasons + + +def test_unprovenanced_persistent_dataset_is_blocked(monkeypatch) -> None: + dataset = Dataset( + id=uuid4(), + project_id=uuid4(), + name="manual.tif", + dataset_type="raster", + source="manual_upload", + ) + monkeypatch.setattr(gate_module, "sa_inspect", lambda _dataset: SimpleNamespace(transient=False)) + + with pytest.raises(AppError) as exc_info: + DatasetConsumptionGate.assert_eligible( + dataset, + purpose="production_inference", + fixture_mode=True, + ) + + assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE" + assert "phase2_provenance_missing" in exc_info.value.details["reasons"] + assert "fixture_source_required" in exc_info.value.details["reasons"] + + +def test_transient_orm_test_double_can_only_bypass_missing_legacy_fields_for_qa() -> None: + transient = Dataset( + id=uuid4(), + project_id=uuid4(), + name="transient-test.tif", + dataset_type="raster", + source="manual_upload", + ) + + decision = DatasetConsumptionGate.assert_eligible(transient, purpose="quality_assessment") + coverage = DatasetConsumptionGate.evaluate(transient, purpose="authoritative_coverage") + with pytest.raises(AppError) as production_error: + DatasetConsumptionGate.assert_eligible(transient, purpose="production_inference") + + assert decision.fixture_legacy_exception is True + assert production_error.value.code == "DATASET_PROVENANCE_INCOMPLETE" + assert coverage.eligible is False + assert "phase2_provenance_missing" in coverage.reasons + + +@pytest.mark.parametrize("purpose", ("production_inference", "derived_processing", "export")) +def test_passed_manual_or_experimental_dataset_cannot_cross_production_boundary(purpose: str) -> None: + """A syntactically valid manual upload remains experimental, never production-ready.""" + + manual = _governed_dataset(source_key="manual", classification="experimental") + manual.source = "manual_upload" + + with pytest.raises(AppError) as exc_info: + DatasetConsumptionGate.assert_eligible(manual, purpose=purpose) # type: ignore[arg-type] + + assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE" + assert "experimental_source_not_allowed_for_purpose" in exc_info.value.details["reasons"] + + +def test_reference_validation_requires_authoritative_ground_truth_reference() -> None: + reference = _governed_dataset() + reference.dataset_type = "vector" + reference.dataset_role = "reference" + + decision = DatasetConsumptionGate.assert_eligible( + reference, + purpose="reference_validation", + reference_task="building_validation", + ) + assert decision.eligible is True + + reference.source_registry.classification = "corroborative" + with pytest.raises(AppError) as exc_info: + DatasetConsumptionGate.assert_eligible( + reference, + purpose="reference_validation", + reference_task="building_validation", + ) + + assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE" + assert "reference_source_not_authoritative" in exc_info.value.details["reasons"] + + +def test_pending_regional_building_authority_cannot_become_truth_without_approval() -> None: + reference = _governed_dataset(source_key="spw_picc", classification="authoritative") + reference.dataset_type = "vector" + reference.dataset_role = "reference" + reference.source_registry.authority_scope_json = {"zone": "Wallonia"} + reference.source_registry.usage_policy_json = { + "ground_truth_allowed": True, + "validation_authority": {"building_validation": "regional_primary_pending_contract"}, + } + + with pytest.raises(AppError) as exc_info: + DatasetConsumptionGate.assert_eligible( + reference, + purpose="reference_validation", + reference_task="building_validation", + ) + + assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE" + assert "reference_task_authority_not_approved" in exc_info.value.details["reasons"] + + +def test_source_snapshot_must_belong_to_the_dataset_source_registry() -> None: + dataset = _governed_dataset() + dataset.source_snapshot.source_registry_id = uuid4() + + with pytest.raises(AppError) as exc_info: + DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference") + + assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE" + assert "source_snapshot_registry_mismatch" in exc_info.value.details["reasons"] + + +@pytest.mark.parametrize("freshness_status", ("unknown", "review_required", "due", "stale")) +def test_non_consumable_source_snapshot_freshness_is_blocked_at_production_boundaries( + freshness_status: str, +) -> None: + dataset = _governed_dataset(snapshot_freshness_status=freshness_status) + + for purpose in ("production_inference", "authoritative_coverage"): + with pytest.raises(AppError) as exc_info: + DatasetConsumptionGate.assert_eligible(dataset, purpose=purpose) # type: ignore[arg-type] + + assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE" + assert "source_snapshot_freshness_not_eligible" in exc_info.value.details["reasons"] + + +def test_coverage_registry_ignores_explicitly_incomplete_materialization() -> None: + definition = next(item for item in SOURCE_DEFINITIONS if item.contract.source_name == "digitaal_vlaanderen") + unsafe_materialization = SimpleNamespace( + id=uuid4(), + status="ready", + source_name="grb", + validation_status="passed", + provenance_status="incomplete", + lineage_status="complete", + quarantine_status="not_quarantined", + ) + + matches, fully_covered = CoverageRegistryService._matching_datasets( + [unsafe_materialization], + definition, + "buildings", + "flanders", + box(4.0, 50.8, 4.1, 50.9), + ) + + assert matches == [] + assert fully_covered is False + + +def test_vector_export_is_fail_closed_before_selection(monkeypatch) -> None: + dataset = _governed_dataset() + dataset.dataset_type = "vector" + dataset.status = "quarantined" + queried = False + + class _Session: + @staticmethod + def get(model, item_id): + return dataset if model is Dataset and item_id == dataset.id else None + + def _unexpected_selection(*_args, **_kwargs): + nonlocal queried + queried = True + raise AssertionError("unsafe dataset must be rejected before querying vector features") + + monkeypatch.setattr( + "app.services.export_service.VectorFeatureService.select_features_by_bbox", + _unexpected_selection, + ) + + with pytest.raises(AppError) as exc_info: + ExportService.export_vector_selection_geojson( + _Session(), + dataset.id, + {"min_x": 4.0, "min_y": 50.0, "max_x": 4.1, "max_y": 50.1, "crs": "EPSG:4326"}, + ) + + assert exc_info.value.code == "DATASET_QUARANTINED" + assert queried is False diff --git a/backend/tests/test_docker_runtime_config.py b/backend/tests/test_docker_runtime_config.py index b29d730d..841a4d9b 100644 --- a/backend/tests/test_docker_runtime_config.py +++ b/backend/tests/test_docker_runtime_config.py @@ -74,6 +74,8 @@ def test_all_in_one_dockerfile_copies_operator_scripts_for_runtime_use() -> None "audit_operator_yolo_dataset_quality.py", "render_operator_yolo_label_qa_contact_sheets.py", "train_operator_yolo_detector.sh", + "training_dataset_eligibility.py", + "training_release_manifest.py", "verify_real_data_detection_qa_workflow.sh", "run_detection_quality_matrix.sh", "run_multi_sample_detection_quality_matrix.sh", diff --git a/backend/tests/test_failure_driven_yolo_sampling.py b/backend/tests/test_failure_driven_yolo_sampling.py index ae437379..b196ce3c 100644 --- a/backend/tests/test_failure_driven_yolo_sampling.py +++ b/backend/tests/test_failure_driven_yolo_sampling.py @@ -3,6 +3,8 @@ from __future__ import annotations import importlib.util from pathlib import Path +import pytest + SCRIPT = Path(__file__).parents[2] / "scripts" / "build_failure_driven_yolo_sampling.py" SPEC = importlib.util.spec_from_file_location("failure_sampling", SCRIPT) @@ -22,7 +24,7 @@ def test_dataset_validation_source_preserves_manifest_path(tmp_path: Path): assert MODULE.dataset_validation_source(source) == "/data/source/internal-val.txt" -def test_sampling_repeats_only_failed_region_train_tiles() -> None: +def test_sampling_rejects_protected_test_and_background_feedback() -> None: manifest = { "samples": [ {"sample_slug": "train-fl", "split": "train", "region": "flanders"}, @@ -54,15 +56,10 @@ def test_sampling_repeats_only_failed_region_train_tiles() -> None: }, "background": {"pure_empty_false_positives": 2}, } - paths, metadata = MODULE.build_sampling( - summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0 - ) - assert paths.count(str(Path("/tmp/fl-pos.png").resolve())) == 3 - assert paths.count(str(Path("/tmp/fl-neg.png").resolve())) == 4 - assert paths.count(str(Path("/tmp/wa-pos.png").resolve())) == 1 - assert not any("protected" in path for path in paths) - assert metadata["protected_samples_in_training"] == [] - assert metadata["weak_recall_regions"] == ["flanders"] + with pytest.raises(ValueError, match="protected test/background evidence"): + MODULE.build_sampling( + summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0 + ) def test_sampling_can_use_calibration_before_test_is_opened() -> None: @@ -171,7 +168,7 @@ def test_sampling_targets_failed_calibration_contexts_without_using_protected_ti assert metadata["recall_dominant_regions"] == ["flanders"] -def test_recall_dominance_does_not_suppress_negatives_when_background_gate_failed() -> None: +def test_sampling_rejects_background_feedback_after_a_protected_background_opening() -> None: manifest = {"samples": [ {"sample_slug": "positive", "split": "train", "region": "flanders", "context": "industrial"}, {"sample_slug": "negative", "split": "train", "region": "flanders", "context": "industrial-hard-negative"}, @@ -190,12 +187,10 @@ def test_recall_dominance_does_not_suppress_negatives_when_background_gate_faile "background": {"pure_empty_false_positives": 1}, } - paths, metadata = MODULE.build_sampling( - summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0, - ) - - assert paths.count(str(Path("/tmp/negative.png").resolve())) == 4 - assert metadata["recall_dominant_regions"] == [] + with pytest.raises(ValueError, match="protected background evidence"): + MODULE.build_sampling( + summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0, + ) def test_region_cap_drops_only_repeats_and_preserves_every_unique_tile() -> None: diff --git a/backend/tests/test_governed_dataset_ingest.py b/backend/tests/test_governed_dataset_ingest.py new file mode 100644 index 00000000..16a20725 --- /dev/null +++ b/backend/tests/test_governed_dataset_ingest.py @@ -0,0 +1,765 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from hashlib import sha256 +import json +from pathlib import Path +from uuid import UUID, uuid4 + +import pytest +from pyproj import Transformer + +from app.core.errors import AppError +from app.models import ( + Area, + Dataset, + DatasetQuarantine, + DatasetVersion, + Project, + SourceRegistry, + SourceSnapshot, + VectorFeature, +) +from app.services.dataset_service import DatasetService, _PartitionedGeoJsonRecords +from app.services.vector_operations_service import VectorOperationsService + + +class _Query: + def __init__(self, session: "_Session", model: type) -> None: + self.session = session + self.model = model + self.predicates = [] + + def filter(self, *predicates): + self.predicates.extend(predicates) + return self + + def one_or_none(self): + matches = self._matches() + if len(matches) > 1: + raise AssertionError( + f"expected one {self.model.__name__}, found {len(matches)}" + ) + return matches[0] if matches else None + + def all(self): + return self._matches() + + def _matches(self): + matches = list(self.session.rows.get(self.model, [])) + for predicate in self.predicates: + field_name = predicate.left.key + expected = predicate.right.value + operator_name = getattr(predicate.operator, "__name__", "") + if operator_name == "in_op": + matches = [ + item for item in matches if getattr(item, field_name) in expected + ] + else: + matches = [ + item for item in matches if getattr(item, field_name) == expected + ] + return matches + + +class _Session: + """Small ORM-shaped harness that exercises the real governed path.""" + + def __init__(self, project: Project) -> None: + self.rows: dict[type, list[object]] = {Project: [project]} + self.commits = 0 + self.rollbacks = 0 + self.flushes = 0 + + def get(self, model: type, item_id: UUID): + return next( + ( + item + for item in self.rows.get(model, []) + if getattr(item, "id", None) == item_id + ), + None, + ) + + def query(self, model: type) -> _Query: + return _Query(self, model) + + def add(self, item: object) -> None: + if getattr(item, "id", None) is None: + setattr(item, "id", uuid4()) + self.rows.setdefault(type(item), []).append(item) + + def flush(self) -> None: + self.flushes += 1 + + def commit(self) -> None: + self.commits += 1 + + def rollback(self) -> None: + self.rollbacks += 1 + + def refresh(self, _item: object) -> None: + return None + + def expunge(self, _item: object) -> None: + return None + + +def _storage_info(tmp_path: Path, content: bytes) -> dict[str, object]: + path = tmp_path / "grb-buildings.geojson" + path.write_bytes(content) + return { + "storage_path": str(path), + "original_filename": path.name, + "stored_filename": path.name, + "content_type": "application/geo+json", + "size_bytes": len(content), + "checksum_sha256": sha256(content).hexdigest(), + } + + +def _valid_payload() -> bytes: + return json.dumps( + { + "type": "FeatureCollection", + "crs": {"type": "name", "properties": {"name": "EPSG:4326"}}, + "features": [ + { + "type": "Feature", + "id": "gbg-1", + "properties": {"id": "gbg-1"}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]] + ], + }, + } + ], + } + ).encode("utf-8") + + +def _grb_payload_without_required_id() -> bytes: + return json.dumps( + { + "type": "FeatureCollection", + "crs": {"type": "name", "properties": {"name": "EPSG:4326"}}, + "features": [ + { + "type": "Feature", + "properties": {"unrelated": "not a GRB identity"}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]] + ], + }, + } + ], + } + ).encode("utf-8") + + +def _lambert_grb_payload() -> tuple[bytes, tuple[float, float, float, float]]: + """Create a valid GRB-shaped source artifact in its declared native CRS.""" + + longitude, latitude = 4.70, 51.10 + max_longitude, max_latitude = 4.7001, 51.1001 + to_lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True) + lambert_ring = [ + to_lambert.transform(longitude, latitude), + to_lambert.transform(max_longitude, latitude), + to_lambert.transform(max_longitude, max_latitude), + to_lambert.transform(longitude, latitude), + ] + return ( + json.dumps( + { + "type": "FeatureCollection", + "crs": {"type": "name", "properties": {"name": "EPSG:31370"}}, + "features": [ + { + "type": "Feature", + "id": "GBG.lambert.1", + "properties": {"id": "GBG.lambert.1"}, + "geometry": {"type": "Polygon", "coordinates": [lambert_ring]}, + } + ], + } + ).encode("utf-8"), + (longitude, latitude, max_longitude, max_latitude), + ) + + +def test_governed_vector_import_persists_snapshot_contract_and_queryable_features( + monkeypatch, tmp_path: Path +) -> None: + project = Project(id=uuid4(), name="Phase 2 governed ingest") + db = _Session(project) + raw = _valid_payload() + monkeypatch.setattr( + "app.services.dataset_service.StorageService.persist_dataset_file", + lambda **_kwargs: _storage_info(tmp_path, raw), + ) + + result = DatasetService.import_vector_bytes( + db, + project_id=project.id, + filename="grb-buildings.geojson", + content=raw, + source="grb_wfs", + source_name="grb", + dataset_role="reference", + reference_layer_name="buildings", + source_metadata={ + "license": "Open data", + "source_url": "https://example.invalid/grb", + }, + provenance_metadata={"adapter": "test"}, + temporal_series_key="grb:2026-08", + observed_at=datetime(2026, 8, 1, tzinfo=UTC), + source_version="2026-08-01", + temporal_granularity="snapshot", + ) + + dataset = next(item for item in db.rows[Dataset] if item.id == result.id) + snapshot = db.rows[SourceSnapshot][0] + source = db.rows[SourceRegistry][0] + assert result.status == "ready" + assert dataset.source_name == "grb" + assert dataset.source_registry_id == source.id + assert dataset.source_snapshot_id == snapshot.id + assert dataset.validation_status == "passed" + assert dataset.provenance_status == "complete" + assert dataset.lineage_status == "complete" + assert dataset.quarantine_status == "not_quarantined" + assert dataset.crs == "EPSG:4326" + assert snapshot.checksum_sha256 == sha256(raw).hexdigest() + assert len(db.rows[VectorFeature]) == 1 + assert db.commits == 1 + + # A retry with identical governed evidence is idempotent and does not + # create a second source snapshot, dataset or vector feature. + repeated = DatasetService.import_vector_bytes( + db, + project_id=project.id, + filename="grb-buildings.geojson", + content=raw, + source="grb_wfs", + source_name="grb", + dataset_role="reference", + reference_layer_name="buildings", + source_metadata={"license": "Open data"}, + provenance_metadata={"adapter": "test"}, + temporal_series_key="grb:2026-08", + observed_at=datetime(2026, 8, 1, tzinfo=UTC), + source_version="2026-08-01", + temporal_granularity="snapshot", + ) + assert repeated.id == result.id + assert len(db.rows[Dataset]) == 1 + assert len(db.rows[SourceSnapshot]) == 1 + assert len(db.rows[VectorFeature]) == 1 + + +def test_governed_lambert_geojson_persists_canonical_consumption_bytes_and_provenance_evidence( + monkeypatch, tmp_path: Path +) -> None: + """Projected source bytes must never be the file that vector operations consume.""" + + project = Project(id=uuid4(), name="Canonical GeoJSON storage") + db = _Session(project) + raw, (longitude, latitude, max_longitude, max_latitude) = _lambert_grb_payload() + consumption_path = tmp_path / "consumption" / "grb-buildings.geojson" + provenance_path = tmp_path / "provenance" / "grb-buildings.geojson" + + def _persist_dataset_file(**kwargs): + stored = kwargs["content"] + consumption_path.parent.mkdir(parents=True, exist_ok=True) + consumption_path.write_bytes(stored) + return _storage_info(consumption_path.parent, stored) + + def _persist_file(storage_path, content, original_filename, content_type): + del storage_path, original_filename, content_type + provenance_path.parent.mkdir(parents=True, exist_ok=True) + provenance_path.write_bytes(content) + return _storage_info(provenance_path.parent, content) + + monkeypatch.setattr( + "app.services.dataset_service.StorageService.persist_dataset_file", + _persist_dataset_file, + ) + monkeypatch.setattr( + "app.services.dataset_service.StorageService.persist_file", + _persist_file, + ) + + result = DatasetService.import_vector_bytes( + db, + project_id=project.id, + filename="grb-lambert.geojson", + content=raw, + source="grb_wfs", + source_name="grb", + dataset_role="reference", + reference_layer_name="buildings", + source_metadata={"license": "Open data"}, + provenance_metadata={"adapter": "test"}, + temporal_series_key="grb:lambert:2026-08", + observed_at=datetime(2026, 8, 1, tzinfo=UTC), + source_version="2026-08-01-lambert", + temporal_granularity="snapshot", + ) + + dataset = next(item for item in db.rows[Dataset] if item.id == result.id) + dataset_version = db.rows[DatasetVersion][0] + snapshot = db.rows[SourceSnapshot][0] + canonical_bytes = Path(str(dataset.storage_path)).read_bytes() + canonical_payload = json.loads(canonical_bytes) + source_artifact = dataset.provenance_metadata["source_artifact"] + + assert result.status == "ready" + assert dataset.crs == "EPSG:4326" + assert canonical_payload["crs"]["properties"]["name"] == "EPSG:4326" + assert canonical_payload["features"][0]["geometry"]["coordinates"][0][0] == pytest.approx( + [longitude, latitude], abs=0.000001 + ) + assert sha256(canonical_bytes).hexdigest() == dataset.checksum_sha256 + assert dataset_version.checksum_sha256 == dataset.checksum_sha256 + assert snapshot.checksum_sha256 == dataset.checksum_sha256 + assert source_artifact["retention"] == "provenance_evidence_only" + assert source_artifact["checksum_sha256"] == sha256(raw).hexdigest() + assert source_artifact["storage_path"] != dataset.storage_path + assert Path(source_artifact["storage_path"]).read_bytes() == raw + assert dataset.provenance_metadata["canonical_consumption_artifact"] == { + "checksum_sha256": dataset.checksum_sha256, + "crs": "EPSG:4326", + "storage_role": "dataset_consumption", + } + + inspection = VectorOperationsService.inspect(db, dataset.id) + assert inspection.crs == "EPSG:4326" + assert inspection.bounds_json == { + "min_x": pytest.approx(longitude, abs=0.000001), + "min_y": pytest.approx(latitude, abs=0.000001), + "max_x": pytest.approx(max_longitude, abs=0.000001), + "max_y": pytest.approx(max_latitude, abs=0.000001), + } + response_payload = DatasetService.get_dataset_geojson(db, dataset.id) + assert response_payload["features"][0]["geometry"]["coordinates"][0][0] == pytest.approx( + [longitude, latitude], abs=0.000001 + ) + + # The storage identity is enforced at the operation boundary too; a + # replacement with different canonical bytes is not silently processed. + Path(str(dataset.storage_path)).write_bytes(canonical_bytes + b"\n") + with pytest.raises(AppError) as exc_info: + VectorOperationsService.inspect(db, dataset.id) + assert exc_info.value.code == "DATASET_STORAGE_CHECKSUM_MISMATCH" + + +def test_metadata_refresh_refuses_mutated_governed_artifact( + monkeypatch, tmp_path: Path +) -> None: + """A passed snapshot cannot be silently re-described from mutable storage.""" + + project = Project(id=uuid4(), name="Phase 2 immutable refresh") + db = _Session(project) + raw = _valid_payload() + monkeypatch.setattr( + "app.services.dataset_service.StorageService.persist_dataset_file", + lambda **_kwargs: _storage_info(tmp_path, raw), + ) + result = DatasetService.import_vector_bytes( + db, + project_id=project.id, + filename="grb-buildings.geojson", + content=raw, + source="grb_wfs", + source_name="grb", + dataset_role="reference", + reference_layer_name="buildings", + source_metadata={"license": "Open data"}, + provenance_metadata={"adapter": "test"}, + temporal_series_key="grb:2026-08", + observed_at=datetime(2026, 8, 1, tzinfo=UTC), + source_version="2026-08-01", + temporal_granularity="snapshot", + ) + dataset = next(item for item in db.rows[Dataset] if item.id == result.id) + original_checksum = dataset.checksum_sha256 + original_metadata = dict(dataset.metadata_json or {}) + original_commit_count = db.commits + + # Simulate an out-of-band storage replacement at the same path. The + # refresh endpoint must not parse it into an already-passed contract row. + Path(str(dataset.storage_path)).write_bytes(_grb_payload_without_required_id()) + with pytest.raises(AppError) as exc_info: + DatasetService.refresh_metadata(db, dataset.id) + + assert exc_info.value.code == "GOVERNED_DATASET_REINGEST_REQUIRED" + assert dataset.status == "ready" + assert dataset.validation_status == "passed" + assert dataset.checksum_sha256 == original_checksum + assert dataset.metadata_json == original_metadata + assert db.commits == original_commit_count + + +def test_governed_import_quarantines_bad_artifacts_and_refuses_unknown_source( + monkeypatch, tmp_path: Path +) -> None: + project = Project(id=uuid4(), name="Phase 2 quarantine") + db = _Session(project) + raw = b'{"type":"FeatureCollection","features":[]}' + monkeypatch.setattr( + "app.services.dataset_service.StorageService.persist_dataset_file", + lambda **_kwargs: _storage_info(tmp_path, raw), + ) + + quarantined = DatasetService.import_vector_bytes( + db, + project_id=project.id, + filename="empty.geojson", + content=raw, + source="grb_wfs", + source_name="grb", + dataset_role="reference", + reference_layer_name="buildings", + source_metadata={"license": "Open data"}, + provenance_metadata={}, + temporal_series_key="grb:2026-08-empty", + observed_at=datetime(2026, 8, 1, tzinfo=UTC), + source_version="2026-08-01-empty", + temporal_granularity="snapshot", + ) + assert quarantined.status == "quarantined" + assert quarantined.validation_status == "failed" + assert quarantined.quarantine_status == "quarantined" + assert len(db.rows[DatasetQuarantine]) == 1 + assert db.rows[SourceSnapshot][0].ingest_status == "quarantined" + + with pytest.raises(AppError) as exc_info: + DatasetService.import_vector_bytes( + db, + project_id=project.id, + filename="unregistered.geojson", + content=_valid_payload(), + source="caller_controlled", + source_name="caller_claimed_grb", + dataset_role="reference", + reference_layer_name="buildings", + source_metadata={"license": "Open data"}, + provenance_metadata={}, + ) + assert exc_info.value.code == "SOURCE_REGISTRY_ENTRY_NOT_FOUND" + + +def test_governed_grb_vector_quarantines_missing_server_owned_required_attribute( + monkeypatch, tmp_path: Path +) -> None: + project = Project(id=uuid4(), name="Phase 2 source schema") + db = _Session(project) + raw = _grb_payload_without_required_id() + monkeypatch.setattr( + "app.services.dataset_service.StorageService.persist_dataset_file", + lambda **kwargs: _storage_info(tmp_path, kwargs["content"]), + ) + + quarantined = DatasetService.import_vector_bytes( + db, + project_id=project.id, + filename="grb-missing-id.geojson", + content=raw, + source="grb_wfs", + source_name="grb", + dataset_role="reference", + reference_layer_name="buildings", + source_metadata={"license": "Open data"}, + provenance_metadata={"adapter": "test"}, + temporal_series_key="grb:missing-id", + observed_at=datetime(2026, 8, 1, tzinfo=UTC), + source_version="2026-08-01-missing-id", + temporal_granularity="snapshot", + ) + + dataset = next(item for item in db.rows[Dataset] if item.id == quarantined.id) + assert quarantined.status == "quarantined" + assert dataset.validation_status == "failed" + assert dataset.quarantine_status == "quarantined" + issue = dataset.validation_report_json["issues"][0] + assert issue["code"] == "SOURCE_SCHEMA_REQUIRED_ATTRIBUTE_MISSING" + assert issue["category"] == "source_schema" + assert len(db.rows[DatasetQuarantine]) == 1 + + +def test_partitioned_vector_ingest_is_idempotent_and_quarantines_noncanonical_partition_coordinates( + monkeypatch, + tmp_path: Path, +) -> None: + project = Project(id=uuid4(), name="Partitioned governed ingest") + area = Area(id=uuid4(), project_id=project.id, name="Partitioned AOI") + db = _Session(project) + db.rows[Area] = [area] + + feature = { + "type": "Feature", + "id": "GBG.1", + "properties": {"id": "GBG.1", "source_feature_id": "GBG.1"}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]] + ], + }, + } + partition_payload = {"type": "FeatureCollection", "features": [feature]} + partition_path = tmp_path / "partition-01.geojson" + partition_path.write_text(json.dumps(partition_payload), encoding="utf-8") + artifact_payload = { + "type": "FeatureCollection", + "crs": "EPSG:4326", + "features": [feature], + } + artifact_path = tmp_path / "grb-partitioned.geojson" + artifact_raw = json.dumps(artifact_payload).encode("utf-8") + artifact_path.write_bytes(artifact_raw) + monkeypatch.setattr( + "app.services.dataset_service.StorageService.persist_dataset_file_from_path", + lambda **_kwargs: _storage_info(tmp_path, artifact_raw), + ) + + result = DatasetService.import_partitioned_vector_artifact( + db, + project_id=project.id, + area_id=area.id, + artifact_path=artifact_path, + partition_paths=[partition_path], + original_filename="grb-partitioned.geojson", + source="operator_official_import", + dataset_role="reference", + source_name="grb", + reference_layer_name="buildings", + metadata_json={ + "feature_count": 1, + "crs": "EPSG:4326", + "bounds_json": { + "min_x": 4.69, + "min_y": 51.09, + "max_x": 4.70, + "max_y": 51.10, + }, + }, + source_metadata={"license": "Open data"}, + provenance_metadata={ + "artifact_sha256": sha256(artifact_raw).hexdigest(), + "partition_checksums": { + partition_path.name: sha256(partition_path.read_bytes()).hexdigest() + }, + }, + temporal_series_key="grb:partitioned:test", + observed_at=datetime(2026, 8, 1, tzinfo=UTC), + source_version="2026-08-01", + ) + + dataset = next(item for item in db.rows[Dataset] if item.id == result.id) + assert result.status == "ready" + assert dataset.validation_status == "passed" + assert dataset.provenance_status == "complete" + assert dataset.source_name == "grb" + assert len(db.rows[SourceSnapshot]) == 1 + assert len(db.rows[VectorFeature]) == 1 + assert dataset.metadata_json["partitioned_geometry_audit"][ + "partition_checksums_sha256" + ] == {partition_path.name: sha256(partition_path.read_bytes()).hexdigest()} + assert dataset.provenance_metadata["partition_checksum_manifest_sha256"] + assert dataset.provenance_metadata["partitioned_artifact_binding_sha256"] + + repeated = DatasetService.import_partitioned_vector_artifact( + db, + project_id=project.id, + area_id=area.id, + artifact_path=artifact_path, + partition_paths=[partition_path], + original_filename="grb-partitioned.geojson", + source="operator_official_import", + dataset_role="reference", + source_name="grb", + reference_layer_name="buildings", + metadata_json={ + "feature_count": 1, + "crs": "EPSG:4326", + "bounds_json": { + "min_x": 4.69, + "min_y": 51.09, + "max_x": 4.70, + "max_y": 51.10, + }, + }, + source_metadata={"license": "Open data"}, + provenance_metadata={ + "artifact_sha256": sha256(artifact_raw).hexdigest(), + "partition_checksums": { + partition_path.name: sha256(partition_path.read_bytes()).hexdigest() + }, + }, + temporal_series_key="grb:partitioned:test", + observed_at=datetime(2026, 8, 1, tzinfo=UTC), + source_version="2026-08-01", + ) + assert repeated.id == result.id + assert len(db.rows[Dataset]) == 1 + assert len(db.rows[VectorFeature]) == 1 + + lambert_feature = { + **feature, + "id": "GBG.lambert", + "properties": {"id": "GBG.lambert"}, + "geometry": { + "type": "Polygon", + "coordinates": [ + [[150000, 170000], [150010, 170000], [150010, 170010], [150000, 170000]] + ], + }, + } + lambert_partition = tmp_path / "partition-lambert.geojson" + lambert_partition.write_text( + json.dumps({"type": "FeatureCollection", "features": [lambert_feature]}), + encoding="utf-8", + ) + lambert_artifact = tmp_path / "grb-lambert.geojson" + lambert_raw = json.dumps( + {"type": "FeatureCollection", "features": [lambert_feature]} + ).encode("utf-8") + lambert_artifact.write_bytes(lambert_raw) + monkeypatch.setattr( + "app.services.dataset_service.StorageService.persist_dataset_file_from_path", + lambda **_kwargs: _storage_info(tmp_path, lambert_raw), + ) + + quarantined = DatasetService.import_partitioned_vector_artifact( + db, + project_id=project.id, + area_id=area.id, + artifact_path=lambert_artifact, + partition_paths=[lambert_partition], + original_filename="grb-lambert.geojson", + source="operator_official_import", + dataset_role="reference", + source_name="grb", + reference_layer_name="buildings", + metadata_json={ + "feature_count": 1, + "crs": "EPSG:4326", + "bounds_json": { + "min_x": 150000, + "min_y": 170000, + "max_x": 150010, + "max_y": 170010, + }, + }, + source_metadata={"license": "Open data"}, + provenance_metadata={ + "artifact_sha256": sha256(lambert_raw).hexdigest(), + "partition_checksums": { + lambert_partition.name: sha256( + lambert_partition.read_bytes() + ).hexdigest() + }, + }, + temporal_series_key="grb:partitioned:lambert", + observed_at=datetime(2026, 8, 2, tzinfo=UTC), + source_version="2026-08-02", + ) + assert quarantined.status == "quarantined" + assert quarantined.validation_status == "failed" + assert quarantined.quarantine_status == "quarantined" + + missing_manifest_partition = tmp_path / "partition-missing-manifest.geojson" + missing_manifest_partition.write_text( + json.dumps(partition_payload), encoding="utf-8" + ) + missing_manifest_artifact = tmp_path / "grb-missing-manifest.geojson" + missing_manifest_raw = json.dumps( + {"type": "FeatureCollection", "features": [feature]} + ).encode("utf-8") + missing_manifest_artifact.write_bytes(missing_manifest_raw) + monkeypatch.setattr( + "app.services.dataset_service.StorageService.persist_dataset_file_from_path", + lambda **_kwargs: _storage_info(tmp_path, missing_manifest_raw), + ) + + missing_manifest = DatasetService.import_partitioned_vector_artifact( + db, + project_id=project.id, + area_id=area.id, + artifact_path=missing_manifest_artifact, + partition_paths=[missing_manifest_partition], + original_filename="grb-missing-manifest.geojson", + source="operator_official_import", + dataset_role="reference", + source_name="grb", + reference_layer_name="buildings", + metadata_json={ + "feature_count": 1, + "crs": "EPSG:4326", + "bounds_json": { + "min_x": 4.69, + "min_y": 51.09, + "max_x": 4.70, + "max_y": 51.10, + }, + }, + source_metadata={"license": "Open data"}, + provenance_metadata={ + "artifact_sha256": sha256(missing_manifest_raw).hexdigest() + }, + temporal_series_key="grb:partitioned:missing-manifest", + observed_at=datetime(2026, 8, 3, tzinfo=UTC), + source_version="2026-08-03", + ) + assert missing_manifest.status == "quarantined" + assert ( + missing_manifest.validation_report_json["issues"][0]["code"] + == "PARTITION_CHECKSUM_MANIFEST_REQUIRED" + ) + + +def test_partitioned_geometry_audit_handles_more_than_generic_topology_limit_without_materializing_geometries( + tmp_path: Path, +) -> None: + feature_count = 10_001 + partition_path = tmp_path / "large-partition.geojson" + partition_path.write_text( + json.dumps( + { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "id": f"GBG.{index}", + "properties": {"id": f"GBG.{index}"}, + "geometry": { + "type": "Point", + "coordinates": [4.0 + index / 10_000_000, 51.0], + }, + } + for index in range(feature_count) + ], + } + ), + encoding="utf-8", + ) + + audit = _PartitionedGeoJsonRecords( + [partition_path], + expected_feature_count=feature_count, + declared_partition_checksums={ + partition_path.name: sha256(partition_path.read_bytes()).hexdigest() + }, + ).audit() + + assert audit.feature_count == feature_count + assert audit.bounds_json["min_x"] == 4.0 + assert audit.bounds_json["max_x"] > audit.bounds_json["min_x"] + assert audit.representative_record.geometry.geom_type == "MultiPoint" diff --git a/backend/tests/test_grayscale_yolo_dataset.py b/backend/tests/test_grayscale_yolo_dataset.py index 9d3291d6..0c11fd74 100644 --- a/backend/tests/test_grayscale_yolo_dataset.py +++ b/backend/tests/test_grayscale_yolo_dataset.py @@ -3,6 +3,7 @@ from __future__ import annotations import json import subprocess import sys +from hashlib import sha256 from pathlib import Path from PIL import Image @@ -13,35 +14,114 @@ SCRIPT = Path(__file__).parents[2] / "scripts" / "build_grayscale_yolo_dataset.p def test_grayscale_builder_preserves_labels_and_split(tmp_path: Path) -> None: source = tmp_path / "source" - (source / "images" / "train").mkdir(parents=True) - (source / "labels" / "train").mkdir(parents=True) - image = source / "images" / "train" / "tile.png" - label = source / "labels" / "train" / "tile.txt" - Image.new("RGB", (8, 8), (255, 0, 0)).save(image) - label.write_text("0 0.5 0.5 0.5 0.5\n", encoding="utf-8") + entries = [] + for split, sample_slug, colour in (("train", "fixture-train", (255, 0, 0)), ("val", "fixture-val", (0, 255, 0))): + image = source / "images" / split / f"{sample_slug}.png" + label = source / "labels" / split / f"{sample_slug}.txt" + image.parent.mkdir(parents=True, exist_ok=True) + label.parent.mkdir(parents=True, exist_ok=True) + Image.new("RGB", (8, 8), colour).save(image) + label.write_text("0 0.5 0.5 0.5 0.5\n", encoding="utf-8") + entries.append((split, sample_slug, image, label)) + manifest = source / "operator_samples_manifest.json" + policy = "geointel-training-source-eligibility/v1" + manifest.write_text( + json.dumps( + { + "immutable": True, + "training_eligibility": {"policy_version": policy, "status": "eligible", "fixture_mode": True}, + "samples": [ + { + "sample_slug": sample_slug, + "split": split, + "raster_dataset_id": f"raster:{sample_slug}", + "reference_dataset_id": f"reference:{sample_slug}", + "training_eligibility": { + "policy_version": policy, + "eligible": True, + "fixture_mode": True, + "raster": {"eligible": True, "reasons": [], "evidence": {"dataset_id": f"raster:{sample_slug}", "checksum_sha256": "a" * 64, "source_registry_id": "fixture-raster", "source_snapshot_id": "fixture-raster-snapshot"}}, + "reference": {"eligible": True, "reasons": [], "evidence": {"dataset_id": f"reference:{sample_slug}", "checksum_sha256": "b" * 64, "source_registry_id": "fixture-reference", "source_snapshot_id": "fixture-reference-snapshot"}}, + }, + } + for split, sample_slug, _image, _label in entries + ], + } + ), + encoding="utf-8", + ) + (source / "corpus-freeze.json").write_text( + json.dumps( + { + "schema_version": 2, + "immutable": True, + "fixture_mode": True, + "training_eligibility_policy": policy, + "manifest_sha256": sha256(manifest.read_bytes()).hexdigest(), + } + ), + encoding="utf-8", + ) + dataset_yaml = source / "dataset.yaml" + dataset_yaml.write_text( + f"path: {source}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n", + encoding="utf-8", + ) + release_script = SCRIPT.parent / "training_release_manifest.py" + subprocess.run( + [ + sys.executable, + str(release_script), + "create", + "--train-yaml", + str(dataset_yaml), + "--corpus-manifest", + str(manifest), + "--fixture-mode", + ], + check=True, + ) + release_path = dataset_yaml.with_name(dataset_yaml.name + ".geointel-training-release.json") + asset_path = dataset_yaml.with_name(dataset_yaml.name + ".geointel-training-assets.json") + assets = json.loads(asset_path.read_text(encoding="utf-8")) summary = source / "yolo_tile_dataset_summary.json" summary.write_text( json.dumps( { + "dataset_yaml": str(dataset_yaml.resolve()), + "training_release_manifest": str(release_path.resolve()), + "training_release_manifest_sha256": sha256(release_path.read_bytes()).hexdigest(), + "training_asset_manifest": str(asset_path.resolve()), + "source_manifest_sha256": sha256(manifest.read_bytes()).hexdigest(), "tiles": [ - { - "split": "train", - "image_path": str(image), - "label_path": str(label), - } - ] + {"split": entry["split"], "image_path": entry["image_path"], "label_path": entry["label_path"]} + for entry in assets["entries"] + ], } ), encoding="utf-8", ) output = tmp_path / "gray" subprocess.run( - [sys.executable, str(SCRIPT), "--summary", str(summary), "--output-dir", str(output)], + [ + sys.executable, + str(SCRIPT), + "--summary", + str(summary), + "--train-yaml", + str(dataset_yaml), + "--corpus-manifest", + str(manifest), + "--fixture-mode", + "--output-dir", + str(output), + ], check=True, ) - converted = Image.open(output / "images" / "train" / "tile.png") + converted = Image.open(output / "images" / "train" / "fixture-train.png") r, g, b = converted.getpixel((0, 0)) assert r == g == b - assert (output / "labels" / "train" / "tile.txt").read_text() == label.read_text() + assert (output / "labels" / "train" / "fixture-train.txt").read_text() == "0 0.5 0.5 0.5 0.5\n" evidence = json.loads((output / "grayscale-dataset-evidence.json").read_text()) - assert evidence["converted_tile_count"] == 1 + assert evidence["converted_tile_count"] == 2 + assert evidence["training_eligible"] is False diff --git a/backend/tests/test_model_asset_catalog.py b/backend/tests/test_model_asset_catalog.py index a5c7be7f..ef0c676a 100644 --- a/backend/tests/test_model_asset_catalog.py +++ b/backend/tests/test_model_asset_catalog.py @@ -1,5 +1,6 @@ from __future__ import annotations +from hashlib import sha256 import json from pathlib import Path from uuid import uuid4 @@ -10,9 +11,10 @@ from fastapi.testclient import TestClient from app.core.config import Settings from app.core.errors import AppError from app.main import app -from app.models import AnalysisRun, Dataset, Detection, Job, Project +from app.models import AnalysisRun, Dataset, Detection, Job, Project, SourceRegistry, SourceSnapshot from app.services.detection_service import DetectionService from app.services.model_asset_catalog_service import ModelAssetCatalogService +from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService class FakeSession: @@ -63,15 +65,47 @@ class MockYoloAdapter: def _project_and_raster_dataset(): project_id = uuid4() dataset_id = uuid4() + source_registry_id = uuid4() + source_snapshot_id = uuid4() + checksum = "a" * 64 project = Project(id=project_id, name="Geel") + source_registry = SourceRegistry( + id=source_registry_id, + source_key="test-derived-raster", + display_name="Governed test-derived raster", + classification="derived", + authority_name="GeoIntel test fixture", + usage_policy_json={"ground_truth_allowed": False}, + ) + source_snapshot = SourceSnapshot( + id=source_snapshot_id, + source_registry_id=source_registry_id, + snapshot_key="test-derived-raster-v1", + checksum_sha256=checksum, + freshness_status="current", + ingest_status="ingested", + ) dataset = Dataset( id=dataset_id, project_id=project_id, name="source.tif", dataset_type="raster", - source="user_upload", + source="test-derived-raster", + source_name="test-derived-raster", storage_path="storage/uploads/source.tif", + checksum_sha256=checksum, + source_registry_id=source_registry_id, + source_snapshot_id=source_snapshot_id, + data_contract_key="geointel.raster.geotiff", + data_contract_version="1.0.0", + validation_status="passed", + provenance_status="complete", + lineage_status="not_applicable", + quarantine_status="not_quarantined", + status="ready", ) + dataset.source_registry = source_registry + dataset.source_snapshot = source_snapshot db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset}) return db, project_id, dataset_id @@ -101,6 +135,72 @@ def _manifest(tmp_path: Path) -> Path: return manifest_path +def _write_model_sidecar( + model_path: Path, + settings: Settings, + *, + db: FakeSession | None = None, +) -> None: + model_sha256 = sha256(model_path.read_bytes()).hexdigest() + source_registry_id = uuid4() + source_snapshot_id = uuid4() + source_version = settings.yolo_model_version or "test-v1" + if db is not None: + source_registry = SourceRegistry( + id=source_registry_id, + source_key="model", + display_name="Governed test model artifact", + classification="experimental", + authority_name="GeoIntel test fixture", + freshness_status="current", + ingest_status="configured", + ) + source_snapshot = SourceSnapshot( + id=source_snapshot_id, + source_registry_id=source_registry_id, + snapshot_key=f"model-{source_version}", + source_version=source_version, + checksum_sha256=model_sha256, + freshness_status="current", + ingest_status="ingested", + ) + db.objects[(SourceRegistry, source_registry_id)] = source_registry + db.objects[(SourceSnapshot, source_snapshot_id)] = source_snapshot + payload = { + "schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION, + "data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"}, + "model": { + "model_id": settings.yolo_model_id, + "task_type": "object_detection", + "sha256": model_sha256, + "model_format": "pytorch", + "framework": "ultralytics/pytorch", + "class_mapping": {"0": "building"}, + "source_version": source_version, + }, + "source": { + "source_registry_id": str(source_registry_id), + "source_snapshot_id": str(source_snapshot_id), + "source_registry_key": "model", + "source_snapshot_checksum_sha256": model_sha256, + }, + "lineage": { + "upstream_asset_ids": ["test-training-corpus"], + "upstream_checksums_sha256": ["a" * 64], + "transformations": [ + {"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64} + ], + }, + "metadata": {"training_manifest_sha256": "c" * 64}, + "imported_at": "2026-08-01T10:00:00+00:00", + } + payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload) + RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text( + json.dumps(payload, sort_keys=True), + encoding="utf-8", + ) + + def test_model_asset_catalog_lists_supported_local_model_files(tmp_path: Path) -> None: model_file = tmp_path / "building-detector.pt" model_file.write_bytes(b"local model") @@ -190,6 +290,7 @@ def test_detection_run_persists_selected_model_asset_parameters(tmp_path: Path) yolo_models_dir=str(tmp_path), yolo_max_tiles=4, ) + _write_model_sidecar(model_file, settings, db=db) result = DetectionService.run_detection( db=db, diff --git a/backend/tests/test_phase2_derived_dataset_governance.py b/backend/tests/test_phase2_derived_dataset_governance.py new file mode 100644 index 00000000..0ad7620e --- /dev/null +++ b/backend/tests/test_phase2_derived_dataset_governance.py @@ -0,0 +1,644 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from hashlib import sha256 +import json +from pathlib import Path +from types import SimpleNamespace +from uuid import uuid4 + +from app.models import Dataset, DatasetVersion, SourceRegistry, SourceSnapshot +from app.services.data_contract_validation import ( + build_vector_ingest_input, + validate_registered_asset, +) +from app.services.demo_workflow_service import DemoWorkflowService +from app.services.derived_dataset_governance_service import ( + DerivedDatasetGovernanceService, +) +from app.services.raster_operations_service import RasterOperationsService +from app.services.storage_service import StorageService +from app.services.vector_feature_service import VectorFeatureService +from app.services.vector_operations_service import VectorOperationsService + + +_CHECKSUM = "a" * 64 + + +class _FakeSession: + def __init__(self, rows=None) -> None: + self.rows = rows or {} + self.added = [] + + def get(self, model, row_id): + row = self.rows.get((model, row_id)) + if row is not None: + return row + return next( + ( + item + for item in self.added + if isinstance(item, model) and item.id == row_id + ), + None, + ) + + def add(self, row) -> None: + self.added.append(row) + + def commit(self) -> None: + return None + + def refresh(self, row) -> None: + return None + + +class _GovernedSession: + """Small ORM-shaped session for the real governance branch. + + Registry persistence is monkeypatched below; the test exercises the + service's orchestration and report decisions without needing PostGIS. + """ + + def __init__(self) -> None: + self.added = [] + self.flushes = 0 + + class _EmptyQuery: + def filter(self, *_args, **_kwargs): + return self + + @staticmethod + def all(): + return [] + + @staticmethod + def one_or_none(): + return None + + def query(self, *_args, **_kwargs): + # Governance now performs a bounded lineage traversal during quarantine. + # This focused harness intentionally has no persisted siblings/edges. + return self._EmptyQuery() + + def add(self, row) -> None: + self.added.append(row) + + def flush(self) -> None: + self.flushes += 1 + + +def _governed_parent() -> Dataset: + source_id = uuid4() + snapshot_id = uuid4() + source = SourceRegistry( + id=source_id, + source_key="grb", + display_name="GRB parent fixture", + classification="authoritative", + authority_name="Digitaal Vlaanderen", + authority_scope_json={"zone": "Flanders"}, + usage_policy_json={"ground_truth_allowed": True}, + ) + snapshot = SourceSnapshot( + id=snapshot_id, + source_registry_id=source_id, + snapshot_key="governed-parent", + checksum_sha256=_CHECKSUM, + freshness_status="current", + ingest_status="ingested", + ) + dataset = Dataset( + id=uuid4(), + project_id=uuid4(), + name="governed.geojson", + dataset_type="vector", + source="grb", + source_name="grb", + status="ready", + checksum_sha256=_CHECKSUM, + data_contract_key="geointel.vector.geojson", + data_contract_version="1.0.0", + validation_status="passed", + provenance_status="complete", + lineage_status="not_applicable", + quarantine_status="not_quarantined", + source_registry_id=source_id, + source_snapshot_id=snapshot_id, + ) + dataset.source_registry = source + dataset.source_snapshot = snapshot + return dataset + + +def test_lineage_evidence_quarantines_ungoverned_parent_without_inventing_a_checksum() -> ( + None +): + parent = _governed_parent() + valid_lineage = DerivedDatasetGovernanceService._lineage_evidence( + parent, "vector.clip", {"area_id": "a"} + ) + valid_report = validate_registered_asset( + build_vector_ingest_input( + asset_id="derived-valid", + source_crs="EPSG:4326", + storage_crs="EPSG:4326", + feature_collection={ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [5.0, 51.0]}, + "properties": {}, + } + ], + }, + checksum_sha256=_CHECKSUM, + computed_checksum_sha256=_CHECKSUM, + source_registry_id="derived-source", + source_snapshot_id="derived-snapshot", + imported_at=datetime.now(timezone.utc), + metadata={ + "license": "internal derived artifact", + "bounds_json": { + "min_x": 5.0, + "min_y": 51.0, + "max_x": 5.0, + "max_y": 51.0, + }, + }, + temporal_unknown_reason="derived input has no precise observation timestamp", + source_version_unknown_reason="transform version is recorded separately", + lineage=valid_lineage, + ) + ) + + assert valid_report.validation_status.value == "passed" + assert valid_lineage.upstream_asset_ids == (str(parent.id),) + assert valid_lineage.upstream_checksums_sha256 == (_CHECKSUM,) + + parent.validation_status = "not_validated" + rejected_lineage = DerivedDatasetGovernanceService._lineage_evidence( + parent, "vector.clip", {} + ) + rejected_report = validate_registered_asset( + build_vector_ingest_input( + asset_id="derived-rejected", + source_crs="EPSG:4326", + storage_crs="EPSG:4326", + feature_collection={ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [5.0, 51.0]}, + "properties": {}, + } + ], + }, + checksum_sha256=_CHECKSUM, + computed_checksum_sha256=_CHECKSUM, + source_registry_id="derived-source", + source_snapshot_id="derived-snapshot", + imported_at=datetime.now(timezone.utc), + metadata={ + "license": "internal derived artifact", + "bounds_json": { + "min_x": 5.0, + "min_y": 51.0, + "max_x": 5.0, + "max_y": 51.0, + }, + }, + temporal_unknown_reason="derived input has no precise observation timestamp", + source_version_unknown_reason="transform version is recorded separately", + lineage=rejected_lineage, + ) + ) + + assert rejected_lineage.upstream_checksums_sha256 == ( + "parent_dataset_not_governed", + ) + assert rejected_report.validation_status.value == "failed" + assert rejected_report.quarantine_status.value == "quarantined" + assert any( + issue.code == "UPSTREAM_CHECKSUM_FORMAT_INVALID" + for issue in rejected_report.issues + ) + + +def test_govern_vector_binds_snapshot_contract_and_lineage_before_marking_ready( + monkeypatch, +) -> None: + from app.services.source_registry_service import SourceRegistryService + + db = _GovernedSession() + parent = _governed_parent() + parent_version_id = uuid4() + dataset = Dataset( + id=uuid4(), + project_id=parent.project_id, + name="derived.geojson", + dataset_type="vector", + source="operation:clip", + source_name="derived", + checksum_sha256=_CHECKSUM, + imported_at=datetime.now(timezone.utc), + crs="EPSG:4326", + metadata_json={ + "bounds_json": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.0, "max_y": 51.0} + }, + status="validating", + ) + version = DatasetVersion( + id=uuid4(), dataset_id=dataset.id, version=1, checksum_sha256=_CHECKSUM + ) + source = SimpleNamespace(id=uuid4(), license_name="internal derived artifact") + snapshot = SimpleNamespace(id=uuid4(), source_registry_id=source.id) + edges = [] + + monkeypatch.setattr( + SourceRegistryService, + "ensure_server_owned_source", + lambda *_args, **_kwargs: source, + ) + monkeypatch.setattr( + SourceRegistryService, "record_snapshot", lambda *_args, **_kwargs: snapshot + ) + monkeypatch.setattr( + DerivedDatasetGovernanceService, + "_latest_parent_version_id", + lambda *_args: parent_version_id, + ) + + def _bind(target, **kwargs): + target.source_registry_id = kwargs["source"].id + target.source_snapshot_id = kwargs["snapshot"].id + target.data_contract_key = kwargs["data_contract_key"] + target.data_contract_version = kwargs["data_contract_version"] + target.validation_status = kwargs["validation_status"] + target.provenance_status = kwargs["provenance_status"] + target.lineage_status = kwargs["lineage_status"] + return target + + monkeypatch.setattr(SourceRegistryService, "bind_dataset_provenance", _bind) + monkeypatch.setattr(SourceRegistryService, "bind_dataset_version_provenance", _bind) + monkeypatch.setattr( + SourceRegistryService, + "record_lineage_edge", + lambda *_args, **kwargs: edges.append(kwargs), + ) + + ready = DerivedDatasetGovernanceService.govern_vector( + db, + dataset=dataset, + dataset_version=version, + feature_collection={ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [5.0, 51.0]}, + "properties": {}, + } + ], + }, + source_key="derived", + operation="vector.clip", + parent_dataset=parent, + operation_parameters={"area_id": "a"}, + ) + + assert ready is True + assert dataset.status == "ready" + assert dataset.validation_status == "passed" + assert dataset.provenance_status == "complete" + assert dataset.source_registry_id == source.id + assert version.source_snapshot_id == snapshot.id + assert db.flushes >= 1 + assert edges[0]["parent_dataset_id"] == parent.id + assert edges[0]["parent_dataset_version_id"] == parent_version_id + assert edges[0]["child_dataset_version_id"] == version.id + + +def test_govern_vector_quarantines_output_when_parent_is_manual_or_experimental( + monkeypatch, +) -> None: + from app.services.source_registry_service import SourceRegistryService + + db = _GovernedSession() + parent = _governed_parent() + parent.source = "manual" + parent.source_name = "manual" + parent.source_registry.source_key = "manual" + parent.source_registry.classification = "experimental" + parent.source_registry.usage_policy_json = {"ground_truth_allowed": False} + dataset = Dataset( + id=uuid4(), + project_id=parent.project_id, + name="manual-derived.geojson", + dataset_type="vector", + source="operation:clip", + source_name="derived", + checksum_sha256=_CHECKSUM, + imported_at=datetime.now(timezone.utc), + crs="EPSG:4326", + metadata_json={ + "bounds_json": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.0, "max_y": 51.0} + }, + status="validating", + ) + version = DatasetVersion( + id=uuid4(), dataset_id=dataset.id, version=1, checksum_sha256=_CHECKSUM + ) + source = SimpleNamespace(id=uuid4(), license_name="internal derived artifact") + snapshot = SimpleNamespace( + id=uuid4(), source_registry_id=source.id, ingest_status="ingested" + ) + + monkeypatch.setattr( + SourceRegistryService, + "ensure_server_owned_source", + lambda *_args, **_kwargs: source, + ) + monkeypatch.setattr( + SourceRegistryService, "record_snapshot", lambda *_args, **_kwargs: snapshot + ) + monkeypatch.setattr( + DerivedDatasetGovernanceService, + "_latest_parent_version_id", + lambda *_args: None, + ) + monkeypatch.setattr( + SourceRegistryService, "record_lineage_edge", lambda *_args, **_kwargs: None + ) + + ready = DerivedDatasetGovernanceService.govern_vector( + db, + dataset=dataset, + dataset_version=version, + feature_collection={ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [5.0, 51.0]}, + "properties": {}, + } + ], + }, + source_key="derived", + operation="vector.clip", + parent_dataset=parent, + ) + + assert ready is False + assert dataset.status == "quarantined" + assert dataset.quarantine_status == "quarantined" + assert dataset.validation_status == "failed" + assert any( + issue["code"] == "PARENT_DATASET_NOT_ELIGIBLE_FOR_DERIVED_PROCESSING" + for issue in dataset.validation_report_json["issues"] + ) + assert snapshot.ingest_status == "quarantined" + + +def test_vector_selection_uses_map_selection_registry_and_skips_features_when_quarantined( + monkeypatch, tmp_path +) -> None: + source = _governed_parent() + source.area_id = None + source.storage_path = str(tmp_path / "source.geojson") + db = _FakeSession({(Dataset, source.id): source}) + output_path = tmp_path / "selection.geojson" + calls = [] + persisted_features = [] + + monkeypatch.setattr( + VectorFeatureService, + "select_features_by_bbox", + lambda *_args, **_kwargs: { + "selection_bbox": { + "min_x": 4.9, + "min_y": 50.9, + "max_x": 5.2, + "max_y": 51.2, + "crs": "EPSG:4326", + }, + "feature_count": 1, + "limit": 250, + "truncated": False, + "geojson": { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "id": "source-feature", + "geometry": {"type": "Point", "coordinates": [5.0, 51.0]}, + "properties": { + "vector_feature_id": "source-feature", + "dataset_id": str(source.id), + }, + } + ], + }, + }, + ) + + def _persist_dataset_file(**kwargs): + output_path.write_bytes(kwargs["content"]) + return { + "original_filename": kwargs["original_filename"], + "stored_filename": output_path.name, + "content_type": kwargs["content_type"], + "size_bytes": len(kwargs["content"]), + "checksum_sha256": _CHECKSUM, + "storage_path": str(output_path), + } + + monkeypatch.setattr(StorageService, "persist_dataset_file", _persist_dataset_file) + + def _quarantine(db, **kwargs): + calls.append(kwargs) + kwargs["dataset"].status = "quarantined" + return False + + monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_vector", _quarantine) + monkeypatch.setattr( + VectorFeatureService, + "persist_geojson_features", + lambda **kwargs: persisted_features.append(kwargs), + ) + + response = VectorOperationsService.derive_selection_dataset( + db=db, + dataset_id=source.id, + bbox={ + "min_x": 4.9, + "min_y": 50.9, + "max_x": 5.2, + "max_y": 51.2, + "crs": "EPSG:4326", + }, + ) + + assert response.status == "quarantined" + assert calls[0]["source_key"] == "map_selection" + assert calls[0]["parent_dataset"] is source + assert calls[0]["operation"] == "vector.selection" + assert persisted_features == [] + + +def test_vector_buffer_uses_projected_metres_instead_of_wgs84_degrees( + monkeypatch, tmp_path +) -> None: + source = _governed_parent() + source.storage_path = str(tmp_path / "source.geojson") + source.crs = "EPSG:4326" + Path(source.storage_path).write_text( + json.dumps( + { + "type": "FeatureCollection", + "crs": "EPSG:4326", + "features": [ + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [5.0, 51.0]}, + "properties": {}, + }, + ], + } + ), + encoding="utf-8", + ) + # A governed consumption artifact must carry the checksum of these exact + # bytes; vector operations deliberately refuse a stale fixture checksum. + source.checksum_sha256 = sha256(Path(source.storage_path).read_bytes()).hexdigest() + source.source_snapshot.checksum_sha256 = source.checksum_sha256 + db = _FakeSession({(Dataset, source.id): source}) + captured = {} + + def _persist(**kwargs): + captured.update(kwargs) + return uuid4() + + monkeypatch.setattr(VectorOperationsService, "_persist_derived_dataset", _persist) + VectorOperationsService.buffer( + db, source.id, distance_m=100.0, dissolve=False, output_name=None + ) + + coordinates = captured["feature_collection"]["features"][0]["geometry"][ + "coordinates" + ][0] + longitudes = [coordinate[0] for coordinate in coordinates] + latitudes = [coordinate[1] for coordinate in coordinates] + assert max(longitudes) - min(longitudes) < 0.01 + assert max(latitudes) - min(latitudes) < 0.01 + + +def test_raster_operation_uses_derived_registry_before_commit( + monkeypatch, tmp_path +) -> None: + source = _governed_parent() + source.dataset_type = "raster" + source.storage_path = str(tmp_path / "source.tif") + output_path = tmp_path / "derived.tif" + output_path.write_bytes(b"derived-raster") + db = _FakeSession() + calls = [] + + def _govern(db, **kwargs): + calls.append(kwargs) + kwargs["dataset"].status = "quarantined" + return False + + monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_raster", _govern) + + result = RasterOperationsService._persist_derived_dataset( + db, + source_dataset=source, + source_dataset_id=source.id, + operation="ndvi", + output_path=str(output_path), + output_name="derived.tif", + metadata={ + "crs": "EPSG:31370", + "bounds": [100000.0, 100000.0, 100001.0, 100001.0], + "resolution": [1.0, 1.0], + "width": 1, + "height": 1, + "band_count": 1, + "dtype": ["float32"], + "operation_parameters": {"nir_band": 4, "red_band": 3}, + }, + ) + + derived = next(item for item in db.added if isinstance(item, Dataset)) + assert result == derived.id + assert derived.status == "quarantined" + assert derived.source_name == "derived" + assert calls[0]["source_key"] == "derived" + assert calls[0]["parent_dataset"] is source + assert calls[0]["operation"] == "raster.ndvi" + + +def test_demo_fixture_creation_is_governed_and_does_not_persist_features_when_rejected( + monkeypatch, tmp_path +) -> None: + project_id = uuid4() + area_id = uuid4() + db = _FakeSession() + calls = [] + features = [] + + monkeypatch.setattr( + StorageService, + "persist_dataset_file", + lambda **kwargs: { + "storage_path": str(tmp_path / kwargs["original_filename"]), + "original_filename": kwargs["original_filename"], + "stored_filename": kwargs["original_filename"], + "content_type": kwargs["content_type"], + "size_bytes": len(kwargs["content"]), + "checksum_sha256": _CHECKSUM, + }, + ) + + def _quarantine(db, **kwargs): + calls.append(kwargs) + kwargs["dataset"].status = "quarantined" + return False + + monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_vector", _quarantine) + monkeypatch.setattr( + VectorFeatureService, + "persist_geojson_features", + lambda **kwargs: features.append(kwargs), + ) + payload = { + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "geometry": {"type": "Point", "coordinates": [5.0, 51.0]}, + "properties": {}, + } + ], + } + + dataset = DemoWorkflowService._create_dataset( + db, + project_id=project_id, + area_id=area_id, + filename="fixture.geojson", + payload=payload, + raw=json.dumps(payload).encode("utf-8"), + role="source", + source_name="fixture", + reference_layer_name=None, + ) + + assert dataset.status == "quarantined" + assert calls[0]["source_key"] == "fixture" + assert calls[0]["operation"] == "demo.fixture_vector" + assert features == [] diff --git a/backend/tests/test_qa_service.py b/backend/tests/test_qa_service.py index aff0ae0e..eb898295 100644 --- a/backend/tests/test_qa_service.py +++ b/backend/tests/test_qa_service.py @@ -1,11 +1,11 @@ from __future__ import annotations import json +from hashlib import sha256 from pathlib import Path -from types import SimpleNamespace from uuid import uuid4 -from app.models import Area, Dataset +from app.models import Dataset, SourceRegistry, SourceSnapshot from app.services.qa_service import QaService @@ -46,6 +46,47 @@ def _write_features(path: Path, features: list[dict]) -> None: path.write_text(json.dumps({"type": "FeatureCollection", "features": features}), encoding="utf-8") +def _authoritative_reference(dataset: Dataset) -> Dataset: + """Give QA reference fixtures the same durable authority proof as GRB.""" + + source_id = uuid4() + snapshot_id = uuid4() + checksum = sha256(Path(str(dataset.storage_path)).read_bytes()).hexdigest() + source = SourceRegistry( + id=source_id, + source_key="grb", + display_name="GRB test reference", + classification="authoritative", + authority_name="Digitaal Vlaanderen", + authority_scope_json={"zone": "Flanders"}, + usage_policy_json={"ground_truth_allowed": True, "validation_authority": {"building_validation": "primary"}}, + ) + snapshot = SourceSnapshot( + id=snapshot_id, + source_registry_id=source_id, + snapshot_key=f"qa-grb-{dataset.id}", + checksum_sha256=checksum, + ingest_status="ingested", + freshness_status="current", + ) + dataset.source = "grb" + dataset.source_name = "grb" + dataset.dataset_role = "reference" + dataset.status = "ready" + dataset.checksum_sha256 = checksum + dataset.source_registry_id = source_id + dataset.source_snapshot_id = snapshot_id + dataset.data_contract_key = "geointel.vector.geojson" + dataset.data_contract_version = "1.0.0" + dataset.validation_status = "passed" + dataset.provenance_status = "complete" + dataset.lineage_status = "complete" + dataset.quarantine_status = "not_quarantined" + dataset.source_registry = source + dataset.source_snapshot = snapshot + return dataset + + def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None: project_id = uuid4() candidate_id = uuid4() @@ -66,7 +107,7 @@ def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None: crs="EPSG:4326", metadata_json={"crs_assumed": False}, ) - reference = Dataset( + reference = _authoritative_reference(Dataset( id=reference_id, project_id=project_id, name="reference.geojson", @@ -75,7 +116,7 @@ def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None: storage_path=str(reference_path), crs="EPSG:4326", metadata_json={"crs_assumed": False}, - ) + )) result = QaService.compare_candidate_with_reference( db=FakeSession([candidate, reference]), @@ -117,7 +158,7 @@ def test_qa_compare_candidate_with_reference_returns_feature_level_evidence(tmp_ crs="EPSG:4326", metadata_json={"crs_assumed": False}, ) - reference = Dataset( + reference = _authoritative_reference(Dataset( id=reference_id, project_id=project_id, name="reference.geojson", @@ -126,7 +167,7 @@ def test_qa_compare_candidate_with_reference_returns_feature_level_evidence(tmp_ storage_path=str(reference_path), crs="EPSG:4326", metadata_json={"crs_assumed": False}, - ) + )) result = QaService.compare_candidate_with_reference( db=FakeSession([candidate, reference]), diff --git a/backend/tests/test_rc4_national_coverage.py b/backend/tests/test_rc4_national_coverage.py index d3013d9b..faeb733d 100644 --- a/backend/tests/test_rc4_national_coverage.py +++ b/backend/tests/test_rc4_national_coverage.py @@ -49,6 +49,54 @@ def scope_area(name: str, geometry): return SimpleNamespace(name=name, geometry=geometry) +def governed_materialization( + *, + source_name: str, + reference_layer_name: str | None, + source_metadata: dict[str, object], + dataset_id=None, +) -> SimpleNamespace: + """Build a complete authoritative materialization for coverage tests. + + Coverage is a production-facing statement. These fixtures must therefore + carry the same registry, immutable snapshot, checksum and freshness state + that a materialized official dataset needs in production. + """ + + source_registry_id = uuid4() + source_snapshot_id = uuid4() + checksum_sha256 = "a" * 64 + return SimpleNamespace( + id=dataset_id or uuid4(), + status="ready", + source=source_name, + source_name=source_name, + reference_layer_name=reference_layer_name, + source_metadata=dict(source_metadata), + checksum_sha256=checksum_sha256, + source_registry_id=source_registry_id, + source_snapshot_id=source_snapshot_id, + data_contract_key="geointel.vector.geojson", + data_contract_version="1.0.0", + validation_status="passed", + provenance_status="complete", + lineage_status="not_applicable", + quarantine_status="not_quarantined", + source_registry=SimpleNamespace( + source_key=source_name, + classification="authoritative", + authority_scope_json={"scope": "coverage test"}, + usage_policy_json={}, + ), + source_snapshot=SimpleNamespace( + source_registry_id=source_registry_id, + checksum_sha256=checksum_sha256, + freshness_status="current", + ingest_status="ingested", + ), + ) + + def test_coverage_catalog_uses_normalized_contracts_and_does_not_change_provider_registry() -> None: catalog = CoverageRegistryService.catalog() @@ -135,9 +183,8 @@ def test_coverage_resolver_only_reports_operational_for_materialized_ready_datas assert without_materialized.items[0].materialized_dataset_ids == [] dataset_id = uuid4() - materialized = SimpleNamespace( - id=dataset_id, - status="ready", + materialized = governed_materialization( + dataset_id=dataset_id, source_name="ngi_adminvector", reference_layer_name="belgium_regions", source_metadata={"coverage_zones": ["belgium", "flanders", "wallonia", "brussels"]}, @@ -156,9 +203,8 @@ def test_coverage_resolver_only_reports_operational_for_materialized_ready_datas def test_statbel_population_materialization_does_not_masquerade_as_admin_data() -> None: project_id = uuid4() statbel_id = uuid4() - statbel = SimpleNamespace( - id=statbel_id, - status="ready", + statbel = governed_materialization( + dataset_id=statbel_id, source_name="statbel", reference_layer_name="population", source_metadata={"coverage_zones": ["belgium", "flanders", "wallonia", "brussels"]}, @@ -187,9 +233,8 @@ def test_statbel_population_materialization_does_not_masquerade_as_admin_data() def test_bounded_api_materialization_only_covers_its_persisted_bbox() -> None: project_id = uuid4() dataset_id = uuid4() - dataset = SimpleNamespace( - id=dataset_id, - status="ready", + dataset = governed_materialization( + dataset_id=dataset_id, source_name="spw_picc", reference_layer_name="buildings", source_metadata={ @@ -230,8 +275,24 @@ def test_bounded_partition_union_can_be_operational() -> None: left_id = uuid4() right_id = uuid4() datasets = [ - SimpleNamespace(id=left_id, status="ready", source_name="spw_picc", reference_layer_name="buildings", source_metadata={"coverage_zones": ["wallonia"], "bbox_epsg4326": [4.50, 50.50, 4.60, 50.60]}), - SimpleNamespace(id=right_id, status="ready", source_name="spw_picc", reference_layer_name="buildings", source_metadata={"coverage_zones": ["wallonia"], "bbox_epsg4326": [4.60, 50.50, 4.70, 50.60]}), + governed_materialization( + dataset_id=left_id, + source_name="spw_picc", + reference_layer_name="buildings", + source_metadata={ + "coverage_zones": ["wallonia"], + "bbox_epsg4326": [4.50, 50.50, 4.60, 50.60], + }, + ), + governed_materialization( + dataset_id=right_id, + source_name="spw_picc", + reference_layer_name="buildings", + source_metadata={ + "coverage_zones": ["wallonia"], + "bbox_epsg4326": [4.60, 50.50, 4.70, 50.60], + }, + ), ] session = FakeSession( project=SimpleNamespace(id=project_id), @@ -252,9 +313,7 @@ def test_spw_bathymetry_materialization_is_source_specific() -> None: scope_area("Wallonia", box(2.5, 49.5, 6.4, 50.8)), ] selection = CoverageBBox(minx=4.851, miny=50.451, maxx=4.869, maxy=50.469) - spw_picc = SimpleNamespace( - id=uuid4(), - status="ready", + spw_picc = governed_materialization( source_name="spw_picc", reference_layer_name="buildings", source_metadata={ @@ -272,9 +331,8 @@ def test_spw_bathymetry_materialization_is_source_specific() -> None: assert without_bathymetry.items[0].materialized_dataset_ids == [] bathymetry_id = uuid4() - bathymetry = SimpleNamespace( - id=bathymetry_id, - status="ready", + bathymetry = governed_materialization( + dataset_id=bathymetry_id, source_name="spw_bathymetry", reference_layer_name=None, source_metadata={ @@ -309,9 +367,8 @@ def test_vha_bathymetry_profiles_are_operational_only_inside_the_persisted_selec assert without_profiles.items[0].source_names == ["vmm_vha_bathymetry_profiles"] profile_id = uuid4() - profiles = SimpleNamespace( - id=profile_id, - status="ready", + profiles = governed_materialization( + dataset_id=profile_id, source_name="vmm_vha_bathymetry_profiles", reference_layer_name="bathymetry_profile_points", source_metadata={ @@ -360,9 +417,8 @@ def test_flemish_materialization_is_theme_specific() -> None: project_id = uuid4() project = SimpleNamespace(id=project_id) orthophoto_id = uuid4() - orthophoto = SimpleNamespace( - id=orthophoto_id, - status="ready", + orthophoto = governed_materialization( + dataset_id=orthophoto_id, source_name="digitaal_vlaanderen_orthophoto", reference_layer_name="orthophoto", source_metadata={"coverage_zones": ["flanders"]}, @@ -429,5 +485,5 @@ def test_frontend_prefers_materialized_national_workspace_and_resolves_drawn_bbo assert "externalApi.resolveCoverage" in coverage_hook assert "coverage.outside_supported_scope" in map_workspace assert "coverageStatusLabel" in map_workspace - assert "coverageSelectionAvailable" in map_workspace + assert "activeThemeSupportsCurrentSelection" in map_workspace assert "activeThemeAvailable && !regionalPartitionedThemeActive" in map_workspace diff --git a/backend/tests/test_run_state_consistency.py b/backend/tests/test_run_state_consistency.py index 7b14e2e8..aa272f25 100644 --- a/backend/tests/test_run_state_consistency.py +++ b/backend/tests/test_run_state_consistency.py @@ -44,8 +44,10 @@ def _project_and_dataset(): project_id=project_id, name="ortho.tif", dataset_type="raster", - source="user_upload", + source="fixture", + source_name="fixture", storage_path="storage/uploads/ortho.tif", + source_metadata={"fixture": True}, ) db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset}) return db, project_id, dataset_id diff --git a/backend/tests/test_runtime_model_provenance_service.py b/backend/tests/test_runtime_model_provenance_service.py new file mode 100644 index 00000000..2751b5d2 --- /dev/null +++ b/backend/tests/test_runtime_model_provenance_service.py @@ -0,0 +1,325 @@ +from __future__ import annotations + +from hashlib import sha256 +import json +from pathlib import Path +from uuid import uuid4 + +import pytest + +from app.core.errors import AppError +from app.models import DatasetQuarantine, SourceRegistry, SourceSnapshot +from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService + + +class FakeSession: + """Explicit database double for production-runtime provenance tests.""" + + def __init__(self, objects: dict[tuple[type, object], object] | None = None) -> None: + self.objects = objects or {} + + def get(self, model, item_id): + return self.objects.get((model, item_id)) + + +def _write_sidecar( + model_path: Path, + *, + model_id: str = "yolo-configured", + task_type: str = "object_detection", + framework: str = "ultralytics/pytorch", + source_version: str = "test-v1", + source_registry_id: str | None = None, + source_snapshot_id: str | None = None, +) -> Path: + model_sha256 = sha256(model_path.read_bytes()).hexdigest() + payload = { + "schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION, + "data_contract": { + "key": "geointel.model.pytorch", + "version": "1.0.0", + }, + "model": { + "model_id": model_id, + "task_type": task_type, + "sha256": model_sha256, + "model_format": "pytorch", + "framework": framework, + "class_mapping": {"0": "building"}, + "source_version": source_version, + }, + "source": { + "source_registry_id": source_registry_id or str(uuid4()), + "source_snapshot_id": source_snapshot_id or str(uuid4()), + "source_registry_key": "model", + "source_snapshot_checksum_sha256": model_sha256, + }, + "lineage": { + "upstream_asset_ids": ["training-corpus:test-v1"], + "upstream_checksums_sha256": ["a" * 64], + "transformations": [ + { + "name": "pytorch-training", + "version": "1.0.0", + "checksum_sha256": "b" * 64, + } + ], + }, + "metadata": { + "training_manifest_sha256": "c" * 64, + }, + "imported_at": "2026-08-01T10:00:00+00:00", + } + payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload) + sidecar_path = RuntimeModelProvenanceService.manifest_path_for_model(model_path) + sidecar_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8") + return sidecar_path + + +def _governed_model_database( + *, + source_registry_id, + source_snapshot_id, + model_checksum: str, + source_version: str = "test-v1", +) -> tuple[FakeSession, SourceRegistry, SourceSnapshot]: + registry = SourceRegistry( + id=source_registry_id, + source_key="model", + display_name="Governed test model artifacts", + classification="experimental", + authority_name="GeoIntel test fixture", + freshness_status="current", + ingest_status="configured", + ) + snapshot = SourceSnapshot( + id=source_snapshot_id, + source_registry_id=source_registry_id, + snapshot_key=f"model-{source_version}", + source_version=source_version, + checksum_sha256=model_checksum, + freshness_status="current", + ingest_status="ingested", + ) + return ( + FakeSession( + { + (SourceRegistry, source_registry_id): registry, + (SourceSnapshot, source_snapshot_id): snapshot, + } + ), + registry, + snapshot, + ) + + +def test_runtime_model_provenance_accepts_byte_bound_pytorch_sidecar_for_structural_preflight(tmp_path: Path) -> None: + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"trusted local model bytes") + sidecar_path = _write_sidecar(model_path, source_version="v1") + + evidence = RuntimeModelProvenanceService.validate_for_runtime( + model_path=model_path, + model_id="yolo-configured", + task_type="object_detection", + expected_model_version="v1", + allowed_frameworks=("ultralytics/pytorch",), + ) + + assert evidence.model_sha256 == sha256(model_path.read_bytes()).hexdigest() + assert evidence.manifest_path == str(sidecar_path.resolve()) + assert evidence.data_contract_key == "geointel.model.pytorch" + assert evidence.data_contract_version == "1.0.0" + assert len(evidence.validation_report_sha256) == 64 + + +def test_production_runtime_requires_db_bound_model_source_snapshot(tmp_path: Path) -> None: + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"governed local model bytes") + source_registry_id = uuid4() + source_snapshot_id = uuid4() + _write_sidecar( + model_path, + source_version="v1", + source_registry_id=str(source_registry_id), + source_snapshot_id=str(source_snapshot_id), + ) + db, _, _ = _governed_model_database( + source_registry_id=source_registry_id, + source_snapshot_id=source_snapshot_id, + model_checksum=sha256(model_path.read_bytes()).hexdigest(), + source_version="v1", + ) + + evidence = RuntimeModelProvenanceService.validate_for_production_runtime( + db=db, + model_path=model_path, + model_id="yolo-configured", + task_type="object_detection", + expected_model_version="v1", + allowed_frameworks=("ultralytics/pytorch",), + ) + + assert evidence.source_registry_id == str(source_registry_id) + assert evidence.source_snapshot_id == str(source_snapshot_id) + assert evidence.source_snapshot_checksum_sha256 == evidence.model_sha256 + + +def test_production_runtime_rejects_missing_database_source_binding(tmp_path: Path) -> None: + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"unbound model bytes") + _write_sidecar(model_path) + + with pytest.raises(AppError) as exc_info: + RuntimeModelProvenanceService.validate_for_production_runtime( + db=FakeSession(), + model_path=model_path, + model_id="yolo-configured", + task_type="object_detection", + ) + + assert exc_info.value.code == "MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND" + + +def test_production_runtime_requires_a_database_session(tmp_path: Path) -> None: + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"model bytes") + _write_sidecar(model_path) + + with pytest.raises(AppError) as exc_info: + RuntimeModelProvenanceService.validate_for_production_runtime( + db=None, + model_path=model_path, + model_id="yolo-configured", + task_type="object_detection", + ) + + assert exc_info.value.code == "MODEL_PROVENANCE_DATABASE_REQUIRED" + + +@pytest.mark.parametrize( + ("mutation", "expected_code"), + ( + ("registry_unsafe", "MODEL_PROVENANCE_SOURCE_REGISTRY_UNSAFE"), + ("snapshot_registry_mismatch", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_REGISTRY_MISMATCH"), + ("snapshot_missing", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_NOT_FOUND"), + ("snapshot_quarantined", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_UNSAFE"), + ("snapshot_checksum_mismatch", "MODEL_PROVENANCE_DATABASE_SNAPSHOT_CHECKSUM_MISMATCH"), + ("active_quarantine", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_QUARANTINED"), + ), +) +def test_production_runtime_rejects_unsafe_or_inconsistent_database_snapshot( + tmp_path: Path, + mutation: str, + expected_code: str, +) -> None: + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"governed model bytes") + source_registry_id = uuid4() + source_snapshot_id = uuid4() + _write_sidecar( + model_path, + source_registry_id=str(source_registry_id), + source_snapshot_id=str(source_snapshot_id), + ) + db, registry, snapshot = _governed_model_database( + source_registry_id=source_registry_id, + source_snapshot_id=source_snapshot_id, + model_checksum=sha256(model_path.read_bytes()).hexdigest(), + ) + if mutation == "registry_unsafe": + registry.ingest_status = "quarantined" + elif mutation == "snapshot_registry_mismatch": + snapshot.source_registry_id = uuid4() + elif mutation == "snapshot_missing": + db.objects.pop((SourceSnapshot, source_snapshot_id)) + elif mutation == "snapshot_quarantined": + snapshot.ingest_status = "quarantined" + elif mutation == "snapshot_checksum_mismatch": + snapshot.checksum_sha256 = "f" * 64 + elif mutation == "active_quarantine": + snapshot.quarantines = [ + DatasetQuarantine( + source_snapshot_id=source_snapshot_id, + stage="test", + reason_code="test_active_quarantine", + status="quarantined", + ) + ] + + with pytest.raises(AppError) as exc_info: + RuntimeModelProvenanceService.validate_for_production_runtime( + db=db, + model_path=model_path, + model_id="yolo-configured", + task_type="object_detection", + ) + + assert exc_info.value.code == expected_code + + +def test_runtime_model_provenance_rejects_missing_sidecar(tmp_path: Path) -> None: + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"unmanifested local model bytes") + + with pytest.raises(AppError) as exc_info: + RuntimeModelProvenanceService.validate_for_runtime( + model_path=model_path, + model_id="yolo-configured", + task_type="object_detection", + ) + + assert exc_info.value.code == "MODEL_PROVENANCE_MANIFEST_MISSING" + + +def test_runtime_model_provenance_rejects_model_bytes_tampered_after_manifest(tmp_path: Path) -> None: + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"original local model bytes") + _write_sidecar(model_path) + model_path.write_bytes(b"tampered local model bytes") + + with pytest.raises(AppError) as exc_info: + RuntimeModelProvenanceService.validate_for_runtime( + model_path=model_path, + model_id="yolo-configured", + task_type="object_detection", + ) + + assert exc_info.value.code == "MODEL_PROVENANCE_MODEL_CHECKSUM_MISMATCH" + + +def test_runtime_model_provenance_rejects_tampered_manifest_contents(tmp_path: Path) -> None: + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"original local model bytes") + sidecar_path = _write_sidecar(model_path) + payload = json.loads(sidecar_path.read_text(encoding="utf-8")) + payload["model"]["class_mapping"]["1"] = "road" + sidecar_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8") + + with pytest.raises(AppError) as exc_info: + RuntimeModelProvenanceService.validate_for_runtime( + model_path=model_path, + model_id="yolo-configured", + task_type="object_detection", + ) + + assert exc_info.value.code == "MODEL_PROVENANCE_MANIFEST_CHECKSUM_MISMATCH" + + +def test_runtime_model_provenance_rejects_other_contract_even_if_structurally_valid(tmp_path: Path) -> None: + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"local model bytes") + sidecar_path = _write_sidecar(model_path) + payload = json.loads(sidecar_path.read_text(encoding="utf-8")) + payload["data_contract"]["key"] = "geointel.vector.geojson" + payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload) + sidecar_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8") + + with pytest.raises(AppError) as exc_info: + RuntimeModelProvenanceService.validate_for_runtime( + model_path=model_path, + model_id="yolo-configured", + task_type="object_detection", + ) + + assert exc_info.value.code == "MODEL_PROVENANCE_MANIFEST_INVALID" diff --git a/backend/tests/test_segmentation_configured_models.py b/backend/tests/test_segmentation_configured_models.py index 0073b865..bc5c09ad 100644 --- a/backend/tests/test_segmentation_configured_models.py +++ b/backend/tests/test_segmentation_configured_models.py @@ -1,5 +1,6 @@ from __future__ import annotations +from hashlib import sha256 import json from pathlib import Path from uuid import uuid4 @@ -8,9 +9,10 @@ import pytest from geoalchemy2.shape import to_shape from app.core.config import Settings -from app.models import Dataset, Project, Segmentation +from app.models import Dataset, Project, Segmentation, SourceRegistry, SourceSnapshot from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon from app.services.model_registry_service import ModelRegistryService +from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService from app.services.segmentation_service import SegmentationService ROOT = Path(__file__).resolve().parents[2] @@ -80,18 +82,58 @@ class MissingDependencySegAdapter(AvailableSegAdapter): return False +class NeverLoadSegAdapter(AvailableSegAdapter): + load_calls = 0 + + def load_model(self, model_path: Path): + type(self).load_calls += 1 + raise AssertionError("unmanifested weights must not reach adapter.load_model") + + def _project_and_dataset(dataset_type: str = "raster"): project_id = uuid4() dataset_id = uuid4() + source_id = uuid4() + snapshot_id = uuid4() + checksum = "a" * 64 project = Project(id=project_id, name="Mol") + source = SourceRegistry( + id=source_id, + source_key="digitaal_vlaanderen_orthophoto", + display_name="Governed orthophoto test source", + classification="contextual", + authority_name="Digitaal Vlaanderen", + authority_scope_json={"zone": "Flanders", "role": "imagery"}, + ) + snapshot = SourceSnapshot( + id=snapshot_id, + source_registry_id=source_id, + snapshot_key="configured-segmentation-orthophoto", + checksum_sha256=checksum, + ingest_status="ingested", + freshness_status="current", + ) dataset = Dataset( id=dataset_id, project_id=project_id, name="ortho.tif", dataset_type=dataset_type, - source="user_upload", + source="digitaal_vlaanderen_orthophoto", + source_name="digitaal_vlaanderen_orthophoto", storage_path="storage/uploads/ortho.tif", + checksum_sha256=checksum, + source_registry_id=source_id, + source_snapshot_id=snapshot_id, + data_contract_key="geointel.raster.geotiff", + data_contract_version="1.0.0", + validation_status="passed", + provenance_status="complete", + lineage_status="not_applicable", + quarantine_status="not_quarantined", + status="ready", ) + dataset.source_registry = source + dataset.source_snapshot = snapshot db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset}) return db, project_id, dataset_id @@ -108,6 +150,102 @@ def _settings(tmp_path: Path, **overrides) -> Settings: return Settings(**values) +def _write_model_sidecar( + model_path: Path, + *, + model_id: str, + framework: str, + source_version: str | None, + db: FakeSession | None = None, +) -> None: + """Create explicit local test evidence; no production code creates sidecars.""" + + model_sha256 = sha256(model_path.read_bytes()).hexdigest() + source_registry_id = uuid4() + source_snapshot_id = uuid4() + resolved_source_version = source_version or "test-v1" + if db is not None: + source_registry = SourceRegistry( + id=source_registry_id, + source_key="model", + display_name="Governed test model artifact", + classification="experimental", + authority_name="GeoIntel test fixture", + freshness_status="current", + ingest_status="configured", + ) + source_snapshot = SourceSnapshot( + id=source_snapshot_id, + source_registry_id=source_registry_id, + snapshot_key=f"model-{model_id}-{resolved_source_version}", + source_version=resolved_source_version, + checksum_sha256=model_sha256, + freshness_status="current", + ingest_status="ingested", + ) + db.objects[(SourceRegistry, source_registry_id)] = source_registry + db.objects[(SourceSnapshot, source_snapshot_id)] = source_snapshot + payload = { + "schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION, + "data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"}, + "model": { + "model_id": model_id, + "task_type": "segmentation", + "sha256": model_sha256, + "model_format": "pytorch", + "framework": framework, + "class_mapping": {"0": "segment"}, + "source_version": resolved_source_version, + }, + "source": { + "source_registry_id": str(source_registry_id), + "source_snapshot_id": str(source_snapshot_id), + "source_registry_key": "model", + "source_snapshot_checksum_sha256": model_sha256, + }, + "lineage": { + "upstream_asset_ids": ["test-training-corpus"], + "upstream_checksums_sha256": ["a" * 64], + "transformations": [ + {"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64} + ], + }, + "metadata": {"training_manifest_sha256": "c" * 64}, + "imported_at": "2026-08-01T10:00:00+00:00", + } + payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload) + RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text( + json.dumps(payload, sort_keys=True), + encoding="utf-8", + ) + + +def _write_configured_model_sidecars( + tmp_path: Path, + settings: Settings, + *, + include_yolo: bool = True, + include_sam: bool = True, + db: FakeSession | None = None, +) -> None: + if include_yolo: + _write_model_sidecar( + tmp_path / "seg.pt", + model_id=settings.yolo_seg_model_id, + framework="ultralytics/pytorch", + source_version=settings.yolo_seg_model_version, + db=db, + ) + if include_sam: + _write_model_sidecar( + tmp_path / "sam.pt", + model_id=settings.sam_model_id, + framework="ultralytics/sam", + source_version=settings.sam_model_version, + db=db, + ) + + def _manifest(tmp_path: Path, tile_count: int = 1) -> Path: tiles = [] for index in range(tile_count): @@ -172,10 +310,31 @@ def test_segmentation_models_report_dependency_unavailable(tmp_path: Path) -> No assert models["sam-configured"].status == "dependency_unavailable" +def test_segmentation_models_require_runtime_provenance_sidecars(tmp_path: Path) -> None: + (tmp_path / "seg.pt").write_bytes(b"unmanifested yolo segmentation weights") + (tmp_path / "sam.pt").write_bytes(b"unmanifested sam weights") + settings = _settings(tmp_path) + + models = { + model.model_id: model + for model in ModelRegistryService.list_segmentation_model_capabilities( + settings=settings, + yolo_seg_adapter_class=AvailableSegAdapter, + sam_adapter_class=ClassAgnosticSamAdapter, + ) + } + + assert models["yolo-seg-configured"].configured is False + assert models["yolo-seg-configured"].status == "contract_incomplete" + assert models["sam-configured"].configured is False + assert models["sam-configured"].status == "contract_incomplete" + + def test_segmentation_models_report_configured_with_local_weights(tmp_path: Path) -> None: (tmp_path / "seg.pt").write_bytes(b"weights") (tmp_path / "sam.pt").write_bytes(b"weights") settings = _settings(tmp_path) + _write_configured_model_sidecars(tmp_path, settings) models = { model.model_id: model @@ -204,6 +363,7 @@ def test_configured_segmentation_requires_tile_manifest(tmp_path: Path) -> None: (tmp_path / "seg.pt").write_bytes(b"weights") db, project_id, dataset_id = _project_and_dataset() settings = _settings(tmp_path) + _write_configured_model_sidecars(tmp_path, settings, include_sam=False, db=db) with pytest.raises(Exception) as exc_info: SegmentationService.run_segmentation( @@ -220,10 +380,60 @@ def test_configured_segmentation_requires_tile_manifest(tmp_path: Path) -> None: assert getattr(exc_info.value, "code", None) == "SEGMENTATION_TILE_MANIFEST_REQUIRED" +def test_configured_segmentation_fails_before_adapter_load_without_sidecar(tmp_path: Path) -> None: + (tmp_path / "seg.pt").write_bytes(b"unmanifested weights") + db, project_id, dataset_id = _project_and_dataset() + settings = _settings(tmp_path) + NeverLoadSegAdapter.load_calls = 0 + + response = SegmentationService.run_segmentation( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="yolo-seg-configured", + confidence_threshold=0.5, + tile_manifest_path=str(_manifest(tmp_path)), + settings=settings, + yolo_seg_adapter_class=NeverLoadSegAdapter, + sam_adapter_class=ClassAgnosticSamAdapter, + ) + + assert response.status == "failed" + assert response.error_code == "SEGMENTATION_MODEL_UNAVAILABLE" + assert NeverLoadSegAdapter.load_calls == 0 + + +def test_configured_segmentation_rejects_unbound_model_snapshot_before_adapter_load(tmp_path: Path) -> None: + (tmp_path / "seg.pt").write_bytes(b"structurally valid but unbound weights") + db, project_id, dataset_id = _project_and_dataset() + settings = _settings(tmp_path) + # The sidecar passes catalog validation but its source registry/snapshot + # was never registered in this production-session fixture. + _write_configured_model_sidecars(tmp_path, settings, include_sam=False) + NeverLoadSegAdapter.load_calls = 0 + + response = SegmentationService.run_segmentation( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="yolo-seg-configured", + confidence_threshold=0.5, + tile_manifest_path=str(_manifest(tmp_path)), + settings=settings, + yolo_seg_adapter_class=NeverLoadSegAdapter, + sam_adapter_class=ClassAgnosticSamAdapter, + ) + + assert response.status == "failed" + assert response.error_code == "MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND" + assert NeverLoadSegAdapter.load_calls == 0 + + def test_configured_yolo_seg_run_persists_georeferenced_masks(tmp_path: Path) -> None: (tmp_path / "seg.pt").write_bytes(b"weights") db, project_id, dataset_id = _project_and_dataset() settings = _settings(tmp_path) + _write_configured_model_sidecars(tmp_path, settings, include_sam=False, db=db) manifest_path = _manifest(tmp_path) response = SegmentationService.run_segmentation( @@ -255,12 +465,14 @@ def test_configured_yolo_seg_run_persists_georeferenced_masks(tmp_path: Path) -> assert segmentation.area_m2 is not None and segmentation.area_m2 > 0 assert segmentation.provenance_json["inference"] == "local" assert segmentation.provenance_json["model_id"] == "yolo-seg-configured" + assert segmentation.provenance_json["runtime_model_provenance"]["data_contract_key"] == "geointel.model.pytorch" def test_configured_sam_run_is_class_agnostic(tmp_path: Path) -> None: (tmp_path / "sam.pt").write_bytes(b"weights") db, project_id, dataset_id = _project_and_dataset() settings = _settings(tmp_path) + _write_configured_model_sidecars(tmp_path, settings, include_yolo=False, db=db) manifest_path = _manifest(tmp_path) response = SegmentationService.run_segmentation( diff --git a/backend/tests/test_sprint107_map_selection_export.py b/backend/tests/test_sprint107_map_selection_export.py index 9bfb15fd..3f32c96a 100644 --- a/backend/tests/test_sprint107_map_selection_export.py +++ b/backend/tests/test_sprint107_map_selection_export.py @@ -11,7 +11,7 @@ from shapely.geometry import box from app.core.errors import AppError from app.main import app -from app.models import Area, Dataset, Export +from app.models import Area, Dataset, Export, SourceRegistry, SourceSnapshot from app.schemas.export import ExportCreateResponse from app.services.export_service import ExportService from app.services.storage_service import StorageService @@ -45,18 +45,56 @@ class FakeSession: return row +def _govern_fixture_dataset(dataset: Dataset) -> Dataset: + source_id = uuid4() + snapshot_id = uuid4() + checksum = "a" * 64 + source = SourceRegistry( + id=source_id, + source_key="grb", + display_name="GRB map export test source", + classification="authoritative", + authority_name="Digitaal Vlaanderen", + authority_scope_json={"zone": "Flanders"}, + usage_policy_json={"ground_truth_allowed": True}, + ) + snapshot = SourceSnapshot( + id=snapshot_id, + source_registry_id=source_id, + snapshot_key=f"map-export-grb-{dataset.id}", + checksum_sha256=checksum, + ingest_status="ingested", + freshness_status="current", + ) + dataset.source = "grb" + dataset.source_name = "grb" + dataset.checksum_sha256 = checksum + dataset.source_registry_id = source_id + dataset.source_snapshot_id = snapshot_id + dataset.data_contract_key = "geointel.vector.geojson" + dataset.data_contract_version = "1.0.0" + dataset.validation_status = "passed" + dataset.provenance_status = "complete" + dataset.lineage_status = "not_applicable" + dataset.quarantine_status = "not_quarantined" + dataset.status = "ready" + dataset.source_registry = source + dataset.source_snapshot = snapshot + return dataset + + def test_vector_selection_geojson_export_persists_handoff_artifact(tmp_path, monkeypatch) -> None: project_id = uuid4() dataset_id = uuid4() export_path = tmp_path / "exports" / "selection.geojson" - dataset = Dataset( + dataset = _govern_fixture_dataset(Dataset( id=dataset_id, project_id=project_id, name="candidate.geojson", dataset_type="vector", source="fixture", status="ready", - ) + )) db = FakeSession({(Dataset, dataset_id): dataset}) selection_bbox = {"min_x": 4.9, "min_y": 50.9, "max_x": 5.2, "max_y": 51.2, "crs": "EPSG:4326"} selection_payload = { @@ -135,14 +173,14 @@ def test_vector_selection_export_uses_exact_area_scope_when_requested(tmp_path, project_id = uuid4() dataset_id = uuid4() area_id = uuid4() - dataset = Dataset( + dataset = _govern_fixture_dataset(Dataset( id=dataset_id, project_id=project_id, name="regional-buildings.geojson", dataset_type="vector", source="fixture", status="ready", - ) + )) area_shape = box(5.0, 51.1, 5.2, 51.3) area_geometry = from_shape(area_shape, srid=4326) area = SimpleNamespace(id=area_id, project_id=project_id, name="Gemeente Mol", geometry=area_geometry) @@ -184,7 +222,7 @@ def test_area_constrained_bbox_uses_intersection_and_disables_full_area_fast_pat project_id = uuid4() dataset_id = uuid4() area_id = uuid4() - dataset = Dataset( + dataset = _govern_fixture_dataset(Dataset( id=dataset_id, project_id=project_id, area_id=area_id, @@ -193,7 +231,7 @@ def test_area_constrained_bbox_uses_intersection_and_disables_full_area_fast_pat source="fixture", source_metadata={"geometry_clipped_to_area": True}, status="ready", - ) + )) area_shape = box(5.0, 51.0, 5.2, 51.2) area = SimpleNamespace( id=area_id, diff --git a/backend/tests/test_sprint108_map_selection_derived_dataset.py b/backend/tests/test_sprint108_map_selection_derived_dataset.py index 21f4ab78..a4a54c59 100644 --- a/backend/tests/test_sprint108_map_selection_derived_dataset.py +++ b/backend/tests/test_sprint108_map_selection_derived_dataset.py @@ -7,7 +7,7 @@ from uuid import uuid4 from fastapi.testclient import TestClient from app.main import app -from app.models import Dataset, VectorFeature +from app.models import Dataset from app.schemas.dataset import DatasetCreateResponse from app.services.storage_service import StorageService from app.services.vector_feature_service import VectorFeatureService @@ -124,7 +124,7 @@ def test_vector_selection_derive_persists_queryable_derived_dataset(tmp_path, mo assert response.provenance_metadata["source_dataset_id"] == str(dataset_id) assert response.provenance_metadata["source_table"] == "vector_features" assert persisted_features[0]["dataset_id"] == derived.id - assert persisted_features[0]["commit"] is True + assert persisted_features[0]["commit"] is False derived_payload = json.loads(output_path.read_text(encoding="utf-8")) props = derived_payload["features"][0]["properties"] assert props["source_vector_feature_id"] == "source-row-1" diff --git a/backend/tests/test_sprint129_operator_yolo_training_dataset.py b/backend/tests/test_sprint129_operator_yolo_training_dataset.py index f29bc607..9d8f5bc0 100644 --- a/backend/tests/test_sprint129_operator_yolo_training_dataset.py +++ b/backend/tests/test_sprint129_operator_yolo_training_dataset.py @@ -75,5 +75,7 @@ def test_operator_yolo_train_smoke_script_contract() -> None: assert '"dataset_summary_sha256"' in script assert '"base_model_sha256"' in script assert '"trained_model_sha256"' in script + assert "training_release_manifest.py" in script + assert "verify" in script assert "download" not in script.lower() assert "fixture_mode" not in script diff --git a/backend/tests/test_sprint13_yolo_preflight.py b/backend/tests/test_sprint13_yolo_preflight.py index 53e3d4fd..1ec51342 100644 --- a/backend/tests/test_sprint13_yolo_preflight.py +++ b/backend/tests/test_sprint13_yolo_preflight.py @@ -1,5 +1,6 @@ from __future__ import annotations +from hashlib import sha256 import json import subprocess import sys @@ -9,6 +10,7 @@ from fastapi.testclient import TestClient from app.core.config import Settings from app.main import app +from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService from app.services.yolo_preflight_service import YoloPreflightService @@ -57,6 +59,43 @@ def _manifest(tmp_path: Path, tile_count: int = 1) -> Path: return manifest_path +def _write_model_sidecar(model_path: Path, settings: Settings) -> None: + model_sha256 = sha256(model_path.read_bytes()).hexdigest() + payload = { + "schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION, + "data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"}, + "model": { + "model_id": settings.yolo_model_id, + "task_type": "object_detection", + "sha256": model_sha256, + "model_format": "pytorch", + "framework": "ultralytics/pytorch", + "class_mapping": {"0": "building"}, + "source_version": settings.yolo_model_version or "test-v1", + }, + "source": { + "source_registry_id": "11111111-1111-4111-8111-111111111111", + "source_snapshot_id": "22222222-2222-4222-8222-222222222222", + "source_registry_key": "model", + "source_snapshot_checksum_sha256": model_sha256, + }, + "lineage": { + "upstream_asset_ids": ["test-training-corpus"], + "upstream_checksums_sha256": ["a" * 64], + "transformations": [ + {"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64} + ], + }, + "metadata": {"training_manifest_sha256": "c" * 64}, + "imported_at": "2026-08-01T10:00:00+00:00", + } + payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload) + RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text( + json.dumps(payload, sort_keys=True), + encoding="utf-8", + ) + + def test_yolo_preflight_reports_disabled_without_loading_model(tmp_path: Path, monkeypatch) -> None: monkeypatch.setenv("YOLO_CONFIG_DIR", str(tmp_path / "ultralytics")) @@ -99,9 +138,11 @@ def test_yolo_preflight_validates_model_and_manifest_without_importing_yolo(tmp_ model_path = tmp_path / "model.pt" model_path.write_bytes(b"weights") manifest_path = _manifest(tmp_path, tile_count=2) + settings = Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4) + _write_model_sidecar(model_path, settings) result = YoloPreflightService.run( - settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4), + settings=settings, tile_manifest_path=str(manifest_path), yolo_adapter_class=AvailableAdapter, ) @@ -109,6 +150,7 @@ def test_yolo_preflight_validates_model_and_manifest_without_importing_yolo(tmp_ assert result["status"] == "ready" assert result["checks"]["dependencies_available"] is True assert result["checks"]["model_file_exists"] is True + assert result["checks"]["model_provenance_valid"] is True assert result["checks"]["manifest_valid"] is True assert result["tile_count"] == 2 assert result["will_download_models"] is False @@ -120,9 +162,11 @@ def test_yolo_preflight_marks_assumed_dependencies_in_runtime_details(tmp_path: model_path = tmp_path / "model.pt" model_path.write_bytes(b"weights") manifest_path = _manifest(tmp_path, tile_count=1) + settings = Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4) + _write_model_sidecar(model_path, settings) result = YoloPreflightService.run( - settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4), + settings=settings, tile_manifest_path=str(manifest_path), yolo_adapter_class=MissingDependencyAdapter, assume_dependencies=True, @@ -138,9 +182,11 @@ def test_yolo_preflight_can_explicitly_smoke_load_local_model(tmp_path: Path) -> model_path = tmp_path / "model.pt" model_path.write_bytes(b"weights") manifest_path = _manifest(tmp_path) + settings = Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4) + _write_model_sidecar(model_path, settings) result = YoloPreflightService.run( - settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4), + settings=settings, tile_manifest_path=str(manifest_path), yolo_adapter_class=AvailableAdapter, check_model_load=True, @@ -156,9 +202,11 @@ def test_yolo_preflight_can_explicitly_smoke_load_local_model(tmp_path: Path) -> def test_yolo_preflight_reports_explicit_model_load_failure(tmp_path: Path) -> None: model_path = tmp_path / "model.pt" model_path.write_bytes(b"weights") + settings = Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4) + _write_model_sidecar(model_path, settings) result = YoloPreflightService.run( - settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4), + settings=settings, tile_manifest_path=str(_manifest(tmp_path)), yolo_adapter_class=FailingLoadAdapter, check_model_load=True, @@ -173,6 +221,7 @@ def test_yolo_preflight_script_outputs_json(tmp_path: Path) -> None: model_path = tmp_path / "model.pt" model_path.write_bytes(b"weights") manifest_path = _manifest(tmp_path) + _write_model_sidecar(model_path, Settings(yolo_enabled=True, yolo_model_path=str(model_path))) result = subprocess.run( [ @@ -201,6 +250,7 @@ def test_yolo_preflight_script_uses_environment_configuration(tmp_path: Path, mo model_path = tmp_path / "model.pt" model_path.write_bytes(b"weights") manifest_path = _manifest(tmp_path) + _write_model_sidecar(model_path, Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4)) monkeypatch.setenv("YOLO_ENABLED", "true") monkeypatch.setenv("YOLO_MODEL_PATH", str(model_path)) monkeypatch.setenv("YOLO_MAX_TILES", "4") @@ -227,6 +277,21 @@ def test_yolo_preflight_script_uses_environment_configuration(tmp_path: Path, mo assert payload["max_tiles"] == 4 +def test_yolo_preflight_refuses_unmanifested_local_weights(tmp_path: Path) -> None: + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"unmanifested weights") + + result = YoloPreflightService.run( + settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path)), + tile_manifest_path=str(_manifest(tmp_path)), + yolo_adapter_class=AvailableAdapter, + ) + + assert result["status"] == "contract_incomplete" + assert result["checks"]["model_provenance_valid"] is False + assert result["error_code"] == "MODEL_PROVENANCE_MANIFEST_MISSING" + + def test_yolo_preflight_script_rejects_assumed_dependencies_for_model_load(tmp_path: Path) -> None: result = subprocess.run( [ diff --git a/backend/tests/test_sprint17_export_foundation.py b/backend/tests/test_sprint17_export_foundation.py index b1cc7c84..ff6b31c1 100644 --- a/backend/tests/test_sprint17_export_foundation.py +++ b/backend/tests/test_sprint17_export_foundation.py @@ -8,7 +8,7 @@ from fastapi.testclient import TestClient from app.core.errors import AppError from app.main import app -from app.models import Area, Dataset, Export, Project, QualityCheck +from app.models import Area, Dataset, Export, Project, QualityCheck, SourceRegistry, SourceSnapshot from app.schemas.export import ExportCreateResponse from app.services.export_service import ExportService from app.services.storage_service import StorageService @@ -66,13 +66,57 @@ class FakeSession: return row +def _govern_fixture_dataset(dataset: Dataset) -> Dataset: + """Give an export fixture a governed authoritative source identity. + + Export is a production boundary: test data must model a source that could + cross it, rather than using the deliberately QA-only ``fixture`` source. + """ + + source_id = uuid4() + snapshot_id = uuid4() + checksum = "a" * 64 + source = SourceRegistry( + id=source_id, + source_key="grb", + display_name="GRB export test source", + classification="authoritative", + authority_name="Digitaal Vlaanderen", + authority_scope_json={"zone": "Flanders"}, + usage_policy_json={"ground_truth_allowed": True}, + ) + snapshot = SourceSnapshot( + id=snapshot_id, + source_registry_id=source_id, + snapshot_key=f"export-grb-{dataset.id}", + checksum_sha256=checksum, + ingest_status="ingested", + freshness_status="current", + ) + dataset.source = "grb" + dataset.source_name = "grb" + dataset.checksum_sha256 = checksum + dataset.source_registry_id = source_id + dataset.source_snapshot_id = snapshot_id + dataset.data_contract_key = "geointel.vector.geojson" + dataset.data_contract_version = "1.0.0" + dataset.validation_status = "passed" + dataset.provenance_status = "complete" + dataset.lineage_status = "not_applicable" + dataset.quarantine_status = "not_quarantined" + dataset.status = "ready" + dataset.source_registry = source + dataset.source_snapshot = snapshot + return dataset + + def test_dataset_geojson_export_persists_export_and_writes_artifact(tmp_path, monkeypatch) -> None: project_id = uuid4() dataset_id = uuid4() dataset_path = tmp_path / "input.geojson" dataset_path.write_text(json.dumps({"type": "FeatureCollection", "features": []}), encoding="utf-8") export_path = tmp_path / "exports" / "buildings.geojson" - dataset = Dataset( + dataset = _govern_fixture_dataset(Dataset( id=dataset_id, project_id=project_id, name="buildings.geojson", @@ -80,7 +124,7 @@ def test_dataset_geojson_export_persists_export_and_writes_artifact(tmp_path, mo source="fixture", storage_path=str(dataset_path), status="ready", - ) + )) db = FakeSession({(Dataset, dataset_id): dataset}) monkeypatch.setattr(StorageService, "dataset_export_path", lambda *_args: str(export_path)) diff --git a/backend/tests/test_sprint196_map_orthophoto_analysis.py b/backend/tests/test_sprint196_map_orthophoto_analysis.py index d4bec672..0ffafa7a 100644 --- a/backend/tests/test_sprint196_map_orthophoto_analysis.py +++ b/backend/tests/test_sprint196_map_orthophoto_analysis.py @@ -18,7 +18,7 @@ from app.core.config import Settings from app.core.errors import AppError from app.db.session import get_db from app.main import app -from app.models import Area, Dataset, DatasetVersion, Job, Project +from app.models import Area, Dataset, DatasetVersion, Job, Project, SourceRegistry, SourceSnapshot from app.schemas.orthophoto import OrthophotoAcquireRequest from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService @@ -41,6 +41,15 @@ class FakeSession: def add(self, row): self.added.append(row) + def flush(self): + # The governed importer persists source identities and immutable + # snapshots before the Dataset. Mirror the database-generated UUIDs + # so this harness exercises that Phase 2 path rather than the legacy + # no-registry fallback. + for row in self.added: + if getattr(row, "id", None) is None: + row.id = uuid4() + def commit(self): return None @@ -50,13 +59,23 @@ class FakeSession: def refresh(self, row): return row - def query(self, _model): - return FakeQuery(self.query_result) + def query(self, model): + rows = [ + row + for (row_model, _row_id), row in self.rows.items() + if row_model is model and isinstance(row, model) + ] + rows.extend(row for row in self.added if isinstance(row, model)) + if isinstance(self.query_result, model): + rows.append(self.query_result) + elif isinstance(self.query_result, list): + rows.extend(row for row in self.query_result if isinstance(row, model)) + return FakeQuery(rows) class FakeQuery: - def __init__(self, result): - self.result = result + def __init__(self, results): + self.results = list(results) def filter(self, *_args): return self @@ -65,7 +84,10 @@ class FakeQuery: return self def first(self): - return self.result + return self.results[0] if self.results else None + + def one_or_none(self): + return self.first() class FakeImageResponse: @@ -222,12 +244,23 @@ def test_regional_orthophoto_products_bind_provider_and_governed_scope( ) dataset = next(row for row in db.added if isinstance(row, Dataset)) + source = next(row for row in db.added if isinstance(row, SourceRegistry)) + snapshot = next(row for row in db.added if isinstance(row, SourceSnapshot)) assert result["provider"] == provider assert result["layer"] == layer assert dataset.source_name == provider assert dataset.source_metadata["coverage_zone"] == coverage_zone assert dataset.source_metadata["license_note"] assert dataset.provenance_metadata["request_url"].startswith(prepared["product"].wms_url) + assert dataset.source_registry_id == source.id + assert dataset.source_snapshot_id == snapshot.id + assert dataset.validation_status == "passed" + assert dataset.provenance_status == "complete" + assert dataset.quarantine_status == "not_quarantined" + assert snapshot.source_registry_id == source.id + assert snapshot.checksum_sha256 == dataset.checksum_sha256 + assert snapshot.ingest_status == "ingested" + assert snapshot.freshness_status == "current" prepared = OrthophotoAcquisitionService._prepared_request(_selection_payload(product_key="1971"), settings) assert prepared["params"]["LAYERS"] == "OKZPAN71VL" @@ -291,6 +324,8 @@ def test_orthophoto_acquisition_persists_georeferenced_raster_and_provenance(tmp assert len(datasets) == 1 assert len(versions) == 1 dataset = datasets[0] + source = next(row for row in db.added if isinstance(row, SourceRegistry)) + snapshot = next(row for row in db.added if isinstance(row, SourceSnapshot)) assert result["output_dataset_id"] == str(dataset.id) assert result["reused"] is False assert dataset.project_id == project_id @@ -301,6 +336,15 @@ def test_orthophoto_acquisition_persists_georeferenced_raster_and_provenance(tmp assert dataset.crs == "EPSG:31370" assert dataset.provenance_metadata["acquisition"] == "explicit_bounded_map_selection" assert dataset.provenance_metadata["request_hash"] == prepared["request_hash"] + assert dataset.source_registry_id == source.id + assert dataset.source_snapshot_id == snapshot.id + assert dataset.validation_status == "passed" + assert dataset.provenance_status == "complete" + assert dataset.lineage_status == "not_applicable" + assert dataset.quarantine_status == "not_quarantined" + assert snapshot.source_registry_id == source.id + assert snapshot.checksum_sha256 == dataset.checksum_sha256 + assert snapshot.freshness_status == "current" assert dataset.source_metadata["attribution"].startswith("Bron: Orthofotomozaiek Vlaanderen") assert dataset.storage_path is not None with rasterio.open(dataset.storage_path) as stored: diff --git a/backend/tests/test_sprint205_dhmv_terrain.py b/backend/tests/test_sprint205_dhmv_terrain.py index 0e9133ff..4b5de6e3 100644 --- a/backend/tests/test_sprint205_dhmv_terrain.py +++ b/backend/tests/test_sprint205_dhmv_terrain.py @@ -17,7 +17,7 @@ from app.core.config import Settings from app.core.errors import AppError from app.db.session import get_db from app.main import app -from app.models import Area, Dataset, DatasetVersion, Job, Project +from app.models import Area, Dataset, DatasetVersion, Job, Project, SourceRegistry, SourceSnapshot from app.schemas.dhmv import DhmvAcquireRequest, TerrainPartitionSelectionRequest, TerrainSelectionRequest from app.services.dhmv_acquisition_service import DhmvAcquisitionService from app.services.terrain_analysis_service import TerrainAnalysisService @@ -27,8 +27,8 @@ ROOT = Path(__file__).resolve().parents[2] class FakeQuery: - def __init__(self, result=None): - self.result = result + def __init__(self, results=None): + self.results = list(results or []) def filter(self, *_args): return self @@ -37,10 +37,13 @@ class FakeQuery: return self def first(self): - return self.result + return self.results[0] if self.results else None + + def one_or_none(self): + return self.first() def all(self): - return self.result if isinstance(self.result, list) else [] + return list(self.results) class FakeSession: @@ -58,6 +61,14 @@ class FakeSession: def add(self, row): self.added.append(row) + def flush(self): + # Exercise the governed source/snapshot import path with database-like + # primary-key assignment instead of silently falling back to legacy + # fixture behavior. + for row in self.added: + if getattr(row, "id", None) is None: + row.id = uuid4() + def commit(self): return None @@ -67,8 +78,18 @@ class FakeSession: def refresh(self, row): return row - def query(self, _model): - return FakeQuery(self.query_result) + def query(self, model): + rows = [ + row + for (row_model, _row_id), row in self.rows.items() + if row_model is model and isinstance(row, model) + ] + rows.extend(row for row in self.added if isinstance(row, model)) + if isinstance(self.query_result, model): + rows.append(self.query_result) + elif isinstance(self.query_result, list): + rows.extend(row for row in self.query_result if isinstance(row, model)) + return FakeQuery(rows) class FakeResponse: @@ -318,6 +339,8 @@ def test_dhmv_acquisition_clips_validates_and_persists_via_dataset_service(tmp_p dataset = next(item for item in db.added if isinstance(item, Dataset)) version = next(item for item in db.added if isinstance(item, DatasetVersion)) + source = next(item for item in db.added if isinstance(item, SourceRegistry)) + snapshot = next(item for item in db.added if isinstance(item, SourceSnapshot)) assert result["output_dataset_id"] == str(dataset.id) assert dataset.source_name == "digitaal_vlaanderen_dhmv" assert dataset.area_id == area_id @@ -331,6 +354,16 @@ def test_dhmv_acquisition_clips_validates_and_persists_via_dataset_service(tmp_p assert dataset.provenance_metadata["water_depth_available"] is False assert dataset.provenance_metadata["water_volume_available"] is False assert len(dataset.provenance_metadata["response_sha256"]) == 64 + assert dataset.source_registry_id == source.id + assert dataset.source_snapshot_id == snapshot.id + assert dataset.validation_status == "passed" + assert dataset.provenance_status == "complete" + assert dataset.lineage_status == "not_applicable" + assert dataset.quarantine_status == "not_quarantined" + assert snapshot.source_registry_id == source.id + assert snapshot.checksum_sha256 == dataset.checksum_sha256 + assert snapshot.ingest_status == "ingested" + assert snapshot.freshness_status == "current" with rasterio.open(dataset.storage_path) as stored: assert stored.crs.to_epsg() == 31370 assert stored.count == 1 diff --git a/backend/tests/test_sprint233_operational_completion.py b/backend/tests/test_sprint233_operational_completion.py index 0e4da5b6..f9463cd1 100644 --- a/backend/tests/test_sprint233_operational_completion.py +++ b/backend/tests/test_sprint233_operational_completion.py @@ -1,8 +1,9 @@ from __future__ import annotations import json +from datetime import UTC, datetime +from hashlib import sha256 from pathlib import Path -from types import SimpleNamespace from uuid import uuid4 import pytest @@ -10,12 +11,13 @@ from fastapi.testclient import TestClient from pydantic import ValidationError from app.main import app -from app.models import Dataset, Export +from app.models import Dataset, Export, SourceRegistry, SourceSnapshot from app.schemas.export import ExportCreateResponse, MapResultExportRequest from app.schemas.project import ProjectRead from app.services.export_service import ExportService from app.services.project_service import ProjectService from app.services.storage_service import StorageService +from app.services.source_registry_service import SourceRegistryService from app.services.temporal_analysis_service import TemporalAnalysisService from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService @@ -48,6 +50,71 @@ def bbox_payload() -> dict: } +def governed_dataset( + *, + project_id, + dataset_id, + name: str, + dataset_type: str, + source_key: str, + dataset_role: str = "source", +) -> Dataset: + """Build an in-memory stand-in for a passed governed dataset. + + Map-result export is an operational consumption boundary. These tests + must therefore model the same source registry/snapshot, checksum and + passed-contract evidence supplied by a real adapter rather than relying + on an old transient Dataset fixture. + """ + + source = SourceRegistry( + id=uuid4(), + **SourceRegistryService.definition_for(source_key).as_model_values(), + ) + checksum = sha256(f"{dataset_id}:{source_key}:{dataset_type}".encode("utf-8")).hexdigest() + snapshot = SourceSnapshot( + id=uuid4(), + source_registry_id=source.id, + snapshot_key=f"test:{source_key}:{checksum}", + checksum_sha256=checksum, + fetched_at=datetime.now(UTC), + crs="EPSG:31370", + units=source.default_units, + spatial_resolution_json={"x": 1.0, "y": 1.0, "unit": "m"}, + temporal_coverage_json={"status": "test-fixture"}, + geographic_coverage_json={"zone": "Flanders"}, + observed_schema_json={"dataset_type": dataset_type}, + freshness_status="current", + ingest_status="ingested", + known_limitations_json=["In-memory governed fixture used only by this export test."], + snapshot_metadata_json={"fixture_mode": True}, + ) + return Dataset( + id=dataset_id, + project_id=project_id, + name=name, + dataset_type=dataset_type, + source="governed test fixture", + dataset_role=dataset_role, + source_name=source.source_key, + source_registry_id=source.id, + source_snapshot_id=snapshot.id, + source_registry=source, + source_snapshot=snapshot, + data_contract_key=("geointel.raster.geotiff" if dataset_type == "raster" else "geointel.vector.geojson"), + data_contract_version="1.0.0", + validation_status="passed", + provenance_status="complete", + lineage_status="not_applicable", + quarantine_status="not_quarantined", + checksum_sha256=checksum, + metadata_json={"fixture_mode": True}, + source_metadata={"fixture_mode": True, "source_registry_key": source.source_key}, + provenance_metadata={"fixture_mode": True, "source_snapshot_id": str(snapshot.id)}, + status="ready", + ) + + def test_map_result_export_request_requires_a_complete_target() -> None: with pytest.raises(ValidationError): MapResultExportRequest(project_id=uuid4(), mode="current", bbox=bbox_payload()) @@ -67,13 +134,12 @@ def test_current_vector_map_result_uses_authoritative_selection_export(monkeypat project_id = uuid4() dataset_id = uuid4() area_id = uuid4() - dataset = Dataset( - id=dataset_id, + dataset = governed_dataset( project_id=project_id, + dataset_id=dataset_id, name="buildings.geojson", dataset_type="vector", - source="fixture", - status="ready", + source_key="grb", ) db = FakeSession({(Dataset, dataset_id): dataset}) expected = ExportCreateResponse( @@ -109,14 +175,12 @@ def test_current_vector_map_result_uses_authoritative_selection_export(monkeypat def test_partitioned_vector_map_result_uses_governed_partition_export(monkeypatch) -> None: project_id = uuid4() dataset_id = uuid4() - dataset = Dataset( - id=dataset_id, + dataset = governed_dataset( project_id=project_id, + dataset_id=dataset_id, name="vha-municipality.geojson", dataset_type="vector", - source="VHA", - source_name="vmm_vha_bathymetry_profiles", - status="ready", + source_key="vmm_vha_bathymetry_profiles", ) db = FakeSession({(Dataset, dataset_id): dataset}) expected = ExportCreateResponse( @@ -159,14 +223,12 @@ def test_partitioned_vector_map_result_uses_governed_partition_export(monkeypatc def test_raster_map_result_is_recomputed_and_persisted(tmp_path, monkeypatch) -> None: project_id = uuid4() dataset_id = uuid4() - dataset = Dataset( - id=dataset_id, + dataset = governed_dataset( project_id=project_id, + dataset_id=dataset_id, name="space-occupation.tif", dataset_type="raster", - source="official", - source_name="department_omgeving_thematic_raster", - status="ready", + source_key="department_omgeving_thematic_raster", ) db = FakeSession({(Dataset, dataset_id): dataset}) export_path = tmp_path / "space-occupation-analysis.json" @@ -209,7 +271,26 @@ def test_evolution_map_result_is_recomputed_and_persisted(tmp_path, monkeypatch) project_id = uuid4() earlier_id = uuid4() later_id = uuid4() - db = FakeSession() + earlier_dataset = governed_dataset( + project_id=project_id, + dataset_id=earlier_id, + name="forest-earlier.geojson", + dataset_type="vector", + source_key="inbo_bwk_natura2000", + ) + later_dataset = governed_dataset( + project_id=project_id, + dataset_id=later_id, + name="forest-later.geojson", + dataset_type="vector", + source_key="inbo_bwk_natura2000", + ) + db = FakeSession( + { + (Dataset, earlier_id): earlier_dataset, + (Dataset, later_id): later_dataset, + } + ) export_path = tmp_path / "forest-evolution.json" captured: dict = {} diff --git a/backend/tests/test_sprint7a_persistence_foundation.py b/backend/tests/test_sprint7a_persistence_foundation.py index be11a73e..b25be683 100644 --- a/backend/tests/test_sprint7a_persistence_foundation.py +++ b/backend/tests/test_sprint7a_persistence_foundation.py @@ -6,6 +6,7 @@ from uuid import uuid4 import pytest from geoalchemy2.shape import to_shape +from pyproj import Transformer from app.api.routes.qa import compare_candidate_with_reference from app.models import Dataset, Metric, Project, QualityCheck, VectorFeature @@ -117,6 +118,54 @@ def test_vector_feature_service_normalizes_source_z_coordinates_to_canonical_2d( assert to_shape(persisted[0].geometry).has_z is False +def test_vector_feature_service_transforms_declared_source_crs_before_epsg4326_storage() -> None: + to_lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True) + x, y = to_lambert.transform(4.7, 51.1) + db = FakeSession() + + persisted = VectorFeatureService.persist_geojson_features( + db=db, + dataset_id=uuid4(), + payload={ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "id": "lambert-point", + "properties": {}, + "geometry": {"type": "Point", "coordinates": [x, y]}, + } + ], + }, + source_crs="EPSG:31370", + ) + + geometry = to_shape(persisted[0].geometry) + assert geometry.x == pytest.approx(4.7, abs=0.000001) + assert geometry.y == pytest.approx(51.1, abs=0.000001) + + +def test_vector_feature_service_rejects_invalid_declared_source_crs() -> None: + with pytest.raises(Exception) as exc_info: + VectorFeatureService.persist_geojson_features( + db=FakeSession(), + dataset_id=uuid4(), + payload={ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": {}, + "geometry": {"type": "Point", "coordinates": [4.7, 51.1]}, + } + ], + }, + source_crs="EPSG:not-a-crs", + ) + + assert getattr(exc_info.value, "code", None) == "INVALID_DATASET_CRS" + + def test_dataset_upload_persists_vector_features(monkeypatch, tmp_path) -> None: project_id = uuid4() db = FakeSession(objects={(Project, project_id): Project(id=project_id, name="Geel")}) diff --git a/backend/tests/test_sprint8_detection_foundation.py b/backend/tests/test_sprint8_detection_foundation.py index 4e144904..259ceba9 100644 --- a/backend/tests/test_sprint8_detection_foundation.py +++ b/backend/tests/test_sprint8_detection_foundation.py @@ -43,7 +43,7 @@ def _project_and_dataset(dataset_type: str = "raster"): project_id=project_id, name="source.tif", dataset_type=dataset_type, - source="user_upload", + source="test", storage_path="storage/uploads/source.tif", ) db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset}) diff --git a/backend/tests/test_sprint8b_yolo_foundation.py b/backend/tests/test_sprint8b_yolo_foundation.py index a1d8bb84..a42eb6f0 100644 --- a/backend/tests/test_sprint8b_yolo_foundation.py +++ b/backend/tests/test_sprint8b_yolo_foundation.py @@ -1,5 +1,6 @@ from __future__ import annotations +from hashlib import sha256 import json from pathlib import Path import sys @@ -10,10 +11,11 @@ import pytest from app.core.config import Settings from app.core.errors import AppError -from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project +from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project, SourceRegistry, SourceSnapshot from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon from app.services.detection_service import DetectionService from app.services.model_registry_service import ModelRegistryService +from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService from app.services.yolo_adapter import YoloDetectionAdapter ROOT = Path(__file__).resolve().parents[2] @@ -79,6 +81,14 @@ class MockYoloAdapter: ] +class NeverLoadUnboundModelAdapter(MockYoloAdapter): + load_calls = 0 + + def load_model(self, model_path: Path): + type(self).load_calls += 1 + raise AssertionError("unbound model provenance must be rejected before adapter.load_model") + + class MixedCaseYoloAdapter(MockYoloAdapter): def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict]: return [ @@ -141,15 +151,47 @@ class ExplodingPredictModel: def _project_and_dataset(dataset_type: str = "raster"): project_id = uuid4() dataset_id = uuid4() + source_registry_id = uuid4() + source_snapshot_id = uuid4() + checksum = "a" * 64 project = Project(id=project_id, name="Geel") + source_registry = SourceRegistry( + id=source_registry_id, + source_key="test-derived-raster", + display_name="Governed test-derived raster", + classification="derived", + authority_name="GeoIntel test fixture", + usage_policy_json={"ground_truth_allowed": False}, + ) + source_snapshot = SourceSnapshot( + id=source_snapshot_id, + source_registry_id=source_registry_id, + snapshot_key="test-derived-raster-v1", + checksum_sha256=checksum, + freshness_status="current", + ingest_status="ingested", + ) dataset = Dataset( id=dataset_id, project_id=project_id, name="source.tif", dataset_type=dataset_type, - source="user_upload", + source="test-derived-raster", + source_name="test-derived-raster", storage_path="storage/uploads/source.tif", + checksum_sha256=checksum, + source_registry_id=source_registry_id, + source_snapshot_id=source_snapshot_id, + data_contract_key="geointel.raster.geotiff", + data_contract_version="1.0.0", + validation_status="passed", + provenance_status="complete", + lineage_status="not_applicable", + quarantine_status="not_quarantined", + status="ready", ) + dataset.source_registry = source_registry + dataset.source_snapshot = source_snapshot db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset}) return db, project_id, dataset_id @@ -165,6 +207,74 @@ def _settings(tmp_path: Path, **overrides) -> Settings: return Settings(**values) +def _write_model_sidecar( + model_path: Path, + settings: Settings, + *, + db: FakeSession | None = None, +) -> None: + """Create explicit test-only evidence; production never self-generates it.""" + + model_sha256 = sha256(model_path.read_bytes()).hexdigest() + source_registry_id = uuid4() + source_snapshot_id = uuid4() + source_version = settings.yolo_model_version or "test-v1" + if db is not None: + source_registry = SourceRegistry( + id=source_registry_id, + source_key="model", + display_name="Governed test model artifact", + classification="experimental", + authority_name="GeoIntel test fixture", + freshness_status="current", + ingest_status="configured", + ) + source_snapshot = SourceSnapshot( + id=source_snapshot_id, + source_registry_id=source_registry_id, + snapshot_key=f"model-{source_version}", + source_version=source_version, + checksum_sha256=model_sha256, + freshness_status="current", + ingest_status="ingested", + ) + db.objects[(SourceRegistry, source_registry_id)] = source_registry + db.objects[(SourceSnapshot, source_snapshot_id)] = source_snapshot + payload = { + "schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION, + "data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"}, + "model": { + "model_id": settings.yolo_model_id, + "task_type": "object_detection", + "sha256": model_sha256, + "model_format": "pytorch", + "framework": "ultralytics/pytorch", + "class_mapping": {"0": "building"}, + "source_version": source_version, + }, + "source": { + "source_registry_id": str(source_registry_id), + "source_snapshot_id": str(source_snapshot_id), + "source_registry_key": "model", + "source_snapshot_checksum_sha256": model_sha256, + }, + "lineage": { + "upstream_asset_ids": ["test-training-corpus"], + "upstream_checksums_sha256": ["a" * 64], + "transformations": [ + {"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64} + ], + }, + "metadata": {"training_manifest_sha256": "c" * 64}, + "imported_at": "2026-08-01T10:00:00+00:00", + } + payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload) + RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text( + json.dumps(payload, sort_keys=True), + encoding="utf-8", + ) + + def _manifest(tmp_path: Path, tile_count: int = 1) -> Path: tiles = [] for index in range(tile_count): @@ -223,10 +333,28 @@ def test_yolo_configured_model_reports_dependency_unavailable(tmp_path: Path) -> assert model.status == "dependency_unavailable" +def test_yolo_configured_model_requires_a_runtime_provenance_sidecar(tmp_path: Path) -> None: + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"unmanifested local weights") + settings = _settings(tmp_path, yolo_model_path=str(model_path)) + + model = ModelRegistryService.get_model_capability( + "yolo-configured", + settings=settings, + yolo_adapter_class=AvailableAdapter, + ) + + assert model is not None + assert model.configured is False + assert model.status == "contract_incomplete" + assert "sidecar" in model.limitation_message + + def test_yolo_configured_model_reports_configured_with_local_model_and_dependencies(tmp_path: Path) -> None: model_path = tmp_path / "model.pt" model_path.write_bytes(b"local weights") settings = _settings(tmp_path, yolo_model_path=str(model_path)) + _write_model_sidecar(model_path, settings) model = ModelRegistryService.get_model_capability("yolo-configured", settings=settings, yolo_adapter_class=AvailableAdapter) @@ -306,11 +434,37 @@ def test_yolo_run_requires_tile_manifest_path(tmp_path: Path) -> None: assert getattr(exc_info.value, "code", None) == "DETECTION_TILE_MANIFEST_REQUIRED" +def test_yolo_run_fails_closed_before_adapter_load_without_sidecar(tmp_path: Path) -> None: + db, project_id, dataset_id = _project_and_dataset() + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"unmanifested local weights") + settings = _settings(tmp_path, yolo_model_path=str(model_path)) + + # AvailableAdapter intentionally has no load_model method. If runtime + # provenance were checked after adapter loading, this would raise instead + # of returning the explicit unavailable capability state. + result = DetectionService.run_detection( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="yolo-configured", + confidence_threshold=0.5, + tile_manifest_path=str(_manifest(tmp_path)), + settings=settings, + yolo_adapter_class=AvailableAdapter, + ) + + assert result.status == "failed" + assert result.error_code == "DETECTION_MODEL_UNAVAILABLE" + assert "sidecar" in result.message + + def test_yolo_run_rejects_manifest_over_tile_limit(tmp_path: Path) -> None: db, project_id, dataset_id = _project_and_dataset() model_path = tmp_path / "model.pt" model_path.write_bytes(b"local weights") settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_max_tiles=1) + _write_model_sidecar(model_path, settings, db=db) manifest_path = _manifest(tmp_path, tile_count=2) result = DetectionService.run_detection( @@ -333,6 +487,7 @@ def test_yolo_run_rejects_missing_tile_manifest_file(tmp_path: Path) -> None: model_path = tmp_path / "model.pt" model_path.write_bytes(b"local weights") settings = _settings(tmp_path, yolo_model_path=str(model_path)) + _write_model_sidecar(model_path, settings, db=db) result = DetectionService.run_detection( db=db, @@ -354,6 +509,7 @@ def test_yolo_run_rejects_invalid_tile_manifest_json(tmp_path: Path) -> None: model_path = tmp_path / "model.pt" model_path.write_bytes(b"local weights") settings = _settings(tmp_path, yolo_model_path=str(model_path)) + _write_model_sidecar(model_path, settings, db=db) manifest_path = tmp_path / "manifest.json" manifest_path.write_text("{not-json", encoding="utf-8") @@ -372,6 +528,32 @@ def test_yolo_run_rejects_invalid_tile_manifest_json(tmp_path: Path) -> None: assert result.error_code == "DETECTION_TILE_MANIFEST_INVALID" +def test_yolo_run_rejects_unbound_model_snapshot_before_adapter_load(tmp_path: Path) -> None: + db, project_id, dataset_id = _project_and_dataset() + model_path = tmp_path / "model.pt" + model_path.write_bytes(b"structurally valid but unbound model") + settings = _settings(tmp_path, yolo_model_path=str(model_path)) + # A catalog/preflight sidecar alone is deliberately insufficient for a + # production call. Do not register the declared source IDs in ``db``. + _write_model_sidecar(model_path, settings) + NeverLoadUnboundModelAdapter.load_calls = 0 + + result = DetectionService.run_detection( + db=db, + project_id=project_id, + dataset_id=dataset_id, + model_id="yolo-configured", + confidence_threshold=0.5, + tile_manifest_path=str(_manifest(tmp_path)), + settings=settings, + yolo_adapter_class=NeverLoadUnboundModelAdapter, + ) + + assert result.status == "failed" + assert result.error_code == "MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND" + assert NeverLoadUnboundModelAdapter.load_calls == 0 + + def test_pixel_bbox_to_epsg4326_polygon_from_gdal_transform() -> None: polygon = pixel_bbox_to_epsg4326_polygon( bbox=[10, 20, 30, 40], @@ -390,6 +572,7 @@ def test_yolo_run_persists_mocked_georeferenced_detections(tmp_path: Path) -> No model_path = tmp_path / "model.pt" model_path.write_bytes(b"local weights") settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_model_version="local-test") + _write_model_sidecar(model_path, settings, db=db) manifest_path = _manifest(tmp_path, tile_count=1) result = DetectionService.run_detection( @@ -416,7 +599,10 @@ def test_yolo_run_persists_mocked_georeferenced_detections(tmp_path: Path) -> No assert detections[0].confidence == 0.91 assert detections[0].source_tile_path.endswith("tile_0000.tif") assert detections[0].bbox_json == {"x_min": 10.0, "y_min": 20.0, "x_max": 30.0, "y_max": 40.0} - assert detections[0].properties_json == {"adapter": "mock", "tile_index": 0} + assert detections[0].properties_json["adapter"] == "mock" + assert detections[0].properties_json["tile_index"] == 0 + assert detections[0].properties_json["runtime_model_provenance"]["model_sha256"] == sha256(model_path.read_bytes()).hexdigest() + assert runs[0].parameters_json["runtime_model_provenance"]["data_contract_key"] == "geointel.model.pytorch" assert runs[0].status == "success" assert jobs[0].status == "success" @@ -426,6 +612,7 @@ def test_yolo_class_filter_is_case_insensitive_and_persists_canonical_class(tmp_ model_path = tmp_path / "model.pt" model_path.write_bytes(b"local weights") settings = _settings(tmp_path, yolo_model_path=str(model_path)) + _write_model_sidecar(model_path, settings, db=db) manifest_path = _manifest(tmp_path, tile_count=1) result = DetectionService.run_detection( @@ -453,6 +640,7 @@ def test_yolo_run_suppresses_cross_tile_duplicate_detections(tmp_path: Path) -> model_path = tmp_path / "model.pt" model_path.write_bytes(b"local weights") settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_duplicate_iou_threshold=0.5) + _write_model_sidecar(model_path, settings, db=db) manifest_path = _manifest(tmp_path, tile_count=2) result = DetectionService.run_detection( diff --git a/backend/tests/test_sprint8c_detection_visualization_qa.py b/backend/tests/test_sprint8c_detection_visualization_qa.py index 7022395a..2fbb7f1e 100644 --- a/backend/tests/test_sprint8c_detection_visualization_qa.py +++ b/backend/tests/test_sprint8c_detection_visualization_qa.py @@ -12,7 +12,7 @@ from shapely.geometry import Polygon, box from app.core.errors import AppError from app.main import app from app.db.session import get_db -from app.models import AnalysisRun, Dataset, Detection, Metric, QualityCheck, VectorFeature +from app.models import AnalysisRun, Dataset, Detection, Metric, QualityCheck, SourceRegistry, SourceSnapshot, VectorFeature from app.services.detection_service import DetectionService @@ -96,11 +96,52 @@ def _source_dataset(project_id, dataset_id): project_id=project_id, name="source.tif", dataset_type="raster", - source="manual", - source_name="manual", + source="test", + source_name="test", ) +def _authoritative_reference(dataset: Dataset) -> Dataset: + """Model the reference as a fully governed GRB fixture, never test data.""" + + source_id = uuid4() + snapshot_id = uuid4() + checksum = "a" * 64 + source = SourceRegistry( + id=source_id, + source_key="grb", + display_name="GRB QA fixture", + classification="authoritative", + authority_name="Digitaal Vlaanderen", + authority_scope_json={"zone": "Flanders"}, + usage_policy_json={"ground_truth_allowed": True, "validation_authority": {"building_validation": "primary"}}, + ) + snapshot = SourceSnapshot( + id=snapshot_id, + source_registry_id=source_id, + snapshot_key=f"detection-qa-{dataset.id}", + checksum_sha256=checksum, + ingest_status="ingested", + freshness_status="current", + ) + dataset.source = "grb" + dataset.source_name = "grb" + dataset.dataset_role = "reference" + dataset.status = "ready" + dataset.checksum_sha256 = checksum + dataset.source_registry_id = source_id + dataset.source_snapshot_id = snapshot_id + dataset.data_contract_key = "geointel.vector.geojson" + dataset.data_contract_version = "1.0.0" + dataset.validation_status = "passed" + dataset.provenance_status = "complete" + dataset.lineage_status = "complete" + dataset.quarantine_status = "not_quarantined" + dataset.source_registry = source + dataset.source_snapshot = snapshot + return dataset + + def test_detection_geojson_feature_collection_shape() -> None: project_id = uuid4() dataset_id = uuid4() @@ -201,14 +242,14 @@ def test_detection_qa_persists_quality_check_and_metrics() -> None: reference_dataset_id = uuid4() analysis_run_id = uuid4() detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1)) - reference_dataset = Dataset( + reference_dataset = _authoritative_reference(Dataset( id=reference_dataset_id, project_id=project_id, name="reference.geojson", dataset_type="vector", - source="manual", + source="test", dataset_role="reference", - ) + )) reference_feature = VectorFeature( id=uuid4(), dataset_id=reference_dataset_id, @@ -262,7 +303,7 @@ def test_detection_qa_rejects_non_overlapping_historical_reference_editions() -> source_dataset.source_metadata = {"product_key": "2020", "supports_detection": False} source_dataset.valid_from = datetime(2020, 1, 1, tzinfo=UTC) source_dataset.valid_to = datetime(2020, 12, 31, 23, 59, 59, tzinfo=UTC) - reference_dataset = Dataset( + reference_dataset = _authoritative_reference(Dataset( id=reference_dataset_id, project_id=project_id, name="current-grb.geojson", @@ -272,7 +313,7 @@ def test_detection_qa_rejects_non_overlapping_historical_reference_editions() -> dataset_role="reference", valid_from=datetime(2026, 7, 1, tzinfo=UTC), valid_to=datetime(2026, 7, 31, 23, 59, 59, tzinfo=UTC), - ) + )) db = FakeSession( objects={ (AnalysisRun, analysis_run_id): AnalysisRun( @@ -305,14 +346,14 @@ def test_detection_qa_no_match_case_persists_zero_scores() -> None: reference_dataset_id = uuid4() analysis_run_id = uuid4() detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1)) - reference_dataset = Dataset( + reference_dataset = _authoritative_reference(Dataset( id=reference_dataset_id, project_id=project_id, name="reference.geojson", dataset_type="vector", - source="manual", + source="test", dataset_role="reference", - ) + )) reference_feature = VectorFeature( id=uuid4(), dataset_id=reference_dataset_id, @@ -349,14 +390,14 @@ def test_configured_yolo_qa_requires_persisted_tile_manifest_provenance() -> Non reference_dataset_id = uuid4() analysis_run_id = uuid4() detection = _detection(project_id, dataset_id, analysis_run_id) - reference_dataset = Dataset( + reference_dataset = _authoritative_reference(Dataset( id=reference_dataset_id, project_id=project_id, name="reference.geojson", dataset_type="vector", - source="manual", + source="test", dataset_role="reference", - ) + )) reference_feature = VectorFeature( id=uuid4(), dataset_id=reference_dataset_id, @@ -421,14 +462,14 @@ def test_detection_qa_excludes_references_outside_persisted_tile_coverage(tmp_pa analysis_run_id = uuid4() manifest_path = _coverage_manifest(tmp_path, dataset_id, bounds=(0.0, 0.0, 1.0, 1.0)) detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0.1, 0.1, 0.9, 0.9)) - reference_dataset = Dataset( + reference_dataset = _authoritative_reference(Dataset( id=reference_dataset_id, project_id=project_id, name="reference.geojson", dataset_type="vector", - source="manual", + source="test", dataset_role="reference", - ) + )) inside_reference = VectorFeature( id=uuid4(), dataset_id=reference_dataset_id, @@ -489,14 +530,14 @@ def test_detection_qa_reports_box_to_footprint_diagnostic_without_changing_stric l_shaped_footprint = Polygon( [(0.0, 0.0), (2.0, 0.0), (2.0, 0.4), (0.4, 0.4), (0.4, 2.0), (0.0, 2.0), (0.0, 0.0)] ) - reference_dataset = Dataset( + reference_dataset = _authoritative_reference(Dataset( id=reference_dataset_id, project_id=project_id, name="reference.geojson", dataset_type="vector", - source="manual", + source="test", dataset_role="reference", - ) + )) reference_feature = VectorFeature( id=uuid4(), dataset_id=reference_dataset_id, diff --git a/backend/tests/test_sprint9_segmentation_foundation.py b/backend/tests/test_sprint9_segmentation_foundation.py index 3959f20f..b35dd933 100644 --- a/backend/tests/test_sprint9_segmentation_foundation.py +++ b/backend/tests/test_sprint9_segmentation_foundation.py @@ -10,7 +10,18 @@ from shapely.geometry import MultiPolygon, box, mapping from app.db.session import get_db from app.main import app -from app.models import AnalysisRun, Dataset, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature +from app.models import ( + AnalysisRun, + Dataset, + Job, + Metric, + Project, + QualityCheck, + Segmentation, + SourceRegistry, + SourceSnapshot, + VectorFeature, +) from app.services.model_registry_service import ModelRegistryService from app.services.segmentation_service import SegmentationService @@ -75,7 +86,7 @@ def _project_and_dataset(dataset_type: str = "raster"): project_id=project_id, name="source.tif", dataset_type=dataset_type, - source="user_upload", + source="test", storage_path="storage/uploads/source.tif", ) db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset}) @@ -105,6 +116,47 @@ def _segmentation(project_id, dataset_id, analysis_run_id, class_name="vegetatio ) +def _authoritative_reference(dataset: Dataset) -> Dataset: + """Model segmentation QA references as governed authoritative GRB evidence.""" + + source_id = uuid4() + snapshot_id = uuid4() + checksum = "a" * 64 + source = SourceRegistry( + id=source_id, + source_key="grb", + display_name="GRB segmentation QA fixture", + classification="authoritative", + authority_name="Digitaal Vlaanderen", + authority_scope_json={"zone": "Flanders"}, + usage_policy_json={"ground_truth_allowed": True, "validation_authority": {"building_validation": "primary"}}, + ) + snapshot = SourceSnapshot( + id=snapshot_id, + source_registry_id=source_id, + snapshot_key=f"segmentation-qa-{dataset.id}", + checksum_sha256=checksum, + ingest_status="ingested", + freshness_status="current", + ) + dataset.source = "grb" + dataset.source_name = "grb" + dataset.dataset_role = "reference" + dataset.status = "ready" + dataset.checksum_sha256 = checksum + dataset.source_registry_id = source_id + dataset.source_snapshot_id = snapshot_id + dataset.data_contract_key = "geointel.vector.geojson" + dataset.data_contract_version = "1.0.0" + dataset.validation_status = "passed" + dataset.provenance_status = "complete" + dataset.lineage_status = "complete" + dataset.quarantine_status = "not_quarantined" + dataset.source_registry = source + dataset.source_snapshot = snapshot + return dataset + + def test_model_registry_returns_segmentation_states() -> None: models = {model.model_id: model for model in ModelRegistryService.list_model_capabilities(task_type="segmentation")} @@ -265,14 +317,14 @@ def test_segmentation_qa_persists_quality_check_and_metrics() -> None: reference_dataset_id = uuid4() analysis_run_id = uuid4() segmentation = _segmentation(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1)) - reference_dataset = Dataset( + reference_dataset = _authoritative_reference(Dataset( id=reference_dataset_id, project_id=project_id, name="reference.geojson", dataset_type="vector", - source="manual", + source="test", dataset_role="reference", - ) + )) reference_feature = VectorFeature( id=uuid4(), dataset_id=reference_dataset_id, @@ -282,6 +334,7 @@ def test_segmentation_qa_persists_quality_check_and_metrics() -> None: db = FakeSession( objects={ (AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="segmentation", status="success", parameters_json={}), + (Dataset, dataset_id): Dataset(id=dataset_id, project_id=project_id, name="fixture.tif", dataset_type="raster", source="test"), (Dataset, reference_dataset_id): reference_dataset, }, query_rows={Segmentation: [segmentation], VectorFeature: [reference_feature]}, @@ -318,14 +371,14 @@ def test_segmentation_qa_no_match_case_persists_zero_scores() -> None: reference_dataset_id = uuid4() analysis_run_id = uuid4() segmentation = _segmentation(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1)) - reference_dataset = Dataset( + reference_dataset = _authoritative_reference(Dataset( id=reference_dataset_id, project_id=project_id, name="reference.geojson", dataset_type="vector", - source="manual", + source="test", dataset_role="reference", - ) + )) reference_feature = VectorFeature( id=uuid4(), dataset_id=reference_dataset_id, @@ -335,6 +388,7 @@ def test_segmentation_qa_no_match_case_persists_zero_scores() -> None: db = FakeSession( objects={ (AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="segmentation", status="success", parameters_json={}), + (Dataset, dataset_id): Dataset(id=dataset_id, project_id=project_id, name="fixture.tif", dataset_type="raster", source="test"), (Dataset, reference_dataset_id): reference_dataset, }, query_rows={Segmentation: [segmentation], VectorFeature: [reference_feature]}, diff --git a/backend/tests/test_training_dataset_eligibility.py b/backend/tests/test_training_dataset_eligibility.py new file mode 100644 index 00000000..67942482 --- /dev/null +++ b/backend/tests/test_training_dataset_eligibility.py @@ -0,0 +1,410 @@ +from __future__ import annotations + +import importlib.util +import hashlib +import json +from pathlib import Path +from types import SimpleNamespace +from uuid import uuid4 + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "training_dataset_eligibility.py" +SPEC = importlib.util.spec_from_file_location("training_dataset_eligibility", SCRIPT) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + +CHECKSUM = "a" * 64 +UNSET = object() + + +def source_registry( + *, + classification: str = "authoritative", + training_allowed: bool = True, + ground_truth_allowed: bool = True, + allowed_tasks: list[str] | None = None, + building_validation_authority: str = "primary", +) -> SimpleNamespace: + return SimpleNamespace( + id="source-registry-1", + source_key="governed-source", + classification=classification, + freshness_status="current", + ingest_status="ingested", + usage_policy_json={ + "training_allowed": training_allowed, + "ground_truth_allowed": ground_truth_allowed, + "allowed_tasks": allowed_tasks or ["building_validation", "building_labels"], + "validation_authority": { + "building_validation": building_validation_authority, + }, + }, + ) + + +def source_snapshot(*, checksum: str = CHECKSUM) -> SimpleNamespace: + return SimpleNamespace( + id="source-snapshot-1", + snapshot_key="2026-08-01", + checksum_sha256=checksum, + freshness_status="current", + ingest_status="ingested", + ) + + +def governed_dataset( + *, + role: str, + registry: SimpleNamespace | None | object = UNSET, + snapshot: SimpleNamespace | None | object = UNSET, + **overrides: object, +) -> SimpleNamespace: + dataset_type = "raster" if role == "raster" else "vector" + values: dict[str, object] = { + "id": f"dataset-{role}", + "dataset_type": dataset_type, + "dataset_role": "source" if role == "raster" else "reference", + "source": "governed_import", + "source_name": "governed-source", + "checksum_sha256": CHECKSUM, + "data_contract_key": f"{role}-contract", + "data_contract_version": "1.0.0", + "validation_status": "passed", + "provenance_status": "complete", + "lineage_status": "not_applicable", + "quarantine_status": "not_quarantined", + "status": "ready", + "metadata_json": {}, + "provenance_metadata": {}, + "source_registry": source_registry() if registry is UNSET else registry, + "source_snapshot": source_snapshot() if snapshot is UNSET else snapshot, + } + values.update(overrides) + return SimpleNamespace(**values) + + +def test_governed_authoritative_pair_is_eligible_for_operational_training() -> None: + raster = governed_dataset( + role="raster", + registry=source_registry(ground_truth_allowed=False), + ) + reference = governed_dataset(role="reference") + + decision = MODULE.training_pair_evidence(raster=raster, reference=reference) + + assert decision["eligible"] is True + assert decision["raster"]["reasons"] == [] + assert decision["reference"]["evidence"]["source_ground_truth_allowed"] is True + + +def test_operational_training_rejects_invalid_quarantined_incomplete_and_untrusted_inputs() -> None: + dataset = governed_dataset( + role="reference", + validation_status="failed", + provenance_status="incomplete", + lineage_status="incomplete", + quarantine_status="quarantined", + source_registry=source_registry( + classification="contextual", + training_allowed=False, + ground_truth_allowed=False, + ), + ) + + decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference") + + assert decision.eligible is False + assert set(decision.reasons) >= { + "validation_failed", + "dataset_quarantined", + "provenance_not_complete", + "lineage_not_complete", + "source_not_allowed_for_training", + "reference_source_not_authoritative", + "reference_source_not_ground_truth_allowed", + } + + +def test_operational_training_rejects_a_due_source_snapshot() -> None: + snapshot = source_snapshot() + snapshot.freshness_status = "due" + dataset = governed_dataset(role="reference", snapshot=snapshot) + + decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference") + + assert decision.eligible is False + assert "source_snapshot_freshness_not_approved" in decision.reasons + + +def test_osm_like_context_is_never_accepted_as_building_ground_truth() -> None: + dataset = governed_dataset( + role="reference", + source_name="osm", + source_registry=source_registry( + classification="contextual", + training_allowed=False, + ground_truth_allowed=False, + ), + ) + + decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference") + + assert decision.eligible is False + assert "source_not_allowed_for_training" in decision.reasons + assert "reference_source_not_authoritative" in decision.reasons + + +def test_regional_building_sources_pending_primary_authority_cannot_enter_training_labels() -> None: + for source_key in ("spw_picc", "urbis"): + dataset = governed_dataset( + role="reference", + source_name=source_key, + source_registry=source_registry( + allowed_tasks=["building_validation", "building_labels"], + building_validation_authority="regional_primary_pending_contract", + ), + ) + + decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference") + + assert decision.eligible is False + assert "reference_building_validation_not_primary" in decision.reasons + + +def test_authoritative_source_without_building_validation_task_cannot_be_used_as_a_label_reference() -> None: + dataset = governed_dataset( + role="reference", + source_registry=source_registry( + allowed_tasks=["elevation_validation"], + building_validation_authority="corroborative", + ), + ) + + decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference") + + assert decision.eligible is False + assert set(decision.reasons) >= { + "reference_source_not_approved_for_building_validation", + "reference_building_validation_not_primary", + } + + +def test_dataset_and_snapshot_registry_bindings_cannot_be_forged() -> None: + snapshot = source_snapshot() + snapshot.source_registry_id = "different-registry" + dataset = governed_dataset( + role="reference", + registry=source_registry(), + snapshot=snapshot, + source_registry_id="different-registry", + source_snapshot_id="different-snapshot", + ) + + decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference") + + assert decision.eligible is False + assert set(decision.reasons) >= { + "dataset_source_registry_binding_mismatch", + "dataset_source_snapshot_binding_mismatch", + "source_snapshot_registry_mismatch", + } + + +def test_fixture_mode_only_relaxes_legacy_provenance_for_explicit_fixtures() -> None: + fixture = governed_dataset( + role="reference", + source="fixture", + source_name="fixture", + validation_status=None, + provenance_status="incomplete", + lineage_status="incomplete", + data_contract_key=None, + data_contract_version=None, + checksum_sha256=None, + source_registry=None, + source_snapshot=None, + metadata_json={"fixture": True}, + ) + unmarked = governed_dataset( + role="reference", + validation_status=None, + provenance_status="incomplete", + lineage_status="incomplete", + source_registry=None, + source_snapshot=None, + ) + + assert MODULE.evaluate_dataset_training_eligibility( + fixture, + role="reference", + fixture_mode=True, + ).eligible is True + rejected = MODULE.evaluate_dataset_training_eligibility( + unmarked, + role="reference", + fixture_mode=True, + ) + assert rejected.eligible is False + assert "fixture_mode_requires_explicit_fixture" in rejected.reasons + + +def test_fixture_mode_never_allows_failed_validation_or_quarantine() -> None: + fixture = governed_dataset( + role="raster", + source="fixture", + source_name="fixture", + validation_status="failed", + quarantine_status="quarantined", + source_registry=None, + source_snapshot=None, + ) + + decision = MODULE.evaluate_dataset_training_eligibility(fixture, role="raster", fixture_mode=True) + + assert decision.eligible is False + assert set(decision.reasons) >= {"validation_failed", "dataset_quarantined"} + + +def test_manifest_gate_rejects_missing_or_tampered_pair_decisions() -> None: + raster = governed_dataset( + role="raster", + registry=source_registry(ground_truth_allowed=False), + ) + reference = governed_dataset(role="reference") + pair = MODULE.training_pair_evidence(raster=raster, reference=reference) + manifest = { + "samples": [{"sample_slug": "governed", "training_eligibility": pair}], + "training_eligibility": { + "policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION, + "status": "eligible", + "fixture_mode": False, + }, + } + + assert MODULE.manifest_training_eligibility_failures(manifest) == [] + tampered = { + **manifest, + "samples": [{"sample_slug": "governed", "training_eligibility": {**pair, "eligible": False}}], + } + failures = MODULE.manifest_training_eligibility_failures(tampered) + assert "governed:training_pair_not_eligible" in failures + assert MODULE.manifest_training_eligibility_failures({"samples": []}) == [ + "manifest_training_eligibility_missing" + ] + + +def test_frozen_manifest_gate_detects_checksum_tampering(tmp_path: Path) -> None: + raster = governed_dataset( + role="raster", + registry=source_registry(ground_truth_allowed=False), + ) + reference = governed_dataset(role="reference") + pair = MODULE.training_pair_evidence(raster=raster, reference=reference) + manifest_path = tmp_path / "operator_samples_manifest.json" + manifest_path.write_text( + json.dumps( + { + "immutable": True, + "training_eligibility": { + "policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION, + "status": "eligible", + "fixture_mode": False, + }, + "samples": [{"sample_slug": "governed", "training_eligibility": pair}], + } + ), + encoding="utf-8", + ) + (tmp_path / "corpus-freeze.json").write_text( + json.dumps( + { + "schema_version": 2, + "manifest_sha256": hashlib.sha256(manifest_path.read_bytes()).hexdigest(), + "immutable": True, + "training_eligibility_policy": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION, + "fixture_mode": False, + } + ), + encoding="utf-8", + ) + + assert MODULE.frozen_manifest_training_eligibility_failures(manifest_path) == [] + manifest_path.write_text(manifest_path.read_text(encoding="utf-8") + "\n", encoding="utf-8") + assert "corpus_manifest_checksum_mismatch" in MODULE.frozen_manifest_training_eligibility_failures( + manifest_path + ) + + +def test_live_manifest_gate_revokes_a_frozen_pair_when_an_upstream_dataset_is_quarantined() -> None: + raster_id = uuid4() + reference_id = uuid4() + raster_registry = source_registry(ground_truth_allowed=False) + reference_registry = source_registry() + raster_snapshot = source_snapshot() + reference_snapshot = source_snapshot() + raster_snapshot.source_registry_id = raster_registry.id + reference_snapshot.source_registry_id = reference_registry.id + raster = governed_dataset( + role="raster", + id=raster_id, + source_registry=raster_registry, + source_snapshot=raster_snapshot, + source_registry_id=raster_registry.id, + source_snapshot_id=raster_snapshot.id, + ) + reference = governed_dataset( + role="reference", + id=reference_id, + source_registry=reference_registry, + source_snapshot=reference_snapshot, + source_registry_id=reference_registry.id, + source_snapshot_id=reference_snapshot.id, + ) + pair = MODULE.training_pair_evidence(raster=raster, reference=reference) + manifest = { + "training_eligibility": { + "policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION, + "status": "eligible", + "fixture_mode": False, + }, + "samples": [ + { + "sample_slug": "governed-aoi", + "raster_dataset_id": str(raster_id), + "reference_dataset_id": str(reference_id), + "training_eligibility": pair, + } + ], + } + + class DatasetModel: + pass + + class Session: + def __init__(self) -> None: + self.closed = False + + @staticmethod + def get(model, item_id): + assert model is DatasetModel + return {raster_id: raster, reference_id: reference}.get(item_id) + + def close(self) -> None: + self.closed = True + + assert MODULE.live_manifest_training_eligibility_failures( + manifest, + session_factory=Session, + dataset_model=DatasetModel, + ) == [] + + raster.quarantine_status = "quarantined" + failures = MODULE.live_manifest_training_eligibility_failures( + manifest, + session_factory=Session, + dataset_model=DatasetModel, + ) + + assert "governed-aoi:raster_live_revoked:dataset_quarantined" in failures diff --git a/backend/tests/test_training_release_manifest.py b/backend/tests/test_training_release_manifest.py new file mode 100644 index 00000000..c9408e77 --- /dev/null +++ b/backend/tests/test_training_release_manifest.py @@ -0,0 +1,414 @@ +from __future__ import annotations + +import hashlib +import importlib.util +import json +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPTS = ROOT / "scripts" +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) +SCRIPT = SCRIPTS / "training_release_manifest.py" +SPEC = importlib.util.spec_from_file_location("training_release_manifest", SCRIPT) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = MODULE +SPEC.loader.exec_module(MODULE) + + +def sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +@pytest.fixture(autouse=True) +def _synthetic_release_uses_a_static_live_registry_spy(monkeypatch: pytest.MonkeyPatch) -> None: + """Filesystem unit fixtures cannot resolve a real database, but must ask for it.""" + + original_assert = MODULE.assert_frozen_manifest_training_eligible + original_failures = MODULE.frozen_manifest_training_eligibility_failures + + def static_assert(*args, **kwargs): + assert kwargs.get("verify_live") is True + kwargs["verify_live"] = False + return original_assert(*args, **kwargs) + + def static_failures(*args, **kwargs): + assert kwargs.get("verify_live") is True + kwargs["verify_live"] = False + return original_failures(*args, **kwargs) + + monkeypatch.setattr(MODULE, "assert_frozen_manifest_training_eligible", static_assert) + monkeypatch.setattr(MODULE, "frozen_manifest_training_eligibility_failures", static_failures) + + +def write_corpus_manifest(tmp_path: Path, *, fixture_mode: bool = False) -> Path: + policy = "geointel-training-source-eligibility/v1" + def pair(sample_slug: str) -> dict: + raster_id = f"dataset:raster:{sample_slug}" + reference_id = f"dataset:reference:{sample_slug}" + return { + "policy_version": policy, + "eligible": True, + "fixture_mode": fixture_mode, + "raster": { + "eligible": True, + "reasons": [], + "evidence": { + "dataset_id": raster_id, + "checksum_sha256": "a" * 64, + "source_registry_id": "registry:orthophoto", + "source_snapshot_id": "snapshot:orthophoto", + }, + }, + "reference": { + "eligible": True, + "reasons": [], + "evidence": { + "dataset_id": reference_id, + "checksum_sha256": "b" * 64, + "source_registry_id": "registry:grb", + "source_snapshot_id": "snapshot:grb", + }, + }, + } + + samples = [] + for sample_slug, split in (("fixture-train", "train"), ("fixture-val", "val")): + samples.append( + { + "sample_slug": sample_slug, + "split": split, + "raster_dataset_id": f"dataset:raster:{sample_slug}", + "reference_dataset_id": f"dataset:reference:{sample_slug}", + "training_eligibility": pair(sample_slug), + } + ) + manifest = tmp_path / "operator_samples_manifest.json" + manifest.write_text( + json.dumps( + { + "immutable": True, + "training_eligibility": { + "policy_version": policy, + "status": "eligible", + "fixture_mode": fixture_mode, + }, + "samples": samples, + } + ), + encoding="utf-8", + ) + (tmp_path / "corpus-freeze.json").write_text( + json.dumps( + { + "schema_version": 2, + "immutable": True, + "fixture_mode": fixture_mode, + "training_eligibility_policy": policy, + "manifest_sha256": sha256(manifest), + } + ), + encoding="utf-8", + ) + return manifest + + +def write_yolo_dataset(tmp_path: Path, *, empty_train_label: bool = False) -> Path: + dataset_root = tmp_path / "dataset" + for split, sample_slug in (("train", "fixture-train"), ("val", "fixture-val")): + image = dataset_root / "images" / split / f"{sample_slug}.png" + label = dataset_root / "labels" / split / f"{sample_slug}.txt" + image.parent.mkdir(parents=True, exist_ok=True) + label.parent.mkdir(parents=True, exist_ok=True) + image.write_bytes(f"{split}-image".encode("utf-8")) + label.write_text("" if split == "train" and empty_train_label else "0 0.5 0.5 0.2 0.2\n", encoding="utf-8") + yaml_path = dataset_root / "dataset.yaml" + yaml_path.write_text( + f"path: {dataset_root}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n", + encoding="utf-8", + ) + return yaml_path + + +def write_accepted_review_audit(tmp_path: Path, corpus_manifest: Path) -> Path: + artifacts = {} + for sample_slug in ("fixture-train", "fixture-val"): + artifact = tmp_path / f"{sample_slug}-contact-sheet.png" + artifact.write_bytes(f"reviewed {sample_slug}".encode("utf-8")) + artifacts[sample_slug] = artifact + decisions = tmp_path / "review-decisions.json" + decisions.write_text( + json.dumps( + { + "decisions": [ + { + "sample_slug": sample_slug, + "decision": "accepted", + "reviewer": "reviewer@example.test", + "reviewed_at": "2026-08-01T12:00:00+00:00", + "reviewed_artifact_path": str(artifact.resolve()), + "reviewed_artifact_sha256": sha256(artifact), + } + for sample_slug, artifact in artifacts.items() + ] + } + ), + encoding="utf-8", + ) + evidence = { + "review_decisions_path": str(decisions.resolve()), + "review_decisions_sha256": sha256(decisions), + "required_sample_count": 2, + "accepted_sample_count": 2, + "accepted_sample_slugs": ["fixture-train", "fixture-val"], + "reviewer_ids": ["reviewer@example.test"], + "reviewed_at_by_sample": { + "fixture-train": "2026-08-01T12:00:00+00:00", + "fixture-val": "2026-08-01T12:00:00+00:00", + }, + "reviewed_artifact_path_by_sample": { + sample_slug: str(artifact.resolve()) for sample_slug, artifact in artifacts.items() + }, + "reviewed_artifact_sha256_by_sample": { + sample_slug: sha256(artifact) for sample_slug, artifact in artifacts.items() + }, + } + audit = tmp_path / "belgium-building-corpus-audit.json" + audit.write_text( + json.dumps( + { + "status": "ok", + "manifest_immutable": True, + "spatial_leakage_status": "ok", + "review_complete": True, + "corpus_manifest_path": str(corpus_manifest.resolve()), + "corpus_manifest_sha256": sha256(corpus_manifest), + "human_review_evidence": evidence, + } + ), + encoding="utf-8", + ) + return audit + + +def test_valid_operational_release_binds_yaml_assets_corpus_and_human_review(tmp_path: Path) -> None: + corpus_manifest = write_corpus_manifest(tmp_path) + yaml_path = write_yolo_dataset(tmp_path) + review_audit = write_accepted_review_audit(tmp_path, corpus_manifest) + + paths = MODULE.create_training_release_manifest( + train_yaml=yaml_path, + corpus_manifest=corpus_manifest, + review_audit_path=review_audit, + ) + + assert all(path.is_file() for path in paths.values()) + assert MODULE.training_release_failures( + train_yaml=yaml_path, + corpus_manifest=corpus_manifest, + ) == [] + + +def test_tile_summary_must_be_an_exact_view_of_the_live_verified_release(tmp_path: Path) -> None: + corpus_manifest = write_corpus_manifest(tmp_path) + yaml_path = write_yolo_dataset(tmp_path) + review_audit = write_accepted_review_audit(tmp_path, corpus_manifest) + paths = MODULE.create_training_release_manifest( + train_yaml=yaml_path, + corpus_manifest=corpus_manifest, + review_audit_path=review_audit, + ) + assets = json.loads(paths["asset_manifest"].read_text(encoding="utf-8")) + summary = yaml_path.parent / "yolo_tile_dataset_summary.json" + summary.write_text( + json.dumps( + { + "dataset_yaml": str(yaml_path.resolve()), + "training_release_manifest": str(paths["release_manifest"].resolve()), + "training_release_manifest_sha256": sha256(paths["release_manifest"]), + "training_asset_manifest": str(paths["asset_manifest"].resolve()), + "source_manifest_sha256": sha256(corpus_manifest), + "tiles": [ + { + "split": entry["split"], + "image_path": entry["image_path"], + "label_path": entry["label_path"], + } + for entry in assets["entries"] + ], + } + ), + encoding="utf-8", + ) + + MODULE.assert_yolo_summary_bound_to_training_release( + summary_path=summary, + train_yaml=yaml_path, + corpus_manifest=corpus_manifest, + ) + payload = json.loads(summary.read_text(encoding="utf-8")) + payload["tiles"] = payload["tiles"][:1] + summary.write_text(json.dumps(payload), encoding="utf-8") + + with pytest.raises(MODULE.TrainingReleaseError, match="complete immutable view"): + MODULE.assert_yolo_summary_bound_to_training_release( + summary_path=summary, + train_yaml=yaml_path, + corpus_manifest=corpus_manifest, + ) + + +def test_unbound_or_changed_yaml_is_rejected_before_training(tmp_path: Path) -> None: + corpus_manifest = write_corpus_manifest(tmp_path) + yaml_path = write_yolo_dataset(tmp_path) + review_audit = write_accepted_review_audit(tmp_path, corpus_manifest) + + assert MODULE.training_release_failures(train_yaml=yaml_path) == ["training_release_manifest_missing"] + MODULE.create_training_release_manifest( + train_yaml=yaml_path, + corpus_manifest=corpus_manifest, + review_audit_path=review_audit, + ) + yaml_path.write_text(yaml_path.read_text(encoding="utf-8") + "# tampered\n", encoding="utf-8") + + assert "training_release_yaml_checksum_mismatch" in MODULE.training_release_failures( + train_yaml=yaml_path, + corpus_manifest=corpus_manifest, + ) + + +def test_changed_label_asset_is_rejected_even_when_yaml_bytes_are_unchanged(tmp_path: Path) -> None: + corpus_manifest = write_corpus_manifest(tmp_path) + yaml_path = write_yolo_dataset(tmp_path) + review_audit = write_accepted_review_audit(tmp_path, corpus_manifest) + MODULE.create_training_release_manifest( + train_yaml=yaml_path, + corpus_manifest=corpus_manifest, + review_audit_path=review_audit, + ) + label = yaml_path.parent / "labels" / "train" / "fixture-train.txt" + label.write_text("0 0.4 0.4 0.2 0.2\n", encoding="utf-8") + + failures = MODULE.training_release_failures( + train_yaml=yaml_path, + corpus_manifest=corpus_manifest, + ) + assert "training_release_asset_manifest_content_mismatch" in failures + + +def test_operational_release_requires_complete_accepted_human_review(tmp_path: Path) -> None: + corpus_manifest = write_corpus_manifest(tmp_path) + yaml_path = write_yolo_dataset(tmp_path) + incomplete_audit = tmp_path / "audit.json" + incomplete_audit.write_text( + json.dumps( + { + "status": "needs_human_review", + "manifest_immutable": True, + "spatial_leakage_status": "ok", + "review_complete": False, + } + ), + encoding="utf-8", + ) + + try: + MODULE.create_training_release_manifest( + train_yaml=yaml_path, + corpus_manifest=corpus_manifest, + review_audit_path=incomplete_audit, + ) + except MODULE.TrainingReleaseError as exc: + assert "review_complete_not_true" in str(exc) + assert "accepted_human_review_evidence_missing" in str(exc) + else: + raise AssertionError("operational release accepted an incomplete human review") + + +def test_operational_release_rejects_tampered_accepted_review_artifact(tmp_path: Path) -> None: + corpus_manifest = write_corpus_manifest(tmp_path) + yaml_path = write_yolo_dataset(tmp_path) + review_audit = write_accepted_review_audit(tmp_path, corpus_manifest) + artifact = tmp_path / "fixture-train-contact-sheet.png" + artifact.write_bytes(b"changed after review") + + try: + MODULE.create_training_release_manifest( + train_yaml=yaml_path, + corpus_manifest=corpus_manifest, + review_audit_path=review_audit, + ) + except MODULE.TrainingReleaseError as exc: + assert "reviewed_artifact_checksum_mismatch" in str(exc) + else: + raise AssertionError("tampered human-review artifact was accepted") + + +def test_fixture_relaxation_requires_explicit_fixture_corpus_and_is_not_operational(tmp_path: Path) -> None: + fixture_manifest = write_corpus_manifest(tmp_path, fixture_mode=True) + yaml_path = write_yolo_dataset(tmp_path) + MODULE.create_training_release_manifest( + train_yaml=yaml_path, + corpus_manifest=fixture_manifest, + fixture_mode=True, + ) + + assert MODULE.training_release_failures( + train_yaml=yaml_path, + corpus_manifest=fixture_manifest, + fixture_mode=True, + ) == [] + assert "training_release_fixture_mode_mismatch" in MODULE.training_release_failures( + train_yaml=yaml_path, + corpus_manifest=fixture_manifest, + fixture_mode=False, + ) + + +def test_release_contracts_a_reviewed_empty_label_as_explicit_pure_background(tmp_path: Path) -> None: + corpus_manifest = write_corpus_manifest(tmp_path) + yaml_path = write_yolo_dataset(tmp_path, empty_train_label=True) + review_audit = write_accepted_review_audit(tmp_path, corpus_manifest) + + paths = MODULE.create_training_release_manifest( + train_yaml=yaml_path, + corpus_manifest=corpus_manifest, + review_audit_path=review_audit, + ) + + labels = json.loads(paths["label_contract_manifest"].read_text(encoding="utf-8")) + assert labels["counts"]["pure_background"] == 1 + assert MODULE.training_release_failures(train_yaml=yaml_path, corpus_manifest=corpus_manifest) == [] + + +def test_empty_label_without_accepted_sample_review_cannot_become_a_background_negative(tmp_path: Path) -> None: + corpus_manifest = write_corpus_manifest(tmp_path) + yaml_path = write_yolo_dataset(tmp_path, empty_train_label=True) + review_audit = write_accepted_review_audit(tmp_path, corpus_manifest) + audit_payload = json.loads(review_audit.read_text(encoding="utf-8")) + evidence = audit_payload["human_review_evidence"] + evidence["accepted_sample_slugs"] = ["fixture-val"] + review_audit.write_text(json.dumps(audit_payload), encoding="utf-8") + review = { + "status": "accepted", + "fixture_only": False, + "review_complete": True, + "evidence": evidence, + } + + try: + MODULE.build_training_label_contract_manifest( + corpus_manifest_path=corpus_manifest, + asset_manifest=MODULE.build_yolo_asset_manifest(yaml_path), + review=review, + fixture_mode=False, + ) + except MODULE.TrainingReleaseError as exc: + assert "Pure-background label sample was not accepted by review" in str(exc) + else: + raise AssertionError("unreviewed empty label was accepted as a background negative") diff --git a/backend/tests/test_vector_operations_service.py b/backend/tests/test_vector_operations_service.py index 1f7fd42e..eb8377ff 100644 --- a/backend/tests/test_vector_operations_service.py +++ b/backend/tests/test_vector_operations_service.py @@ -1,7 +1,6 @@ from __future__ import annotations from pathlib import Path -from types import SimpleNamespace from uuid import UUID, uuid4 from app.core.errors import AppError diff --git a/deploy/unraid/Dockerfile.all-in-one b/deploy/unraid/Dockerfile.all-in-one index 43a31706..52d83233 100644 --- a/deploy/unraid/Dockerfile.all-in-one +++ b/deploy/unraid/Dockerfile.all-in-one @@ -128,6 +128,8 @@ COPY scripts/evaluate_belgium_building_candidate.py /app/scripts/evaluate_belgiu COPY scripts/assess_belgium_building_training_iteration.py /app/scripts/assess_belgium_building_training_iteration.py COPY scripts/run_belgium_building_training_loop.py /app/scripts/run_belgium_building_training_loop.py COPY scripts/build_failure_driven_yolo_sampling.py /app/scripts/build_failure_driven_yolo_sampling.py +COPY scripts/training_dataset_eligibility.py /app/scripts/training_dataset_eligibility.py +COPY scripts/training_release_manifest.py /app/scripts/training_release_manifest.py COPY scripts/build_grayscale_yolo_dataset.py /app/scripts/build_grayscale_yolo_dataset.py COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py diff --git a/docs/API_CONTRACTS.md b/docs/API_CONTRACTS.md index f997c8bd..d17b1eea 100644 --- a/docs/API_CONTRACTS.md +++ b/docs/API_CONTRACTS.md @@ -393,15 +393,48 @@ Fields: - `file`: dataset file. - `dataset_type`: `vector`, `geojson` (legacy), `raster`. -- `source`: free text, e.g. `user_upload`, `grb`, `osm`. +- `source`: descriptive caller text, e.g. `user_upload`. It is retained as a + claim only and never establishes authority. - `dataset_role`: `source`, `derived`, or `reference` (default `source`). -- `source_name`: optional source identity, e.g. `manual`, `grb`, `osm`; reference uploads default to `manual` when omitted. +- `source_name`: optional descriptive claim. Public uploads are always bound + to the server-owned `manual` registry entry, including when this field says + `grb`, `osm` or another official name. Reference uploads also default to + `manual`. - `reference_layer_name`: optional reference layer label, e.g. `buildings`; only retained for reference datasets. - `area_id`: optional. -Response: `DatasetRead` with extracted metadata if supported. +Response: `DatasetRead` with extracted metadata if supported, plus +`source_registry_id`, `source_snapshot_id`, exact data-contract key/version, +validation report/status, provenance/lineage status, quarantine status and an +idempotent ingest key. A malformed or doubtful artifact is retained as +`status=quarantined`; it is not silently discarded or made ready. -Vector uploads remain stored as original files and are also persisted into `vector_features` as queryable PostGIS state. +Vector uploads remain stored as original files and are also persisted into +`vector_features` as queryable PostGIS state only after their contract passes. +Non-EPSG:4326 vector coordinates are explicitly transformed before canonical +feature persistence; relabelling Lambert coordinates as EPSG:4326 is rejected. + +### GET `/api/v1/source-registry` + +Lists server-owned source definitions. Optional query parameter +`classification` is one of `authoritative`, `corroborative`, `contextual`, +`derived`, or `experimental`. This endpoint reports policy and snapshot counts; +it does not assert that historical datasets carrying a matching text field are +trusted. It is operator-only because source snapshots can contain +operator-acquisition provenance. A demo session uses its project-scoped +dataset provenance endpoint instead. + +### GET `/api/v1/source-registry/{source_key}` + +Returns one source definition and its immutable snapshots, including authority +scope, licence/restrictions, expected geometry/attributes, freshness and known +limitations. Unknown source keys return `SOURCE_REGISTRY_ENTRY_NOT_FOUND`. + +### GET `/api/v1/projects/{project_id}/datasets/{dataset_id}/provenance` + +Returns the dataset's bound source/snapshot, contract report, lineage edges and +quarantine records. A missing binding is visible as incomplete provenance; it +is never backfilled from a display name or caller metadata. ### GET `/api/v1/projects/{project_id}/datasets/orthophoto/products` diff --git a/docs/DATABASE_IMPLEMENTATION_PLAN.md b/docs/DATABASE_IMPLEMENTATION_PLAN.md index e1fbf637..afc3f68e 100644 --- a/docs/DATABASE_IMPLEMENTATION_PLAN.md +++ b/docs/DATABASE_IMPLEMENTATION_PLAN.md @@ -69,6 +69,41 @@ Spatial index required on `geometry`. - `status text default 'created'` - `created_at timestamptz` +### Source registry, snapshots and quarantine (Accuracy Phase 2) + +Migration `202608010001_source_registry_provenance` adds a server-owned +provenance boundary without deleting or inventing facts for existing rows. + +`source_registry` stores one governed source definition per `source_key`: + +- classification (`authoritative`, `corroborative`, `contextual`, `derived`, + `experimental`), authority/scope and provider adapter key; +- licence, usage restrictions, expected CRS/units/resolution, temporal and + geographic coverage, expected geometry/attributes and known limitations; +- usage policy, freshness and ingest status. + +`source_snapshots` stores immutable acquired editions keyed by +`(source_registry_id, snapshot_key)`, including source version/snapshot/fetch +time, lowercase SHA-256 checksum, CRS/units/resolution, coverage, observed +schema, freshness, ingest status and raw snapshot metadata. + +`datasets` and `dataset_versions` receive `source_registry_id`, +`source_snapshot_id`, data-contract key/version, validation report/status, +provenance/lineage status and idempotent `ingest_key`. New governed records +must have a passing report before they become `ready`. Legacy rows are retained +with explicit incomplete/not-validated state; a migration never promotes them +based on a historical `source_name` string. + +`dataset_lineage_edges` records parent/child Dataset(+Version), transform name +and version, parameters and input/output hashes. `dataset_quarantines` retains +the rejected dataset/version/snapshot, stage, reason, validation evidence and +artifact location. Quarantine is a stateful preservation record, not a delete. + +The database constraints enforce accepted status vocabularies, nonblank ingest +keys, immutable snapshot identity/checksum format, unique ingest idempotency +keys and non-self lineage edges. Service-level contract validation remains +responsible for CRS, topology, bbox, units and source-task semantics. + ### vector_features Used for imported vector datasets and derived vector outputs when feature-level storage is needed. Original files remain source artifacts; this table is the queryable PostGIS state for vector features. diff --git a/scripts/assemble_belgium_building_corpus.py b/scripts/assemble_belgium_building_corpus.py index b03a3b3d..a1fcc3b7 100644 --- a/scripts/assemble_belgium_building_corpus.py +++ b/scripts/assemble_belgium_building_corpus.py @@ -26,6 +26,10 @@ from app.db.session import SessionLocal # noqa: E402 from app.models import Dataset # noqa: E402 from normalize_belgium_building_labels import normalize # noqa: E402 +from training_dataset_eligibility import ( # noqa: E402 + TRAINING_ELIGIBILITY_POLICY_VERSION, + training_pair_evidence, +) REGION_SOURCES = { "flanders": ({"digitaal_vlaanderen_orthophoto"}, "grb"), @@ -52,7 +56,13 @@ def _dataset_path(dataset: Dataset) -> Path: return path -def _validate_pair(sample: dict[str, Any], raster: Dataset, reference: Dataset) -> tuple[str, str]: +def _validate_pair( + sample: dict[str, Any], + raster: Dataset, + reference: Dataset, + *, + fixture_mode: bool, +) -> tuple[str, str, dict[str, Any]]: region = str(sample.get("region") or "").lower() if region not in REGION_SOURCES: raise SystemExit(f"Unsupported region for {sample.get('sample_slug')}: {region}") @@ -66,7 +76,23 @@ def _validate_pair(sample: dict[str, Any], raster: Dataset, reference: Dataset) raise SystemExit(f"Unsupported split for {sample['sample_slug']}: {split}") if raster.status != "ready" or reference.status != "ready": raise SystemExit(f"Dataset pair is not ready for {sample['sample_slug']}") - return region, expected_reference + eligibility = training_pair_evidence( + raster=raster, + reference=reference, + fixture_mode=fixture_mode, + ) + if not eligibility["eligible"]: + reasons = sorted( + { + reason + for role in ("raster", "reference") + for reason in eligibility[role]["reasons"] + } + ) + raise SystemExit( + f"Dataset pair is not eligible for training for {sample['sample_slug']}: {', '.join(reasons)}" + ) + return region, expected_reference, eligibility def audit_spatial_leakage(samples: list[dict[str, Any]], buffer_m: float = 64.0) -> dict[str, Any]: @@ -104,6 +130,14 @@ def main() -> int: parser.add_argument("--min-label-px", type=float, default=3.0) parser.add_argument("--merge-touching-roofs", action="store_true") parser.add_argument("--freeze", action="store_true") + parser.add_argument( + "--fixture-mode", + action="store_true", + help=( + "Allow only explicitly marked fixture datasets with legacy provenance. " + "Never use this mode for an operational corpus." + ), + ) args = parser.parse_args() spec = json.loads(args.spec.read_text(encoding="utf-8-sig")) @@ -129,7 +163,12 @@ def main() -> int: reference = db.get(Dataset, UUID(str(sample["reference_dataset_id"]))) if raster is None or reference is None: raise SystemExit(f"Persisted Dataset pair not found for {slug}") - region, reference_source = _validate_pair(sample, raster, reference) + region, reference_source, eligibility = _validate_pair( + sample, + raster, + reference, + fixture_mode=args.fixture_mode, + ) raster_source = _dataset_path(raster) reference_source_path = _dataset_path(reference) sample_dir = pairs_dir / slug @@ -173,14 +212,20 @@ def main() -> int: "raster_sha256": sha256(raster_target), "reference_sha256": sha256(normalized_target), "label_audit_sha256": sha256(audit_target), + "training_eligibility": eligibility, "bbox_epsg4326": (raster.source_metadata or {}).get("bbox_epsg4326") or sample.get("bbox_epsg4326"), } ) manifest = { - "schema_version": 1, + "schema_version": 2, "dataset_version": args.version, "immutable": bool(args.freeze), + "training_eligibility": { + "policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION, + "status": "eligible", + "fixture_mode": bool(args.fixture_mode), + }, "samples": manifest_samples, } manifest_path = output_dir / "operator_samples_manifest.json" @@ -192,10 +237,13 @@ def main() -> int: if leakage_audit["status"] != "ok": raise SystemExit("Spatial split leakage audit failed") freeze = { + "schema_version": 2, "dataset_version": args.version, "manifest_sha256": sha256(manifest_path), "sample_count": len(manifest_samples), "immutable": bool(args.freeze), + "training_eligibility_policy": TRAINING_ELIGIBILITY_POLICY_VERSION, + "fixture_mode": bool(args.fixture_mode), } (output_dir / "corpus-freeze.json").write_text(json.dumps(freeze, indent=2), encoding="utf-8") print(json.dumps(freeze)) diff --git a/scripts/audit_belgium_building_corpus.py b/scripts/audit_belgium_building_corpus.py index 3586425a..d167cd38 100644 --- a/scripts/audit_belgium_building_corpus.py +++ b/scripts/audit_belgium_building_corpus.py @@ -4,13 +4,49 @@ from __future__ import annotations import argparse +import hashlib import json +import sys from collections import Counter +from datetime import datetime from pathlib import Path from typing import Any +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from training_dataset_eligibility import frozen_manifest_training_eligibility_failures # noqa: E402 + REQUIRED_SPLITS = ("train", "val", "calibration", "test", "background-test") REGIONS = ("flanders", "wallonia", "brussels") +SHA256_LENGTH = 64 + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def valid_review_timestamp(value: Any) -> bool: + if not isinstance(value, str) or not value.strip(): + return False + try: + parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00")) + except ValueError: + return False + return parsed.tzinfo is not None + + +def valid_sha256(value: Any) -> bool: + return ( + isinstance(value, str) + and len(value) == SHA256_LENGTH + and all(character in "0123456789abcdefABCDEF" for character in value) + ) def main() -> int: @@ -19,8 +55,15 @@ def main() -> int: parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--review-decisions", type=Path) args = parser.parse_args() - manifest = json.loads((args.corpus_dir / "operator_samples_manifest.json").read_text(encoding="utf-8")) + manifest_path = args.corpus_dir / "operator_samples_manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) leakage = json.loads((args.corpus_dir / "spatial-leakage-audit.json").read_text(encoding="utf-8")) + manifest_policy = manifest.get("training_eligibility") + fixture_mode = bool(manifest_policy.get("fixture_mode")) if isinstance(manifest_policy, dict) else False + eligibility_failures = frozen_manifest_training_eligibility_failures( + manifest_path, + fixture_mode=fixture_mode, + ) samples = manifest["samples"] split_counts = Counter((sample["region"], sample["split"]) for sample in samples) decision_counts: Counter[str] = Counter() @@ -28,7 +71,7 @@ def main() -> int: total_input = 0 total_accepted = 0 temporal_unknown = 0 - failures: list[str] = [] + failures: list[str] = list(eligibility_failures) for region in REGIONS: for split in REQUIRED_SPLITS: minimum = 4 if split == "train" else 2 @@ -62,8 +105,12 @@ def main() -> int: failures.append("spatial leakage audit failed") reviewed = 0 review_complete = False + review_evidence_failures: list[str] = [] + accepted_review_evidence: dict[str, dict[str, Any]] = {} + review_decisions_path: Path | None = None if args.review_decisions and args.review_decisions.is_file(): - decisions = json.loads(args.review_decisions.read_text(encoding="utf-8")) + review_decisions_path = args.review_decisions.resolve(strict=False) + decisions = json.loads(review_decisions_path.read_text(encoding="utf-8")) by_slug = {item["sample_slug"]: item for item in decisions.get("decisions", [])} for item in review_queue: decision = by_slug.get(item["sample_slug"]) @@ -71,13 +118,70 @@ def main() -> int: item["decision"] = decision.get("decision") item["reviewer"] = decision.get("reviewer") item["notes"] = decision.get("notes") - if item["decision"] in {"accepted", "rejected"} and item.get("reviewer"): + item["reviewed_at"] = decision.get("reviewed_at") + item["reviewed_artifact_path"] = decision.get("reviewed_artifact_path") + item["reviewed_artifact_sha256"] = decision.get("reviewed_artifact_sha256") + reviewed_artifact = ( + Path(item["reviewed_artifact_path"]).expanduser().resolve(strict=False) + if isinstance(item.get("reviewed_artifact_path"), str) + and item["reviewed_artifact_path"].strip() + else None + ) + has_evidence = ( + item["decision"] == "accepted" + and isinstance(item.get("reviewer"), str) + and bool(item["reviewer"].strip()) + and valid_review_timestamp(item.get("reviewed_at")) + and isinstance(item.get("reviewed_artifact_path"), str) + and bool(item["reviewed_artifact_path"].strip()) + and valid_sha256(item.get("reviewed_artifact_sha256")) + and reviewed_artifact is not None + and reviewed_artifact.is_file() + and item["reviewed_artifact_sha256"] == sha256(reviewed_artifact) + ) + if has_evidence: reviewed += 1 + accepted_review_evidence[str(item["sample_slug"])] = { + "reviewer": item["reviewer"].strip(), + "reviewed_at": item["reviewed_at"], + "reviewed_artifact_path": item["reviewed_artifact_path"], + "reviewed_artifact_sha256": item["reviewed_artifact_sha256"], + } + elif item["decision"] in {"accepted", "rejected"}: + review_evidence_failures.append( + f"{item['sample_slug']}:accepted review lacks reviewer/timestamp/artifact evidence" + ) review_complete = reviewed == len(review_queue) and all(item["decision"] == "accepted" for item in review_queue) + human_review_evidence: dict[str, Any] | None = None + if review_decisions_path is not None: + human_review_evidence = { + "review_decisions_path": str(review_decisions_path), + "review_decisions_sha256": sha256(review_decisions_path), + "required_sample_count": len(review_queue), + "accepted_sample_count": reviewed, + "accepted_sample_slugs": sorted(accepted_review_evidence), + "reviewer_ids": sorted( + {item["reviewer"] for item in accepted_review_evidence.values()} + ), + "reviewed_at_by_sample": { + slug: accepted_review_evidence[slug]["reviewed_at"] + for slug in sorted(accepted_review_evidence) + }, + "reviewed_artifact_path_by_sample": { + slug: accepted_review_evidence[slug]["reviewed_artifact_path"] + for slug in sorted(accepted_review_evidence) + }, + "reviewed_artifact_sha256_by_sample": { + slug: accepted_review_evidence[slug]["reviewed_artifact_sha256"] + for slug in sorted(accepted_review_evidence) + }, + } status = "failed" if failures else ("ok" if review_complete else "needs_human_review") report = { "status": status, "dataset_version": manifest["dataset_version"], + "corpus_manifest_path": str(manifest_path.resolve(strict=False)), + "corpus_manifest_sha256": sha256(manifest_path), "manifest_immutable": manifest["immutable"], "sample_count": len(samples), "split_counts": {f"{region}/{split}": split_counts[(region, split)] for region in REGIONS for split in REQUIRED_SPLITS}, @@ -86,8 +190,13 @@ def main() -> int: "decision_counts": dict(sorted(decision_counts.items())), "temporal_unknown_sample_count": temporal_unknown, "spatial_leakage_status": leakage.get("status"), + "training_eligibility_status": manifest_policy.get("status") if isinstance(manifest_policy, dict) else None, + "training_eligibility_fixture_mode": fixture_mode, + "training_eligibility_failures": eligibility_failures, "reviewed_sample_count": reviewed, "review_complete": review_complete, + "human_review_evidence": human_review_evidence, + "review_evidence_failures": sorted(set(review_evidence_failures)), "failures": failures, "review_queue": review_queue, } diff --git a/scripts/build_building_proposal_classifier_dataset.py b/scripts/build_building_proposal_classifier_dataset.py index eb5fa222..e2422a9d 100644 --- a/scripts/build_building_proposal_classifier_dataset.py +++ b/scripts/build_building_proposal_classifier_dataset.py @@ -9,7 +9,7 @@ import json import sys from collections import Counter from pathlib import Path -from typing import Any +from typing import Any, Mapping from PIL import Image @@ -17,10 +17,21 @@ SCRIPT_DIR = Path(__file__).resolve().parent if str(SCRIPT_DIR) not in sys.path: sys.path.insert(0, str(SCRIPT_DIR)) -from evaluate_belgium_building_candidate import iou, read_references +from evaluate_belgium_building_candidate import iou, read_references # noqa: E402 +from training_dataset_eligibility import ( # noqa: E402 + TrainingEligibilityError, + assert_frozen_manifest_training_eligible, +) +from training_release_manifest import ( # noqa: E402 + TrainingReleaseError, + assert_yolo_summary_bound_to_embedded_training_release, +) PROTECTED_SPLITS = {"calibration", "test", "background-test"} +PROPOSAL_DATASET_PROVENANCE_NAME = "proposal-dataset-provenance.json" +PROPOSAL_DATASET_EVIDENCE_NAME = "proposal-dataset-evidence.json" +PROPOSAL_DATASET_SCHEMA_VERSION = 1 def sha256(path: Path) -> str: @@ -31,6 +42,125 @@ def sha256(path: Path) -> str: return digest.hexdigest() +def _canonical_json_bytes(payload: Mapping[str, Any]) -> bytes: + return json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8") + + +def _immutable_payload_sha256(payload: Mapping[str, Any], *, field: str = "manifest_sha256") -> str: + normalized = dict(payload) + normalized.pop(field, None) + return hashlib.sha256(_canonical_json_bytes(normalized)).hexdigest() + + +def _write_immutable_json(path: Path, payload: Mapping[str, Any]) -> None: + """Persist one immutable sidecar without silently replacing prior evidence.""" + + encoded = json.dumps(dict(payload), ensure_ascii=False, indent=2, sort_keys=True) + "\n" + if path.exists(): + if path.read_text(encoding="utf-8") != encoded: + raise RuntimeError(f"immutable provenance artifact already exists with different content: {path}") + return + path.write_text(encoded, encoding="utf-8") + + +def assert_summary_source_manifest_binding(summary: Mapping[str, Any], corpus_manifest_path: Path) -> str: + """Require the proposal summary to name the exact frozen corpus bytes.""" + + expected = sha256(corpus_manifest_path) + observed = summary.get("source_manifest_sha256") + if observed != expected: + raise ValueError( + "proposal source summary is not bound to the supplied governed corpus manifest " + f"(expected {expected}, observed {observed!r})" + ) + return expected + + +def load_governed_corpus_manifest(corpus_manifest_path: Path, *, fixture_mode: bool) -> dict[str, Any]: + """Validate frozen corpus evidence and re-check the live Dataset state. + + This must run before importing Ultralytics/PyTorch so a revocation cannot + consume GPU work or create proposal crops. + """ + + try: + manifest = assert_frozen_manifest_training_eligible( + corpus_manifest_path, + fixture_mode=fixture_mode, + verify_live=True, + ) + except TrainingEligibilityError as exc: + raise ValueError(str(exc)) from exc + if not isinstance(manifest, dict): # pragma: no cover - defensive contract boundary + raise ValueError("governed corpus manifest must be a JSON object") + return manifest + + +def _source_file_evidence(tile: Mapping[str, Any], *, field: str) -> tuple[Path, str]: + raw_path = tile.get(field) + if not isinstance(raw_path, str) or not raw_path.strip(): + raise ValueError(f"proposal source tile has no {field}: {tile.get('sample_slug')!r}") + path = Path(raw_path).expanduser().resolve(strict=False) + if not path.is_file(): + raise ValueError(f"proposal source tile {field} is unavailable: {path}") + return path, sha256(path) + + +def _source_tile_key(tile: Mapping[str, Any]) -> tuple[str, str, str]: + """Return the canonical source identity used for the pre-inference cache.""" + + return ( + str(tile.get("sample_slug") or ""), + str(Path(str(tile.get("image_path") or "")).expanduser().resolve(strict=False)), + str(Path(str(tile.get("label_path") or "")).expanduser().resolve(strict=False)), + ) + + +def _assert_file_unchanged(path: Path, expected_sha256: str, *, role: str) -> None: + """Reject a mutable source changing between evidence capture and use.""" + + if sha256(path) != expected_sha256: + raise RuntimeError(f"proposal {role} changed after evidence capture: {path}") + + +def _crop_entry( + *, + output_dir: Path, + crop_path: Path, + tile: Mapping[str, Any], + label: str, + proposal_index: int, + proposal_score: float, + source_box_xyxy: tuple[float, float, float, float], + source_image_path: Path, + source_image_sha256: str, + source_label_path: Path, + source_label_sha256: str, +) -> dict[str, Any]: + relative_path = crop_path.resolve(strict=False).relative_to(output_dir.resolve(strict=False)).as_posix() + sample_slug = tile.get("sample_slug") + split = tile.get("split") + if not isinstance(sample_slug, str) or not sample_slug.strip(): + raise ValueError("proposal source tile has no sample_slug") + if split not in {"train", "val"}: + raise ValueError(f"proposal crop has unsupported split: {split!r}") + return { + "relative_path": relative_path, + "sha256": sha256(crop_path), + "size_bytes": crop_path.stat().st_size, + "split": split, + "label": label, + "sample_slug": sample_slug, + "proposal_index": proposal_index, + "proposal_score": proposal_score, + "source_box_xyxy": [float(value) for value in source_box_xyxy], + "source_image_path": str(source_image_path), + "source_image_sha256": source_image_sha256, + "source_label_path": str(source_label_path), + "source_label_sha256": source_label_sha256, + } + + def classify_proposals( predictions: list[tuple[tuple[float, float, float, float], float]], references: list[tuple[float, float, float, float]], @@ -99,29 +229,74 @@ def main() -> int: parser.add_argument("--max-negative-per-tile", type=int, default=24) parser.add_argument("--device", default="cuda:0") parser.add_argument("--imgsz", type=int, default=640) + parser.add_argument( + "--fixture-mode", + action="store_true", + help="Accept only an explicitly fixture-only corpus manifest; never use for operational training data.", + ) args = parser.parse_args() if args.output_dir.exists(): parser.error(f"output already exists: {args.output_dir}") - from ultralytics import YOLO - - summary = json.loads(args.summary.read_text(encoding="utf-8")) - manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8")) + try: + summary = json.loads(args.summary.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise SystemExit(f"proposal source summary is unreadable: {args.summary}") from exc + if not isinstance(summary, dict): + raise SystemExit("proposal source summary must be a JSON object") + try: + source_release = assert_yolo_summary_bound_to_embedded_training_release( + summary_path=args.summary, + corpus_manifest=args.corpus_manifest, + fixture_mode=args.fixture_mode, + ) + manifest = load_governed_corpus_manifest(args.corpus_manifest, fixture_mode=args.fixture_mode) + corpus_manifest_sha256 = assert_summary_source_manifest_binding(summary, args.corpus_manifest) + except (ValueError, TrainingReleaseError) as exc: + raise SystemExit(str(exc)) from exc tiles = eligible_tiles(summary, manifest, args.region) + model_path = args.model.expanduser().resolve(strict=False) + if not model_path.is_file(): + raise SystemExit(f"proposal model is unavailable: {model_path}") + model_sha256 = sha256(model_path) + + # Hash every input image/label before GPU inference. A crop sidecar then + # binds each generated JPEG to the exact source tile rather than its + # filename or an unverified aggregate summary. + source_evidence: dict[tuple[str, str, str], tuple[Path, str, Path, str]] = {} + for tile in tiles: + image_path, image_sha256 = _source_file_evidence(tile, field="image_path") + label_path, label_sha256 = _source_file_evidence(tile, field="label_path") + key = _source_tile_key(tile) + source_evidence[key] = (image_path, image_sha256, label_path, label_sha256) + args.output_dir.mkdir(parents=True) counts: Counter[str] = Counter() sample_counts: Counter[str] = Counter() - model = YOLO(str(args.model)) + crop_entries: list[dict[str, Any]] = [] + + # Importing the model is deliberately after the source-manifest and live + # Dataset checks above. A revoked corpus must not start GPU work. + from ultralytics import YOLO + + model = YOLO(str(model_path)) for start in range(0, len(tiles), 16): batch_tiles = tiles[start : start + 16] + batch_sources = [source_evidence[_source_tile_key(tile)] for tile in batch_tiles] + for image_path, image_sha256, label_path, label_sha256 in batch_sources: + _assert_file_unchanged(image_path, image_sha256, role="source image") + _assert_file_unchanged(label_path, label_sha256, role="source label") results = model.predict( - [tile["image_path"] for tile in batch_tiles], conf=args.confidence, + [str(source[0]) for source in batch_sources], conf=args.confidence, device=args.device, imgsz=args.imgsz, max_det=1000, iou=0.7, verbose=False, ) for tile, result in zip(batch_tiles, results, strict=True): - with Image.open(tile["image_path"]) as opened: + image_path, image_sha256, label_path, label_sha256 = source_evidence[_source_tile_key(tile)] + _assert_file_unchanged(image_path, image_sha256, role="source image") + _assert_file_unchanged(label_path, label_sha256, role="source label") + with Image.open(image_path) as opened: source = opened.convert("RGB") - references = read_references(Path(tile["label_path"]), source.width, source.height) + references = read_references(label_path, source.width, source.height) proposals = [ (tuple(map(float, box)), float(score)) for box, score in zip( @@ -139,25 +314,113 @@ def main() -> int: target_dir = args.output_dir / split / label target_dir.mkdir(parents=True, exist_ok=True) name = f"{tile['sample_slug']}__{Path(tile['image_path']).stem}__{proposal_index:04d}.jpg" - crop_square(source, box, args.crop_scale).save(target_dir / name, quality=92) + crop_path = target_dir / name + if crop_path.exists(): + raise RuntimeError(f"proposal crop identity collision: {crop_path}") + crop_square(source, box, args.crop_scale).save(crop_path, quality=92) + crop_entries.append( + _crop_entry( + output_dir=args.output_dir, + crop_path=crop_path, + tile=tile, + label=label, + proposal_index=proposal_index, + proposal_score=score, + source_box_xyxy=box, + source_image_path=image_path, + source_image_sha256=image_sha256, + source_label_path=label_path, + source_label_sha256=label_sha256, + ) + ) counts[f"{split}/{label}"] += 1 sample_counts[tile["sample_slug"]] += 1 for split in ("train", "val"): for label in ("positive", "negative"): if counts[f"{split}/{label}"] == 0: raise RuntimeError(f"empty proposal class: {split}/{label}") + + crop_entries.sort(key=lambda item: str(item["relative_path"])) + if len({str(item["relative_path"]) for item in crop_entries}) != len(crop_entries): + raise RuntimeError("proposal crop manifest contains duplicate relative paths") + source_summary_sha256 = sha256(args.summary) + corpus_freeze_path = args.corpus_manifest.parent / "corpus-freeze.json" + if not corpus_freeze_path.is_file(): # guarded above; retain a local invariant for provenance output + raise RuntimeError(f"governed corpus freeze is unavailable: {corpus_freeze_path}") + provenance: dict[str, Any] = { + "schema_version": PROPOSAL_DATASET_SCHEMA_VERSION, + "status": "ok", + "immutable": True, + "dataset_kind": "building_proposal_classifier_crops", + "fixture_mode": bool(args.fixture_mode), + "governed_corpus_live_recheck": True, + "source": { + "corpus_manifest": { + "path": str(args.corpus_manifest.expanduser().resolve(strict=False)), + "sha256": corpus_manifest_sha256, + }, + "corpus_freeze": { + "path": str(corpus_freeze_path.resolve(strict=False)), + "sha256": sha256(corpus_freeze_path), + }, + "summary": { + "path": str(args.summary.expanduser().resolve(strict=False)), + "sha256": source_summary_sha256, + "source_manifest_sha256": summary["source_manifest_sha256"], + "training_release_manifest": summary["training_release_manifest"], + "training_release_manifest_sha256": summary["training_release_manifest_sha256"], + "training_asset_manifest": summary["training_asset_manifest"], + }, + "training_release": { + "dataset_yaml_path": source_release["dataset_yaml"]["path"], + "dataset_yaml_sha256": source_release["dataset_yaml"]["sha256"], + "corpus_manifest_sha256": source_release["corpus"]["manifest_sha256"], + }, + "proposal_model": { + "path": str(model_path), + "sha256": model_sha256, + }, + }, + "parameters": { + "region": args.region, + "confidence": args.confidence, + "match_iou": args.match_iou, + "crop_scale": args.crop_scale, + "max_positive_per_tile": args.max_positive_per_tile, + "max_negative_per_tile": args.max_negative_per_tile, + "device": args.device, + "imgsz": args.imgsz, + }, + "counts": dict(sorted(counts.items())), + "sample_counts": dict(sorted(sample_counts.items())), + "tile_count": len(tiles), + "crop_count": len(crop_entries), + "crops_sha256": hashlib.sha256(_canonical_json_bytes({"crops": crop_entries})).hexdigest(), + "crops": crop_entries, + } + provenance["manifest_sha256"] = _immutable_payload_sha256(provenance) + provenance_path = args.output_dir / PROPOSAL_DATASET_PROVENANCE_NAME + _write_immutable_json(provenance_path, provenance) + evidence = { - "schema_version": 1, "status": "ok", "model": str(args.model), - "model_sha256": sha256(args.model), "summary": str(args.summary), - "summary_sha256": sha256(args.summary), "corpus_manifest": str(args.corpus_manifest), - "corpus_manifest_sha256": sha256(args.corpus_manifest), "region": args.region, + "schema_version": PROPOSAL_DATASET_SCHEMA_VERSION, + "status": "ok", + "model": str(model_path), + "model_sha256": model_sha256, + "summary": str(args.summary), + "summary_sha256": source_summary_sha256, + "corpus_manifest": str(args.corpus_manifest), + "corpus_manifest_sha256": corpus_manifest_sha256, + "region": args.region, "confidence": args.confidence, "match_iou": args.match_iou, "crop_scale": args.crop_scale, "counts": dict(sorted(counts.items())), "sample_counts": dict(sorted(sample_counts.items())), "protected_samples_in_training": [], "tile_count": len(tiles), + "fixture_mode": bool(args.fixture_mode), + "proposal_dataset_provenance": str(provenance_path), + "proposal_dataset_provenance_sha256": sha256(provenance_path), + "proposal_dataset_manifest_sha256": provenance["manifest_sha256"], } - (args.output_dir / "proposal-dataset-evidence.json").write_text( - json.dumps(evidence, indent=2), encoding="utf-8" - ) + _write_immutable_json(args.output_dir / PROPOSAL_DATASET_EVIDENCE_NAME, evidence) print(json.dumps(evidence, indent=2)) return 0 diff --git a/scripts/build_failure_driven_yolo_sampling.py b/scripts/build_failure_driven_yolo_sampling.py index b062fc2d..4c3c0848 100644 --- a/scripts/build_failure_driven_yolo_sampling.py +++ b/scripts/build_failure_driven_yolo_sampling.py @@ -8,10 +8,25 @@ import hashlib import json import math import re +import sys from collections import Counter from pathlib import Path from typing import Any +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from training_dataset_eligibility import ( # noqa: E402 + TrainingEligibilityError, + assert_frozen_manifest_training_eligible, +) +from training_release_manifest import ( # noqa: E402 + TrainingReleaseError, + assert_yolo_summary_bound_to_embedded_training_release, + create_training_release_manifest, +) + PRECISION_NEGATIVE_CONTEXTS = { "coastal-urban": {"port-hard-negative", "dunes-negative"}, @@ -80,6 +95,17 @@ def build_sampling( ) -> tuple[list[str], dict[str, Any]]: if assessment.get("status") != "continue_training_loop": raise ValueError("Failure-driven sampling requires a failed assessment") + protected_feedback = [ + role + for role in ("test", "background") + if assessment.get(role) is not None + ] + if protected_feedback: + raise ValueError( + "Failure-driven sampling is prohibited after protected " + + "/".join(protected_feedback) + + " evidence was opened" + ) if min( positive_repeat, negative_repeat, @@ -97,9 +123,9 @@ def build_sampling( samples = {item["sample_slug"]: item for item in manifest["samples"]} gates = assessment["gates"] - evaluation = assessment.get("test") or assessment.get("calibration") + evaluation = assessment.get("calibration") if not evaluation or "regions" not in evaluation: - raise ValueError("Assessment has no regional calibration or test evidence") + raise ValueError("Assessment has no regional calibration evidence") regions = evaluation["regions"] weak_recall_regions = { region @@ -246,7 +272,7 @@ def build_sampling( "schema_version": 1, "status": "ok", "strategy": "failed-region-positive-and-hard-negative-repeat", - "failure_evidence_source": "test" if assessment.get("test") else "calibration", + "failure_evidence_source": "calibration", "weak_recall_regions": sorted(weak_recall_regions), "weak_precision_regions": sorted(weak_precision_regions), "recall_dominant_regions": sorted(recall_dominant_regions), @@ -290,6 +316,11 @@ def main() -> int: parser.add_argument("--corpus-manifest", type=Path, required=True) parser.add_argument("--assessment", type=Path, required=True) parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument( + "--review-audit", + type=Path, + help="Passed corpus audit containing accepted human-review evidence for the frozen corpus.", + ) parser.add_argument("--positive-repeat", type=int, default=3) parser.add_argument("--negative-repeat", type=int, default=4) parser.add_argument("--precision-positive-repeat", type=int, default=1) @@ -299,8 +330,26 @@ def main() -> int: parser.add_argument("--sampling-round", type=int) parser.add_argument("--precision-guard-band", type=float, default=0.03) parser.add_argument("--recall-guard-band", type=float, default=0.03) + parser.add_argument( + "--fixture-mode", + action="store_true", + help="Accept only an explicitly fixture-only corpus manifest; never use for operational sampling.", + ) args = parser.parse_args() + try: + assert_frozen_manifest_training_eligible( + args.corpus_manifest, + fixture_mode=args.fixture_mode, + verify_live=True, + ) + source_release = assert_yolo_summary_bound_to_embedded_training_release( + summary_path=args.summary, + corpus_manifest=args.corpus_manifest, + fixture_mode=args.fixture_mode, + ) + except (TrainingEligibilityError, TrainingReleaseError) as exc: + raise SystemExit(str(exc)) from exc summary = json.loads(args.summary.read_text(encoding="utf-8")) manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8")) assessment = json.loads(args.assessment.read_text(encoding="utf-8")) @@ -325,7 +374,7 @@ def main() -> int: args.output_dir.mkdir(parents=True, exist_ok=True) train_list = args.output_dir / "train-failure-driven.txt" train_list.write_text("\n".join(paths) + "\n", encoding="utf-8") - source_yaml = args.summary.parent / "dataset.yaml" + source_yaml = Path(str(source_release["dataset_yaml"]["path"])) val_source = dataset_validation_source(source_yaml) dataset_yaml = args.output_dir / "dataset.yaml" dataset_yaml.write_text( @@ -335,6 +384,15 @@ def main() -> int: "names:\n 0: building\n", encoding="utf-8", ) + try: + release_paths = create_training_release_manifest( + train_yaml=dataset_yaml, + corpus_manifest=args.corpus_manifest, + review_audit_path=args.review_audit, + fixture_mode=args.fixture_mode, + ) + except TrainingReleaseError as exc: + raise SystemExit(str(exc)) from exc metadata.update( { "summary": str(args.summary), @@ -346,6 +404,11 @@ def main() -> int: "source_dataset_yaml": str(source_yaml), "train_list": str(train_list), "dataset_yaml": str(dataset_yaml), + "training_release_manifest": str(release_paths["release_manifest"]), + "training_release_manifest_sha256": file_sha256(release_paths["release_manifest"]), + "training_release_freeze": str(release_paths["release_freeze"]), + "training_asset_manifest": str(release_paths["asset_manifest"]), + "fixture_mode": bool(args.fixture_mode), } ) output = args.output_dir / "failure-driven-sampling.json" diff --git a/scripts/build_grayscale_yolo_dataset.py b/scripts/build_grayscale_yolo_dataset.py index 01ef3f18..cbce2b66 100644 --- a/scripts/build_grayscale_yolo_dataset.py +++ b/scripts/build_grayscale_yolo_dataset.py @@ -7,11 +7,24 @@ import argparse import hashlib import json import shutil +import sys from pathlib import Path from PIL import Image +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from training_release_manifest import ( # noqa: E402 + TrainingReleaseError, + assert_yolo_summary_bound_to_training_release, + file_sha256, + training_release_paths, +) + + def sha256(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as stream: @@ -23,9 +36,25 @@ def sha256(path: Path) -> str: def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--summary", type=Path, required=True) + parser.add_argument("--train-yaml", type=Path, required=True) + parser.add_argument("--corpus-manifest", type=Path, required=True) parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument( + "--fixture-mode", + action="store_true", + help="Accept only an explicitly fixture-only release; never creates a production-ready derived dataset.", + ) parser.add_argument("--force", action="store_true") args = parser.parse_args() + try: + release = assert_yolo_summary_bound_to_training_release( + summary_path=args.summary, + train_yaml=args.train_yaml, + corpus_manifest=args.corpus_manifest, + fixture_mode=args.fixture_mode, + ) + except TrainingReleaseError as exc: + raise SystemExit(str(exc)) from exc if args.output_dir.exists(): if not args.force: raise SystemExit(f"Output exists: {args.output_dir}") @@ -54,6 +83,16 @@ def main() -> int: f"path: {args.output_dir}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n", encoding="utf-8", ) + # The source release binds the original image bytes. These transformed + # bytes cannot inherit it, so remove its operational release pointers and + # retain them only as explicitly non-trainable parent evidence below. + for field_name in ( + "training_release_manifest", + "training_release_manifest_sha256", + "training_release_freeze", + "training_asset_manifest", + ): + summary.pop(field_name, None) summary.update( { "output_dir": str(args.output_dir), @@ -71,10 +110,19 @@ def main() -> int: "preprocessing": "luminance_rgb_replicated", "source_summary": str(args.summary), "source_summary_sha256": sha256(args.summary), + "source_training_release": str(training_release_paths(args.train_yaml)["release_manifest"]), + "source_training_release_sha256": file_sha256( + training_release_paths(args.train_yaml)["release_manifest"] + ), + "source_corpus_manifest_sha256": release["corpus"]["manifest_sha256"], "converted_tile_count": converted, "output_summary": str(output_summary), "output_summary_sha256": sha256(output_summary), "dataset_yaml": str(dataset_yaml), + "training_eligible": False, + "training_eligibility_reason": ( + "Derived image bytes require a new governed corpus, validation report and immutable training release." + ), } (args.output_dir / "grayscale-dataset-evidence.json").write_text( json.dumps(evidence, indent=2), encoding="utf-8" diff --git a/scripts/build_regional_yolo_dataset.py b/scripts/build_regional_yolo_dataset.py index c1bd5298..bc194045 100644 --- a/scripts/build_regional_yolo_dataset.py +++ b/scripts/build_regional_yolo_dataset.py @@ -6,10 +6,25 @@ from __future__ import annotations import argparse import hashlib import json +import sys from collections import Counter from pathlib import Path from typing import Any +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from training_dataset_eligibility import ( # noqa: E402 + TrainingEligibilityError, + assert_frozen_manifest_training_eligible, +) +from training_release_manifest import ( # noqa: E402 + TrainingReleaseError, + assert_yolo_summary_bound_to_embedded_training_release, + create_training_release_manifest, +) + PROTECTED_SPLITS = {"calibration", "test", "background-test"} @@ -138,11 +153,34 @@ def main() -> int: parser.add_argument("--priority-context", action="append", default=[]) parser.add_argument("--priority-repeat", type=int, default=2) parser.add_argument("--negative-repeat", type=int, default=2) + parser.add_argument( + "--review-audit", + type=Path, + help="Accepted corpus-audit evidence; mandatory for an operational training release.", + ) + parser.add_argument( + "--fixture-mode", + action="store_true", + help="Accept only an explicitly fixture-only corpus manifest; never use for operational training data.", + ) args = parser.parse_args() if args.priority_repeat < 1 or args.negative_repeat < 1: parser.error("repeat counts must be positive") summary = json.loads(args.summary.read_text(encoding="utf-8")) manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8")) + try: + assert_frozen_manifest_training_eligible( + args.corpus_manifest, + fixture_mode=args.fixture_mode, + verify_live=True, + ) + assert_yolo_summary_bound_to_embedded_training_release( + summary_path=args.summary, + corpus_manifest=args.corpus_manifest, + fixture_mode=args.fixture_mode, + ) + except (TrainingEligibilityError, TrainingReleaseError) as exc: + raise SystemExit(str(exc)) from exc train, val, evidence = build( summary=summary, manifest=manifest, @@ -161,6 +199,15 @@ def main() -> int: f"path: /\ntrain: {train_path}\nval: {val_path}\nnames:\n 0: building\n", encoding="utf-8", ) + try: + release_paths = create_training_release_manifest( + train_yaml=yaml_path, + corpus_manifest=args.corpus_manifest, + review_audit_path=args.review_audit, + fixture_mode=args.fixture_mode, + ) + except TrainingReleaseError as exc: + raise SystemExit(str(exc)) from exc evidence.update({ "source_summary": str(args.summary), "source_summary_sha256": sha256(args.summary), @@ -169,6 +216,11 @@ def main() -> int: "train_sha256": sha256(train_path), "validation_sha256": sha256(val_path), "dataset_yaml": str(yaml_path), + "training_release_manifest": str(release_paths["release_manifest"]), + "training_release_manifest_sha256": sha256(release_paths["release_manifest"]), + "training_release_freeze": str(release_paths["release_freeze"]), + "training_asset_manifest": str(release_paths["asset_manifest"]), + "fixture_mode": bool(args.fixture_mode), }) encoded_evidence = json.dumps(evidence, indent=2) (args.output_dir / "regional-dataset-evidence.json").write_text(encoded_evidence, encoding="utf-8") diff --git a/scripts/build_regional_yolo_expert_dataset.py b/scripts/build_regional_yolo_expert_dataset.py index 95fa0e9a..b281d81f 100644 --- a/scripts/build_regional_yolo_expert_dataset.py +++ b/scripts/build_regional_yolo_expert_dataset.py @@ -6,8 +6,23 @@ from __future__ import annotations import argparse import hashlib import json +import sys from pathlib import Path +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from training_dataset_eligibility import ( # noqa: E402 + TrainingEligibilityError, + assert_frozen_manifest_training_eligible, +) +from training_release_manifest import ( # noqa: E402 + TrainingReleaseError, + assert_yolo_summary_bound_to_embedded_training_release, + create_training_release_manifest, +) + def sha256(path: Path) -> str: digest = hashlib.sha256() @@ -31,12 +46,35 @@ def main() -> int: default=0, help="Include each train tile outside the expert region this many times.", ) + parser.add_argument( + "--fixture-mode", + action="store_true", + help="Accept only an explicitly fixture-only corpus manifest; never use for operational training data.", + ) + parser.add_argument( + "--review-audit", + type=Path, + help="Accepted corpus-audit evidence; mandatory for an operational training release.", + ) args = parser.parse_args() if args.positive_repeat < 1 or args.negative_repeat < 1 or args.other_region_repeat < 0: raise SystemExit("regional repeats must be positive and other-region repeat non-negative") summary = json.loads(args.summary.read_text(encoding="utf-8")) manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8")) + try: + assert_frozen_manifest_training_eligible( + args.corpus_manifest, + fixture_mode=args.fixture_mode, + verify_live=True, + ) + assert_yolo_summary_bound_to_embedded_training_release( + summary_path=args.summary, + corpus_manifest=args.corpus_manifest, + fixture_mode=args.fixture_mode, + ) + except (TrainingEligibilityError, TrainingReleaseError) as exc: + raise SystemExit(str(exc)) from exc samples = {item["sample_slug"]: item for item in manifest["samples"]} paths: list[str] = [] selected_samples: set[str] = set() @@ -69,6 +107,15 @@ def main() -> int: f"val: {args.summary.parent / 'images' / 'val'}\nnames:\n 0: building\n", encoding="utf-8", ) + try: + release_paths = create_training_release_manifest( + train_yaml=dataset_yaml, + corpus_manifest=args.corpus_manifest, + review_audit_path=args.review_audit, + fixture_mode=args.fixture_mode, + ) + except TrainingReleaseError as exc: + raise SystemExit(str(exc)) from exc evidence = { "schema_version": 1, "status": "ok", @@ -88,6 +135,11 @@ def main() -> int: "protected_samples_in_training": [], "train_list": str(train_list), "dataset_yaml": str(dataset_yaml), + "training_release_manifest": str(release_paths["release_manifest"]), + "training_release_manifest_sha256": sha256(release_paths["release_manifest"]), + "training_release_freeze": str(release_paths["release_freeze"]), + "training_asset_manifest": str(release_paths["asset_manifest"]), + "fixture_mode": bool(args.fixture_mode), } evidence_path = args.output_dir / "regional-expert-dataset.json" evidence_path.write_text(json.dumps(evidence, indent=2), encoding="utf-8") diff --git a/scripts/export_operator_yolo_dataset.py b/scripts/export_operator_yolo_dataset.py index b5d6e952..701ca553 100644 --- a/scripts/export_operator_yolo_dataset.py +++ b/scripts/export_operator_yolo_dataset.py @@ -9,6 +9,7 @@ new provider data. from __future__ import annotations import argparse +import hashlib import json import os import shutil @@ -16,6 +17,19 @@ import sys from pathlib import Path from typing import Any, Iterable +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from training_dataset_eligibility import ( # noqa: E402 + TrainingEligibilityError, + assert_frozen_manifest_training_eligible, +) +from training_release_manifest import ( # noqa: E402 + TrainingReleaseError, + create_training_release_manifest, +) + DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/operator_samples_manifest.json") DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-dataset") @@ -24,6 +38,14 @@ Transformer: Any = None Image: Any = None +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description="Export operator real-data samples to a YOLO detection dataset.", @@ -45,6 +67,12 @@ def parse_args() -> argparse.Namespace: default=os.environ.get("OPERATOR_YOLO_VAL_SAMPLES", "turnhout"), help="Comma/space separated sample slugs assigned to validation. Defaults to turnhout.", ) + parser.add_argument( + "--review-audit", + type=Path, + required=True, + help="Passed corpus audit with accepted human-review evidence for this frozen source manifest.", + ) parser.add_argument( "--force", action="store_true", @@ -218,12 +246,16 @@ def ensure_yolo_directories(output_dir: Path) -> None: def main() -> int: args = parse_args() + manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig")) + try: + assert_frozen_manifest_training_eligible(args.manifest_path, verify_live=True) + except TrainingEligibilityError as exc: + raise SystemExit(str(exc)) from exc ensure_dependencies() if args.force and args.output_dir.exists(): shutil.rmtree(args.output_dir) args.output_dir.mkdir(parents=True, exist_ok=True) ensure_yolo_directories(args.output_dir) - manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig")) samples = manifest.get("samples") or [] if not samples: raise SystemExit("Operator sample manifest contains no samples") @@ -234,12 +266,26 @@ def main() -> int: if not any(item["split"] == "val" for item in exported): raise SystemExit("YOLO dataset export produced no validation samples") dataset_yaml = write_dataset_yaml(args.output_dir) + try: + release_paths = create_training_release_manifest( + train_yaml=dataset_yaml, + corpus_manifest=args.manifest_path, + review_audit_path=args.review_audit, + ) + except TrainingReleaseError as exc: + raise SystemExit(str(exc)) from exc summary = { "status": "ok", "dataset_yaml": str(dataset_yaml), + "training_release_manifest": str(release_paths["release_manifest"]), + "training_release_manifest_sha256": file_sha256(release_paths["release_manifest"]), + "training_release_freeze": str(release_paths["release_freeze"]), + "training_asset_manifest": str(release_paths["asset_manifest"]), "output_dir": str(args.output_dir), "class_names": ["building"], "sample_count": len(exported), + "source_manifest": str(args.manifest_path), + "source_manifest_sha256": file_sha256(args.manifest_path), "train_sample_count": sum(1 for item in exported if item["split"] == "train"), "val_sample_count": sum(1 for item in exported if item["split"] == "val"), "label_count": sum(item["label_count"] for item in exported), diff --git a/scripts/export_operator_yolo_tile_dataset.py b/scripts/export_operator_yolo_tile_dataset.py index bc772f7d..c98dd1c3 100644 --- a/scripts/export_operator_yolo_tile_dataset.py +++ b/scripts/export_operator_yolo_tile_dataset.py @@ -18,6 +18,19 @@ import sys from pathlib import Path from typing import Any, Iterable +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from training_dataset_eligibility import ( # noqa: E402 + TrainingEligibilityError, + assert_frozen_manifest_training_eligible, +) +from training_release_manifest import ( # noqa: E402 + TrainingReleaseError, + create_training_release_manifest, +) + DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/operator_samples_manifest.json") DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-tile-dataset") @@ -45,6 +58,14 @@ Transformer: Any = None Image: Any = None +def file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + @dataclass(frozen=True) class TileWindow: row_off: int @@ -87,6 +108,12 @@ def parse_args() -> argparse.Namespace: default=os.environ.get("OPERATOR_YOLO_REFERENCE_SOURCE", DEFAULT_REFERENCE_SOURCE), help="Required source_name in reference GeoJSON features.", ) + parser.add_argument( + "--review-audit", + type=Path, + required=True, + help="Passed corpus audit with accepted human-review evidence for this frozen source manifest.", + ) parser.add_argument( "--reference-layer", default=os.environ.get("OPERATOR_YOLO_REFERENCE_LAYER", DEFAULT_REFERENCE_LAYER), @@ -626,13 +653,17 @@ def main() -> int: ): if not value or any(character not in "abcdefghijklmnopqrstuvwxyz0123456789_-" for character in value): raise SystemExit(f"YOLO {label} must be a non-empty canonical slug") + manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig")) + try: + assert_frozen_manifest_training_eligible(args.manifest_path, verify_live=True) + except TrainingEligibilityError as exc: + raise SystemExit(str(exc)) from exc ensure_dependencies() if args.force and args.output_dir.exists(): shutil.rmtree(args.output_dir) args.output_dir.mkdir(parents=True, exist_ok=True) ensure_yolo_directories(args.output_dir) - manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig")) manifest_samples = manifest.get("samples") or [] if not manifest_samples: raise SystemExit("Operator sample manifest contains no samples") @@ -672,6 +703,14 @@ def main() -> int: if not args.allow_empty_validation and not any(tile["split"] == "val" for tile in kept_tiles): raise SystemExit("YOLO tile dataset export produced no validation tiles") dataset_yaml = write_dataset_yaml(args.output_dir, class_name) + try: + release_paths = create_training_release_manifest( + train_yaml=dataset_yaml, + corpus_manifest=args.manifest_path, + review_audit_path=args.review_audit, + ) + except TrainingReleaseError as exc: + raise SystemExit(str(exc)) from exc positive_tiles = [tile for tile in kept_tiles if not tile["is_negative"]] negative_tiles = [tile for tile in kept_tiles if tile["is_negative"]] skipped_negative_tiles = [tile for tile in exported_tiles if not tile["kept"] and tile["is_negative"]] @@ -684,6 +723,10 @@ def main() -> int: summary = { "status": "ok", "dataset_yaml": str(dataset_yaml), + "training_release_manifest": str(release_paths["release_manifest"]), + "training_release_manifest_sha256": file_sha256(release_paths["release_manifest"]), + "training_release_freeze": str(release_paths["release_freeze"]), + "training_asset_manifest": str(release_paths["asset_manifest"]), "output_dir": str(args.output_dir), "class_names": [class_name], "reference_source": reference_source, @@ -697,6 +740,8 @@ def main() -> int: "drop_low_variance_negatives": args.drop_low_variance_negatives, "blank_range_threshold": args.blank_range_threshold, "source_manifest_sample_count": len(manifest_samples), + "source_manifest": str(args.manifest_path), + "source_manifest_sha256": file_sha256(args.manifest_path), "source_sample_count": len(samples), "selected_sample_slugs": sorted( str(sample.get("sample_slug") or "").strip().lower() for sample in samples diff --git a/scripts/prepare_operator_real_data_samples.py b/scripts/prepare_operator_real_data_samples.py index 253a339f..a825886b 100644 --- a/scripts/prepare_operator_real_data_samples.py +++ b/scripts/prepare_operator_real_data_samples.py @@ -16,6 +16,12 @@ from dataclasses import dataclass, replace from pathlib import Path from typing import Any +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from training_dataset_eligibility import TRAINING_ELIGIBILITY_POLICY_VERSION # noqa: E402 + WMS_URL = "https://geo.api.vlaanderen.be/omwrgbmrvl/wms" GRB_GBG_URL = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items" @@ -803,6 +809,14 @@ def main() -> int: manifest = { "schema_version": 2, "description": "GeoIntel operator real-data samples for configured-YOLO QA validation.", + "training_eligibility": { + "policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION, + "status": "not_eligible", + "reason": ( + "Direct provider downloads are QA-only until re-ingested through the governed " + "dataset source registry, contract validator and provenance snapshot flow." + ), + }, "output_dir": str(output_dir), "sample_width": args.width, "sample_height": args.height, diff --git a/scripts/provision_regional_grb_buildings.py b/scripts/provision_regional_grb_buildings.py index 1a8ec40f..07e96e0a 100644 --- a/scripts/provision_regional_grb_buildings.py +++ b/scripts/provision_regional_grb_buildings.py @@ -626,7 +626,7 @@ def provision_dataset( from app.services.dataset_service import DatasetService partition_checksums = { - summary["nis_code"]: summary["sha256"] + summary["filename"]: summary["sha256"] for summary in manifest["partitions"] } metadata_json = { diff --git a/scripts/provision_regional_grb_context.py b/scripts/provision_regional_grb_context.py index afb103f8..fc211416 100644 --- a/scripts/provision_regional_grb_context.py +++ b/scripts/provision_regional_grb_context.py @@ -43,7 +43,6 @@ from provision_regional_grb_buildings import ( list_paginated_items, next_page_url, observed_at, - response_data, reusable_manifest, safe_slug, sha256_file, @@ -758,7 +757,7 @@ def provision_dataset( "source_urls": manifest["grb_source_urls"], "artifact_sha256": manifest["artifact_sha256"], "artifact_size_bytes": manifest["artifact_size_bytes"], - "partition_checksums": {summary["nis_code"]: summary["sha256"] for summary in manifest["partitions"]}, + "partition_checksums": {summary["filename"]: summary["sha256"] for summary in manifest["partitions"]}, "partition_assignment_rule": manifest["partition_assignment_rule"], "reference_truncated": False, } diff --git a/scripts/refine_yolo_labels_with_sam.py b/scripts/refine_yolo_labels_with_sam.py index 3ed8ebf9..34d0e74d 100644 --- a/scripts/refine_yolo_labels_with_sam.py +++ b/scripts/refine_yolo_labels_with_sam.py @@ -10,8 +10,20 @@ import json import math import os import shutil +import sys from pathlib import Path -from typing import Any + + +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from training_release_manifest import ( # noqa: E402 + TrainingReleaseError, + assert_yolo_summary_bound_to_training_release, + file_sha256, + training_release_paths, +) def sha256(path: Path) -> str: @@ -97,6 +109,8 @@ def link_or_copy(source: Path, target: Path) -> None: def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--summary", type=Path, required=True) + parser.add_argument("--train-yaml", type=Path, required=True) + parser.add_argument("--corpus-manifest", type=Path, required=True) parser.add_argument("--model", type=Path, required=True) parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--device", default="cuda:0") @@ -110,7 +124,21 @@ def main() -> int: parser.add_argument("--max-prompts-per-pass", type=int, default=96) parser.add_argument("--fallback-policy", choices=("retain", "drop"), default="retain") parser.add_argument("--force", action="store_true") + parser.add_argument( + "--fixture-mode", + action="store_true", + help="Accept only an explicitly fixture-only source release; refined labels remain non-trainable.", + ) args = parser.parse_args() + try: + source_release = assert_yolo_summary_bound_to_training_release( + summary_path=args.summary, + train_yaml=args.train_yaml, + corpus_manifest=args.corpus_manifest, + fixture_mode=args.fixture_mode, + ) + except TrainingReleaseError as exc: + raise SystemExit(str(exc)) from exc if args.output_dir.exists(): if not args.force: raise SystemExit(f"Output exists: {args.output_dir}") @@ -194,6 +222,16 @@ def main() -> int: print(f"{index}/{len(summary['tiles'])} {tile['sample_slug']}: {len(source_boxes)}", flush=True) output_summary = dict(summary) + # SAM changes label bytes and semantics. It therefore cannot inherit the + # source release; keep the parent evidence separately and require a new + # governed corpus/review/release before any training consumer can use it. + for field_name in ( + "training_release_manifest", + "training_release_manifest_sha256", + "training_release_freeze", + "training_asset_manifest", + ): + output_summary.pop(field_name, None) output_summary.update( { "output_dir": str(args.output_dir), @@ -207,6 +245,10 @@ def main() -> int: "label_count": refined_count if args.fallback_policy == "drop" else summary["label_count"], "positive_tile_count": sum(not tile["is_negative"] for tile in output_tiles), "negative_tile_count": sum(tile["is_negative"] for tile in output_tiles), + "training_eligible": False, + "training_eligibility_reason": ( + "SAM-refined labels require a new governed corpus, validation report, human review and immutable training release." + ), } ) summary_path = args.output_dir / "yolo_tile_dataset_summary.json" @@ -220,6 +262,11 @@ def main() -> int: "status": "ok", "source_summary": str(args.summary), "source_summary_sha256": sha256(args.summary), + "source_training_release": str(training_release_paths(args.train_yaml)["release_manifest"]), + "source_training_release_sha256": file_sha256( + training_release_paths(args.train_yaml)["release_manifest"] + ), + "source_corpus_manifest_sha256": source_release["corpus"]["manifest_sha256"], "sam_model": str(args.model), "sam_model_sha256": sha256(args.model), "device": args.device, @@ -236,6 +283,8 @@ def main() -> int: "fallback_label_count": fallback_count, "dropped_fallback_label_count": fallback_count if args.fallback_policy == "drop" else 0, "fallback_reason_counts": reason_counts, + "training_eligible": False, + "fixture_mode": bool(args.fixture_mode), } (args.output_dir / "sam-refinement.json").write_text(json.dumps(evidence, indent=2), encoding="utf-8") print(json.dumps(evidence, indent=2)) diff --git a/scripts/rotate_belgium_building_holdouts.py b/scripts/rotate_belgium_building_holdouts.py index 6c0bf7de..558cbb4b 100644 --- a/scripts/rotate_belgium_building_holdouts.py +++ b/scripts/rotate_belgium_building_holdouts.py @@ -7,6 +7,7 @@ import argparse import hashlib import json import shutil +import sys from collections import Counter from pathlib import Path from typing import Any @@ -15,6 +16,19 @@ from pyproj import Transformer from shapely.geometry import box from shapely.ops import transform as shapely_transform +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from training_dataset_eligibility import ( # noqa: E402 + TrainingEligibilityError, + assert_frozen_manifest_training_eligible, +) +from training_release_manifest import ( # noqa: E402 + TrainingReleaseError, + assert_yolo_summary_bound_to_embedded_training_release, +) + def sha256(path: Path) -> str: digest = hashlib.sha256() @@ -73,6 +87,11 @@ def main() -> int: parser.add_argument("--internal-val-samples", required=True) parser.add_argument("--version", required=True) parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument( + "--fixture-mode", + action="store_true", + help="Accept only an explicitly fixture-only source corpus; never use for operational holdout rotation.", + ) args = parser.parse_args() role_slugs = { @@ -86,6 +105,20 @@ def main() -> int: raise SystemExit("rotated holdout lists overlap") manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8")) + try: + assert_frozen_manifest_training_eligible( + args.corpus_manifest, + fixture_mode=args.fixture_mode, + verify_live=True, + ) + for path in args.source_summary: + assert_yolo_summary_bound_to_embedded_training_release( + summary_path=path, + corpus_manifest=args.corpus_manifest, + fixture_mode=args.fixture_mode, + ) + except (TrainingEligibilityError, TrainingReleaseError) as exc: + raise SystemExit(str(exc)) from exc source_samples = {item["sample_slug"]: item for item in manifest["samples"]} missing = sorted(all_holdouts - source_samples.keys()) if missing: @@ -126,6 +159,18 @@ def main() -> int: args.output_dir.mkdir(parents=True, exist_ok=True) manifest_path = args.output_dir / "operator_samples_manifest.json" write_json(manifest_path, rotated_manifest) + write_json( + args.output_dir / "corpus-freeze.json", + { + "schema_version": 2, + "dataset_version": rotated_manifest["dataset_version"], + "manifest_sha256": sha256(manifest_path), + "sample_count": len(rotated_manifest["samples"]), + "immutable": True, + "training_eligibility_policy": rotated_manifest["training_eligibility"]["policy_version"], + "fixture_mode": bool(args.fixture_mode), + }, + ) leakage = audit_spatial_leakage(rotated_manifest["samples"]) write_json(args.output_dir / "spatial-leakage-audit.json", leakage) if leakage["status"] != "ok": @@ -193,6 +238,11 @@ def main() -> int: "fit_samples_in_internal_validation": sorted( {tile["sample_slug"] for tile in train_tiles} & {tile["sample_slug"] for tile in internal_val_tiles} ), + "fixture_mode": bool(args.fixture_mode), + "training_eligible": False, + "training_eligibility_reason": ( + "A rotated split needs a new human review, label contract validation and immutable training release." + ), } if evidence["protected_samples_in_training"]: raise SystemExit("protected samples leaked into rotated training lists") diff --git a/scripts/run_accuracy_phase2_foundation_audit.py b/scripts/run_accuracy_phase2_foundation_audit.py new file mode 100644 index 00000000..3462a547 --- /dev/null +++ b/scripts/run_accuracy_phase2_foundation_audit.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +"""Emit reproducible local evidence for the Phase-2 data foundation. + +The collector does not connect to PostgreSQL, source providers, a GPU or any +training corpus. It inventories the versioned source-policy and contract +definitions that are present in this checkout. Runtime migration/application +evidence is recorded separately because it needs an explicitly chosen target +database and must never be inferred from this static report. +""" + +from __future__ import annotations + +import argparse +import json +import subprocess +import sys +from collections import Counter +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +BACKEND = ROOT / "backend" +if str(BACKEND) not in sys.path: + sys.path.insert(0, str(BACKEND)) + +from app.services.data_contract_validation import build_default_data_contract_registry # noqa: E402 +from app.services.source_registry_service import SERVER_OWNED_SOURCE_DEFINITIONS # noqa: E402 + + +def _git_value(*args: str) -> str | None: + try: + return subprocess.check_output( + ["git", *args], + cwd=ROOT, + text=True, + stderr=subprocess.DEVNULL, + ).strip() or None + except (OSError, subprocess.CalledProcessError): + return None + + +def collect() -> dict[str, Any]: + definitions = list(SERVER_OWNED_SOURCE_DEFINITIONS.values()) + contracts = build_default_data_contract_registry().registered_contracts() + classification_counts = Counter(item.classification for item in definitions) + source_items = [ + { + "source_key": item.source_key, + "classification": item.classification, + "authority_name": item.authority_name, + "authority_scope": item.authority_scope, + "provider_adapter_key": item.provider_adapter_key, + "default_crs": item.default_crs, + "default_units": item.default_units, + "freshness_status": item.freshness_status, + "ingest_status": item.ingest_status, + "ground_truth_allowed": bool((item.usage_policy or {}).get("ground_truth_allowed")), + "training_allowed": bool((item.usage_policy or {}).get("training_allowed")), + } + for item in sorted(definitions, key=lambda definition: definition.source_key) + ] + contract_items = [ + { + "key": item.key, + "version": item.version, + "kind": item.kind.value, + "fingerprint_sha256": item.fingerprint(), + "canonical_storage_crs": item.canonical_storage_crs, + "accepted_source_crs": sorted(item.accepted_source_crs), + "required_metadata_fields": list(item.required_metadata_fields), + "requires_source_registry": item.lineage_rules.require_source_registry, + "requires_source_snapshot": item.lineage_rules.require_source_snapshot, + "requires_upstream_assets": item.lineage_rules.require_upstream_assets, + } + for item in sorted(contracts, key=lambda contract: (contract.kind.value, contract.key, contract.version)) + ] + return { + "schema_version": 1, + "program": "GeoIntel Accuracy Improvement Program", + "phase": "P2", + "collected_at": datetime.now(UTC).isoformat(), + "repository": { + "branch": _git_value("branch", "--show-current"), + "head": _git_value("rev-parse", "HEAD"), + "dirty": bool(_git_value("status", "--porcelain")), + }, + "scope": "Belgium and the Belgian North Sea", + "migration_revision": "202608010001", + "source_registry": { + "definition_count": len(source_items), + "classification_counts": dict(sorted(classification_counts.items())), + "required_building_policy": { + "grb_primary_building_validation": SERVER_OWNED_SOURCE_DEFINITIONS["grb"].usage_policy[ + "validation_authority" + ].get("building_validation"), + "buildings_register_classification": SERVER_OWNED_SOURCE_DEFINITIONS[ + "digitaal_vlaanderen_buildings_addresses_register" + ].classification, + "sentinel_2_classification": SERVER_OWNED_SOURCE_DEFINITIONS["sentinel_2"].classification, + "dhmv_classification": SERVER_OWNED_SOURCE_DEFINITIONS["digitaal_vlaanderen_dhmv"].classification, + "osm_ground_truth_allowed": SERVER_OWNED_SOURCE_DEFINITIONS["osm"].usage_policy[ + "ground_truth_allowed" + ], + }, + "definitions": source_items, + }, + "data_contracts": contract_items, + "claim_boundary": ( + "Static policy/contract inventory only. It does not attest that a database migration ran, " + "that legacy rows are complete, or that a model/corpus is release-ready." + ), + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--output", + type=Path, + default=ROOT / "artifacts" / "evidence" / "accuracy" / "P2" / "source-contract-inventory.json", + ) + args = parser.parse_args() + result = collect() + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8") + print(json.dumps({"output": str(args.output), "source_count": result["source_registry"]["definition_count"]})) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_belgium_building_training_loop.py b/scripts/run_belgium_building_training_loop.py index 1d59c3c7..b44dc055 100644 --- a/scripts/run_belgium_building_training_loop.py +++ b/scripts/run_belgium_building_training_loop.py @@ -13,6 +13,21 @@ from datetime import UTC, datetime from pathlib import Path from typing import Any +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from training_dataset_eligibility import ( # noqa: E402 + TrainingEligibilityError, + assert_frozen_manifest_training_eligible, +) +from training_release_manifest import ( # noqa: E402 + TrainingReleaseError, + assert_training_release_eligible, + human_review_audit_failures, + training_release_paths, +) + def sha256(path: Path) -> str: digest = hashlib.sha256() @@ -32,11 +47,14 @@ def write_json(path: Path, value: dict[str, Any]) -> None: def dataset_audit_failures( audit: dict[str, Any], train_quality_audit: dict[str, Any], + *, + fixture_mode: bool = False, ) -> list[str]: - """Return automated corpus blockers while leaving final human review deferred.""" + """Return fail-closed corpus blockers, including human review in normal mode.""" failures = [str(item) for item in audit.get("failures") or []] status = audit.get("status") - if status not in {"ok", "needs_human_review"}: + permitted_statuses = {"ok", "needs_human_review"} if fixture_mode else {"ok"} + if status not in permitted_statuses: failures.append(f"unsupported audit status: {status}") if audit.get("manifest_immutable") is not True: failures.append("corpus manifest is not immutable") @@ -50,9 +68,56 @@ def dataset_audit_failures( failures.append("train tile quality audit contains missing label files") if int(train_quality_audit.get("low_variance_positive_tile_count", -1)) != 0: failures.append("dataset contains blank/low-variance positive tiles") + if not fixture_mode: + failures.extend(human_review_audit_failures(audit)) return failures +def verify_training_inputs( + *, + train_yaml: Path, + corpus_manifest: Path, + fixture_mode: bool, +) -> dict[str, Any]: + """Re-check every immutable input before initial, retry or resume training.""" + + try: + assert_frozen_manifest_training_eligible( + corpus_manifest, + fixture_mode=fixture_mode, + verify_live=True, + ) + return assert_training_release_eligible( + train_yaml=train_yaml, + corpus_manifest=corpus_manifest, + fixture_mode=fixture_mode, + ) + except (TrainingEligibilityError, TrainingReleaseError) as exc: + raise TrainingReleaseError(str(exc)) from exc + + +def assert_dataset_audit_bound_to_release( + *, + release: dict[str, Any], + dataset_audit: Path, + fixture_mode: bool, +) -> None: + """Do not let a caller swap the reviewed corpus audit after release sealing.""" + + if fixture_mode: + return + review = release.get("human_review") + if not isinstance(review, dict): + raise TrainingReleaseError("Training release has no human-review audit binding") + review_audit_path = review.get("audit_path") + if not isinstance(review_audit_path, str) or not review_audit_path: + raise TrainingReleaseError("Training release human-review audit path is missing") + if Path(review_audit_path).resolve(strict=False) != dataset_audit.resolve(strict=False): + raise TrainingReleaseError( + "--dataset-audit does not match the immutable training-release human-review audit" + ) + + def select_calibration_threshold(report: dict[str, Any]) -> dict[str, Any]: """Choose a threshold without consulting test or background evidence.""" eligible = [item for item in report["sweeps"] if item["pure_empty_false_positives"] == 0] @@ -111,6 +176,16 @@ def rejected_candidate_score(assessment: dict[str, Any]) -> tuple[float, ...]: return tuple(normalized) +def protected_feedback_roles(assessment: dict[str, Any]) -> list[str]: + """Return protected evidence roles that make iterative retraining illegal.""" + + return [ + role + for role in ("test", "background") + if assessment.get(role) is not None + ] + + def training_command( yolo: str, *, @@ -183,17 +258,23 @@ def failure_sampling_command( corpus_manifest: Path, assessment: Path, output_dir: Path, + review_audit: Path, sampling_round: int = 0, + fixture_mode: bool = False, ) -> list[str]: - return [ + command = [ sys.executable, str(scripts_dir / "build_failure_driven_yolo_sampling.py"), "--summary", str(train_summary), "--corpus-manifest", str(corpus_manifest), "--assessment", str(assessment), "--output-dir", str(output_dir), + "--review-audit", str(review_audit), "--sampling-round", str(sampling_round), ] + if fixture_mode: + command.append("--fixture-mode") + return command def resumable_training_command(yolo: str, checkpoint: Path) -> list[str]: @@ -259,6 +340,14 @@ def main() -> int: parser.add_argument("--min-region-recall", type=float, default=0.4) parser.add_argument("--max-pure-empty-fp", type=int, default=0) parser.add_argument("--dry-run", action="store_true") + parser.add_argument( + "--fixture-mode", + action="store_true", + help=( + "Accept an explicitly fixture-only corpus manifest. " + "This mode is prohibited for operational training." + ), + ) parser.add_argument( "--evaluate-initial-model", action="store_true", @@ -267,9 +356,29 @@ def main() -> int: args = parser.parse_args() if args.iterations < 1: raise SystemExit("--iterations must be positive") + try: + initial_release = verify_training_inputs( + train_yaml=args.train_yaml, + corpus_manifest=args.corpus_manifest, + fixture_mode=args.fixture_mode, + ) + except TrainingReleaseError as exc: + raise SystemExit(str(exc)) from exc + try: + assert_dataset_audit_bound_to_release( + release=initial_release, + dataset_audit=args.dataset_audit, + fixture_mode=args.fixture_mode, + ) + except TrainingReleaseError as exc: + raise SystemExit(str(exc)) from exc dataset_audit = json.loads(args.dataset_audit.read_text(encoding="utf-8")) train_quality_audit = json.loads(args.train_quality_audit.read_text(encoding="utf-8")) - audit_failures = dataset_audit_failures(dataset_audit, train_quality_audit) + audit_failures = dataset_audit_failures( + dataset_audit, + train_quality_audit, + fixture_mode=args.fixture_mode, + ) if audit_failures: raise SystemExit(f"Dataset audit is not eligible for training: {audit_failures}") @@ -285,6 +394,13 @@ def main() -> int: "train_quality_audit": str(args.train_quality_audit), "train_quality_audit_sha256": sha256(args.train_quality_audit), "corpus_manifest": str(args.corpus_manifest), + "corpus_manifest_sha256": sha256(args.corpus_manifest), + "initial_training_release": str(training_release_paths(args.train_yaml)["release_manifest"]), + "initial_training_release_sha256": sha256( + training_release_paths(args.train_yaml)["release_manifest"] + ), + "initial_training_release_contract": initial_release["contract_version"], + "fixture_mode": bool(args.fixture_mode), "iterations": [], } if state_path.is_file(): @@ -304,6 +420,26 @@ def main() -> int: evaluate_existing = args.evaluate_initial_model and offset == 0 and not state["iterations"] partial_checkpoint = train_run / "weights" / "last.pt" resume_partial = not evaluate_existing and partial_checkpoint.is_file() + try: + release = verify_training_inputs( + train_yaml=train_yaml, + corpus_manifest=args.corpus_manifest, + fixture_mode=args.fixture_mode, + ) + except TrainingReleaseError as exc: + raise RuntimeError( + f"Training inputs changed before {name}; refusing initial/retry/resume execution: {exc}" + ) from exc + try: + assert_dataset_audit_bound_to_release( + release=release, + dataset_audit=args.dataset_audit, + fixture_mode=args.fixture_mode, + ) + except TrainingReleaseError as exc: + raise RuntimeError( + f"Training audit changed before {name}; refusing initial/retry/resume execution: {exc}" + ) from exc command = None if evaluate_existing else ( resumable_training_command(args.yolo, partial_checkpoint) if resume_partial else training_command( @@ -454,11 +590,27 @@ def main() -> int: "candidate_sha256": sha256(candidate), "training_skipped_for_existing_checkpoint": evaluate_existing, "training_resumed_from_partial_checkpoint": resume_partial, + "training_release": str(training_release_paths(train_yaml)["release_manifest"]), + "training_release_sha256": sha256(training_release_paths(train_yaml)["release_manifest"]), + "training_release_asset_manifest_sha256": release["asset_manifest"]["sha256"], "assessment": str(assessment), "status": decision["status"], "failures": decision["failures"], } state["iterations"].append(record) + protected_feedback = protected_feedback_roles(decision) + if decision["status"] != "training_complete" and protected_feedback: + record["protected_feedback_blocked"] = protected_feedback + record["retraining_prohibited"] = True + state["status"] = "protected_evaluation_rejected" + state["stopped_at"] = datetime.now(UTC).isoformat() + state["stop_reason"] = ( + "Protected test/background evidence was opened for a rejected candidate; " + "its results cannot generate another training YAML." + ) + write_json(state_path, state) + print(json.dumps(state, indent=2)) + return 3 score = rejected_candidate_score(decision) if decision["status"] != "training_complete" else () incumbent_score = tuple(state.get("incumbent_rejected_score", ())) if not incumbent_score or score > incumbent_score: @@ -482,14 +634,29 @@ def main() -> int: corpus_manifest=args.corpus_manifest, assessment=assessment, output_dir=sampling_dir, + review_audit=args.dataset_audit, sampling_round=index, + fixture_mode=args.fixture_mode, ), iteration_dir / "failure-driven-sampling.log", ) sampling_evidence = sampling_dir / "failure-driven-sampling.json" next_train_yaml = sampling_dir / "dataset.yaml" - if not sampling_evidence.is_file() or not next_train_yaml.is_file(): + next_release_paths = training_release_paths(next_train_yaml) + if not sampling_evidence.is_file() or not next_train_yaml.is_file() or not all( + path.is_file() for path in next_release_paths.values() + ): raise RuntimeError("Failure-driven sampling produced incomplete evidence") + try: + verify_training_inputs( + train_yaml=next_train_yaml, + corpus_manifest=args.corpus_manifest, + fixture_mode=args.fixture_mode, + ) + except TrainingReleaseError as exc: + raise RuntimeError( + f"Failure-driven sampling produced an unbound training release: {exc}" + ) from exc record["failure_driven_sampling"] = str(sampling_evidence) record["failure_driven_sampling_sha256"] = sha256(sampling_evidence) record["next_train_yaml"] = str(next_train_yaml) diff --git a/scripts/run_golden_qa_benchmark.py b/scripts/run_golden_qa_benchmark.py index 7c3fc29f..34d6291d 100644 --- a/scripts/run_golden_qa_benchmark.py +++ b/scripts/run_golden_qa_benchmark.py @@ -3,6 +3,8 @@ from __future__ import annotations import argparse import json import sys +from datetime import UTC, datetime +from hashlib import sha256 from pathlib import Path from uuid import uuid4 @@ -12,9 +14,10 @@ BACKEND_ROOT = ROOT / "backend" if str(BACKEND_ROOT) not in sys.path: sys.path.insert(0, str(BACKEND_ROOT)) -from app.models import Area, Dataset, Metric, QualityCheck # noqa: E402 +from app.models import Dataset, Metric, QualityCheck, SourceRegistry, SourceSnapshot # noqa: E402 from app.services.qa_service import QaService # noqa: E402 from app.services.quality_service import QualityService # noqa: E402 +from app.services.source_registry_service import SourceRegistryService # noqa: E402 class BenchmarkSession: @@ -55,18 +58,58 @@ def _load_manifest() -> dict: def _dataset(dataset_id, project_id, name: str, path: Path, *, role: str) -> Dataset: + # The benchmark is still synthetic evidence, but it intentionally models + # the same complete Phase 2 provenance binding required at the QA service + # boundary. The candidate is a derived fixture; the reference simulates + # a governed GRB building snapshot. No legacy fixture exception is used. + source_key = "grb" if role == "reference" else "derived" + source = SourceRegistry( + id=uuid4(), + **SourceRegistryService.definition_for(source_key).as_model_values(), + ) + checksum = sha256(path.read_bytes()).hexdigest() + snapshot = SourceSnapshot( + id=uuid4(), + source_registry_id=source.id, + snapshot_key=f"golden-qa:{source_key}:{checksum}", + checksum_sha256=checksum, + fetched_at=datetime.now(UTC), + crs="EPSG:4326", + units=source.default_units, + spatial_resolution_json={"status": "fixture"}, + temporal_coverage_json={"status": "fixture"}, + geographic_coverage_json={"scope": "golden-qa-fixture"}, + observed_schema_json={"dataset_type": "vector", "fixture_mode": True}, + freshness_status="current", + ingest_status="ingested", + known_limitations_json=["Synthetic golden benchmark fixture; not production evidence."], + snapshot_metadata_json={"fixture_mode": True, "benchmark": "golden-qa"}, + ) return Dataset( id=dataset_id, project_id=project_id, name=name, dataset_type="vector", - source="golden_fixture", - dataset_role=role, - source_name="fixture", + source="governed golden benchmark fixture", + dataset_role="reference" if role == "reference" else "derived", + source_name=source.source_key, reference_layer_name="buildings" if role == "reference" else None, storage_path=str(path), crs="EPSG:4326", - metadata_json={"crs_assumed": False}, + checksum_sha256=checksum, + source_registry_id=source.id, + source_snapshot_id=snapshot.id, + source_registry=source, + source_snapshot=snapshot, + data_contract_key="geointel.vector.geojson", + data_contract_version="1.0.0", + validation_status="passed", + provenance_status="complete", + lineage_status="not_applicable", + quarantine_status="not_quarantined", + metadata_json={"crs_assumed": False, "fixture_mode": True}, + source_metadata={"fixture_mode": True, "source_registry_key": source.source_key}, + provenance_metadata={"fixture_mode": True, "source_snapshot_id": str(snapshot.id)}, status="ready", ) diff --git a/scripts/run_readiness_check.sh b/scripts/run_readiness_check.sh index 1f2414cf..a223e5d3 100644 --- a/scripts/run_readiness_check.sh +++ b/scripts/run_readiness_check.sh @@ -109,6 +109,10 @@ ${PYTHON_BIN} -m py_compile scripts/audit_source_freshness.py ${PYTHON_BIN} -m py_compile scripts/manage_grb_refresh.py ${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py ${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py +${PYTHON_BIN} -m py_compile scripts/training_dataset_eligibility.py +${PYTHON_BIN} -m py_compile scripts/training_release_manifest.py +${PYTHON_BIN} -m py_compile scripts/run_belgium_building_training_loop.py +${PYTHON_BIN} -m py_compile scripts/build_failure_driven_yolo_sampling.py ${PYTHON_BIN} -m py_compile scripts/audit_operator_yolo_dataset_quality.py ${PYTHON_BIN} -m py_compile scripts/build_detection_model_promotion_report.py ${PYTHON_BIN} -m py_compile scripts/build_mol_operational_benchmark_report.py diff --git a/scripts/supervise_container_yolo_training.py b/scripts/supervise_container_yolo_training.py index c4e98fe1..1686ecc3 100644 --- a/scripts/supervise_container_yolo_training.py +++ b/scripts/supervise_container_yolo_training.py @@ -6,10 +6,20 @@ from __future__ import annotations import argparse import json import subprocess +import sys import time from datetime import UTC, datetime from pathlib import Path +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from training_release_manifest import ( # noqa: E402 + TrainingReleaseError, + assert_training_release_eligible, +) + def container_running(container: str) -> bool: result = subprocess.run( @@ -44,11 +54,33 @@ def load_completion_command(path: Path) -> list[str]: return command +def verify_training_release( + *, + train_yaml: Path, + corpus_manifest: Path, + fixture_mode: bool, +) -> None: + """Re-check the exact release before each detached resume command.""" + + assert_training_release_eligible( + train_yaml=train_yaml, + corpus_manifest=corpus_manifest, + fixture_mode=fixture_mode, + ) + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--container", required=True) parser.add_argument("--host-run-dir", type=Path, required=True) parser.add_argument("--container-checkpoint", required=True) + parser.add_argument("--train-yaml", type=Path, required=True) + parser.add_argument("--corpus-manifest", type=Path, required=True) + parser.add_argument( + "--fixture-mode", + action="store_true", + help="Only valid for an explicitly fixture-only frozen corpus release.", + ) parser.add_argument("--run-marker", required=True) parser.add_argument("--yolo", default="/opt/geointel/venv/bin/yolo") parser.add_argument("--poll-seconds", type=int, default=30) @@ -101,6 +133,17 @@ def main() -> int: state["status"] = "resume_budget_exhausted" write_state(state_path, state) return 3 + try: + verify_training_release( + train_yaml=args.train_yaml, + corpus_manifest=args.corpus_manifest, + fixture_mode=args.fixture_mode, + ) + except TrainingReleaseError as exc: + state["status"] = "training_release_verification_failed" + state["training_release_error"] = str(exc) + write_state(state_path, state) + return 6 result = subprocess.run( ["docker", "exec", "-d", args.container, args.yolo, "train", f"resume={args.container_checkpoint}", "device=0"], diff --git a/scripts/train_building_proposal_classifier.py b/scripts/train_building_proposal_classifier.py index 542fecba..b0dff734 100644 --- a/scripts/train_building_proposal_classifier.py +++ b/scripts/train_building_proposal_classifier.py @@ -4,8 +4,248 @@ from __future__ import annotations import argparse +import hashlib import json -from pathlib import Path +import sys +from collections import Counter +from pathlib import Path, PurePosixPath +from typing import Any, Mapping + + +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from training_dataset_eligibility import ( # noqa: E402 + TrainingEligibilityError, + assert_frozen_manifest_training_eligible, +) +from training_release_manifest import ( # noqa: E402 + TrainingReleaseError, + assert_yolo_summary_bound_to_embedded_training_release, +) + + +PROPOSAL_DATASET_PROVENANCE_NAME = "proposal-dataset-provenance.json" +PROPOSAL_DATASET_SCHEMA_VERSION = 1 +_CROP_IMAGE_SUFFIXES = {".bmp", ".jpeg", ".jpg", ".png", ".tif", ".tiff", ".webp"} + + +class ProposalDatasetProvenanceError(ValueError): + """Raised when proposal crops no longer match their governed provenance.""" + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _canonical_json_bytes(payload: Mapping[str, Any]) -> bytes: + return json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8") + + +def _payload_sha256(payload: Mapping[str, Any]) -> str: + normalized = dict(payload) + normalized.pop("manifest_sha256", None) + return hashlib.sha256(_canonical_json_bytes(normalized)).hexdigest() + + +def _is_sha256(value: Any) -> bool: + return isinstance(value, str) and len(value) == 64 and all(character in "0123456789abcdef" for character in value) + + +def _require_mapping(value: Any, *, field: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise ProposalDatasetProvenanceError(f"proposal dataset provenance {field} must be an object") + return value + + +def _require_sha256(value: Any, *, field: str) -> str: + if not _is_sha256(value): + raise ProposalDatasetProvenanceError(f"proposal dataset provenance {field} must be a lowercase SHA-256") + return str(value) + + +def _require_nonempty_path(value: Any, *, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ProposalDatasetProvenanceError(f"proposal dataset provenance {field} must be a non-empty path") + return value + + +def _safe_relative_crop_path(value: Any, *, dataset_root: Path) -> tuple[str, Path]: + if not isinstance(value, str) or not value or "\\" in value: + raise ProposalDatasetProvenanceError("proposal crop relative_path must be a non-empty POSIX relative path") + relative = PurePosixPath(value) + if relative.is_absolute() or ".." in relative.parts or "." in relative.parts: + raise ProposalDatasetProvenanceError(f"proposal crop has an unsafe relative path: {value!r}") + path = (dataset_root / Path(*relative.parts)).resolve(strict=False) + try: + path.relative_to(dataset_root) + except ValueError as exc: # pragma: no cover - resolved-path defence + raise ProposalDatasetProvenanceError(f"proposal crop escapes dataset directory: {value!r}") from exc + return relative.as_posix(), path + + +def load_governed_corpus_manifest(corpus_manifest_path: Path, *, fixture_mode: bool) -> dict[str, Any]: + """Assert frozen evidence and current live Dataset eligibility before PyTorch.""" + + try: + manifest = assert_frozen_manifest_training_eligible( + corpus_manifest_path, + fixture_mode=fixture_mode, + verify_live=True, + ) + except TrainingEligibilityError as exc: + raise ProposalDatasetProvenanceError(str(exc)) from exc + if not isinstance(manifest, dict): # pragma: no cover - defensive contract boundary + raise ProposalDatasetProvenanceError("governed corpus manifest must be a JSON object") + return manifest + + +def validate_proposal_dataset_provenance( + dataset_dir: Path, + corpus_manifest_path: Path, + *, + fixture_mode: bool, +) -> dict[str, Any]: + """Fail closed unless every trainable crop still matches its source evidence. + + ImageFolder discovers files from the directory tree. Merely verifying a + manifest is insufficient: any unbound image placed under train/ or val/ + would otherwise enter PyTorch training. This validator compares the + complete tree with the immutable crop manifest and re-binds it to the + supplied frozen corpus and summary. + """ + + root = dataset_dir.expanduser().resolve(strict=False) + if not root.is_dir(): + raise ProposalDatasetProvenanceError(f"proposal dataset directory is unavailable: {root}") + manifest_path = root / PROPOSAL_DATASET_PROVENANCE_NAME + try: + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ProposalDatasetProvenanceError(f"proposal dataset provenance is unreadable: {manifest_path}") from exc + if not isinstance(payload, dict): + raise ProposalDatasetProvenanceError("proposal dataset provenance must be a JSON object") + if payload.get("schema_version") != PROPOSAL_DATASET_SCHEMA_VERSION: + raise ProposalDatasetProvenanceError("proposal dataset provenance schema_version is unsupported") + if payload.get("status") != "ok" or payload.get("immutable") is not True: + raise ProposalDatasetProvenanceError("proposal dataset provenance is not an immutable successful release") + if payload.get("governed_corpus_live_recheck") is not True: + raise ProposalDatasetProvenanceError("proposal dataset provenance lacks the governed live-source recheck") + if bool(payload.get("fixture_mode")) != fixture_mode: + raise ProposalDatasetProvenanceError("proposal dataset provenance fixture-mode binding mismatches this invocation") + if payload.get("manifest_sha256") != _payload_sha256(payload): + raise ProposalDatasetProvenanceError("proposal dataset provenance manifest checksum mismatches its content") + + expected_corpus_sha256 = sha256(corpus_manifest_path) + source = _require_mapping(payload.get("source"), field="source") + recorded_corpus = _require_mapping(source.get("corpus_manifest"), field="source.corpus_manifest") + if _require_sha256(recorded_corpus.get("sha256"), field="source.corpus_manifest.sha256") != expected_corpus_sha256: + raise ProposalDatasetProvenanceError("proposal crops are not bound to the supplied governed corpus manifest") + + corpus_freeze_path = corpus_manifest_path.parent / "corpus-freeze.json" + recorded_freeze = _require_mapping(source.get("corpus_freeze"), field="source.corpus_freeze") + if not corpus_freeze_path.is_file() or _require_sha256( + recorded_freeze.get("sha256"), field="source.corpus_freeze.sha256" + ) != sha256(corpus_freeze_path): + raise ProposalDatasetProvenanceError("proposal crops are not bound to the current frozen corpus sidecar") + + summary = _require_mapping(source.get("summary"), field="source.summary") + summary_path_value = _require_nonempty_path(summary.get("path"), field="source.summary.path") + summary_path = Path(summary_path_value).expanduser().resolve(strict=False) + if not summary_path.is_file(): + raise ProposalDatasetProvenanceError(f"proposal source summary is unavailable: {summary_path}") + if _require_sha256(summary.get("sha256"), field="source.summary.sha256") != sha256(summary_path): + raise ProposalDatasetProvenanceError("proposal source summary checksum no longer matches the crop provenance") + try: + summary_payload = json.loads(summary_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise ProposalDatasetProvenanceError(f"proposal source summary is unreadable: {summary_path}") from exc + if not isinstance(summary_payload, Mapping) or summary_payload.get("source_manifest_sha256") != expected_corpus_sha256: + raise ProposalDatasetProvenanceError("proposal source summary is not bound to the supplied governed corpus manifest") + if summary.get("source_manifest_sha256") != expected_corpus_sha256: + raise ProposalDatasetProvenanceError("proposal crop provenance summary binding is invalid") + try: + source_release = assert_yolo_summary_bound_to_embedded_training_release( + summary_path=summary_path, + corpus_manifest=corpus_manifest_path, + fixture_mode=fixture_mode, + ) + except TrainingReleaseError as exc: + raise ProposalDatasetProvenanceError( + "proposal source summary has no eligible immutable training release" + ) from exc + recorded_release = _require_mapping(source.get("training_release"), field="source.training_release") + if ( + recorded_release.get("dataset_yaml_path") != source_release["dataset_yaml"]["path"] + or recorded_release.get("dataset_yaml_sha256") != source_release["dataset_yaml"]["sha256"] + or recorded_release.get("corpus_manifest_sha256") != source_release["corpus"]["manifest_sha256"] + ): + raise ProposalDatasetProvenanceError("proposal crop provenance release binding is invalid") + + proposal_model = _require_mapping(source.get("proposal_model"), field="source.proposal_model") + _require_nonempty_path(proposal_model.get("path"), field="source.proposal_model.path") + _require_sha256(proposal_model.get("sha256"), field="source.proposal_model.sha256") + entries = payload.get("crops") + if not isinstance(entries, list) or not entries: + raise ProposalDatasetProvenanceError("proposal dataset provenance has no crop entries") + + expected_paths: set[str] = set() + calculated_counts: Counter[str] = Counter() + normalized_entries: list[dict[str, Any]] = [] + for entry in entries: + item = _require_mapping(entry, field="crops[]") + relative_path, crop_path = _safe_relative_crop_path(item.get("relative_path"), dataset_root=root) + if relative_path in expected_paths: + raise ProposalDatasetProvenanceError(f"proposal crop provenance has duplicate path: {relative_path}") + expected_paths.add(relative_path) + split = item.get("split") + label = item.get("label") + if split not in {"train", "val"} or label not in {"negative", "positive"}: + raise ProposalDatasetProvenanceError(f"proposal crop has invalid split/class: {relative_path}") + if not isinstance(item.get("sample_slug"), str) or not item["sample_slug"].strip(): + raise ProposalDatasetProvenanceError(f"proposal crop has no sample identity: {relative_path}") + if not crop_path.is_file(): + raise ProposalDatasetProvenanceError(f"proposal crop is missing: {relative_path}") + if _require_sha256(item.get("sha256"), field=f"crops[{relative_path}].sha256") != sha256(crop_path): + raise ProposalDatasetProvenanceError(f"proposal crop checksum mismatches provenance: {relative_path}") + if item.get("size_bytes") != crop_path.stat().st_size: + raise ProposalDatasetProvenanceError(f"proposal crop byte size mismatches provenance: {relative_path}") + _require_nonempty_path(item.get("source_image_path"), field=f"crops[{relative_path}].source_image_path") + _require_sha256(item.get("source_image_sha256"), field=f"crops[{relative_path}].source_image_sha256") + _require_nonempty_path(item.get("source_label_path"), field=f"crops[{relative_path}].source_label_path") + _require_sha256(item.get("source_label_sha256"), field=f"crops[{relative_path}].source_label_sha256") + calculated_counts[f"{split}/{label}"] += 1 + normalized_entries.append(dict(item)) + + canonical_entries = sorted(normalized_entries, key=lambda item: str(item["relative_path"])) + expected_crops_sha256 = hashlib.sha256(_canonical_json_bytes({"crops": canonical_entries})).hexdigest() + if payload.get("crops_sha256") != expected_crops_sha256: + raise ProposalDatasetProvenanceError("proposal crop collection checksum mismatches provenance") + if payload.get("crop_count") != len(entries): + raise ProposalDatasetProvenanceError("proposal crop count mismatches provenance") + if payload.get("counts") != dict(sorted(calculated_counts.items())): + raise ProposalDatasetProvenanceError("proposal crop class counts mismatch provenance") + if any(calculated_counts[f"{split}/{label}"] < 1 for split in ("train", "val") for label in ("negative", "positive")): + raise ProposalDatasetProvenanceError("proposal dataset has an empty train/validation class") + + actual_paths = { + file_path.relative_to(root).as_posix() + for file_path in root.rglob("*") + if file_path.is_file() and file_path.suffix.lower() in _CROP_IMAGE_SUFFIXES + } + if actual_paths != expected_paths: + unexpected = sorted(actual_paths - expected_paths) + missing = sorted(expected_paths - actual_paths) + raise ProposalDatasetProvenanceError( + "proposal dataset files are not exactly the immutable crop manifest " + f"(unexpected={unexpected}, missing={missing})" + ) + return payload def binary_metrics(scores: list[float], labels: list[int], threshold: float = 0.5) -> dict[str, float | int]: @@ -21,16 +261,44 @@ def binary_metrics(scores: list[float], labels: list[int], threshold: float = 0. def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--dataset-dir", type=Path, required=True) + parser.add_argument( + "--corpus-manifest", + type=Path, + required=True, + help="The exact frozen governed corpus manifest that produced the proposal crops.", + ) parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--epochs", type=int, default=12) parser.add_argument("--batch", type=int, default=64) parser.add_argument("--lr", type=float, default=1e-4) parser.add_argument("--device", default="cuda:0") parser.add_argument("--export-existing-best", action="store_true") + parser.add_argument( + "--fixture-mode", + action="store_true", + help="Permit only an explicitly fixture-only corpus; operational runs always resolve live dataset evidence.", + ) args = parser.parse_args() if args.output_dir.exists() and not args.export_existing_best: parser.error(f"output already exists: {args.output_dir}") + # All governed-source checks run before importing torch/torchvision or + # touching CUDA. A source revocation therefore cannot start PyTorch work. + try: + governed_corpus = load_governed_corpus_manifest( + args.corpus_manifest, + fixture_mode=args.fixture_mode, + ) + proposal_provenance = validate_proposal_dataset_provenance( + args.dataset_dir, + args.corpus_manifest, + fixture_mode=args.fixture_mode, + ) + except ProposalDatasetProvenanceError as exc: + raise SystemExit(str(exc)) from exc + proposal_provenance_path = args.dataset_dir / PROPOSAL_DATASET_PROVENANCE_NAME + proposal_provenance_sha256 = sha256(proposal_provenance_path) + import torch from torch import nn from torch.utils.data import DataLoader @@ -54,7 +322,18 @@ def main() -> int: model.load_state_dict(torch.load(state_path, map_location=device)) model.eval() torch.jit.script(model).save(str(args.output_dir / "proposal-classifier.torchscript.pt")) - print(json.dumps({"status": "exported_existing_best", "model": str(state_path)})) + print( + json.dumps( + { + "status": "exported_existing_best", + "model": str(state_path), + "proposal_dataset_provenance": str(proposal_provenance_path), + "proposal_dataset_provenance_sha256": proposal_provenance_sha256, + "governed_corpus_manifest_sha256": sha256(args.corpus_manifest), + "governed_corpus_live_recheck": True, + } + ) + ) return 0 train_loader = DataLoader(train_ds, batch_size=args.batch, shuffle=True, num_workers=0) val_loader = DataLoader(val_ds, batch_size=args.batch, shuffle=False, num_workers=0) @@ -93,11 +372,27 @@ def main() -> int: model.load_state_dict(torch.load(args.output_dir / "best-state.pt", map_location=device)) model.eval() scripted = torch.jit.script(model) - scripted.save(str(args.output_dir / "proposal-classifier.torchscript.pt")) - report = {"schema_version": 1, "status": "ok", "classes": train_ds.class_to_idx, - "train_count": len(train_ds), "validation_count": len(val_ds), - "best_validation_f1": best_f1, "history": history, - "model": str(args.output_dir / "proposal-classifier.torchscript.pt")} + model_path = args.output_dir / "proposal-classifier.torchscript.pt" + scripted.save(str(model_path)) + report = { + "schema_version": 1, + "status": "ok", + "classes": train_ds.class_to_idx, + "train_count": len(train_ds), + "validation_count": len(val_ds), + "best_validation_f1": best_f1, + "history": history, + "model": str(model_path), + "model_sha256": sha256(model_path), + "proposal_dataset_provenance": str(proposal_provenance_path), + "proposal_dataset_provenance_sha256": proposal_provenance_sha256, + "proposal_dataset_manifest_sha256": proposal_provenance["manifest_sha256"], + "corpus_manifest": str(args.corpus_manifest), + "corpus_manifest_sha256": sha256(args.corpus_manifest), + "fixture_mode": bool(args.fixture_mode), + "governed_corpus_live_recheck": True, + "governed_corpus_sample_count": len(governed_corpus.get("samples", [])), + } (args.output_dir / "training-report.json").write_text(json.dumps(report, indent=2), encoding="utf-8") return 0 diff --git a/scripts/train_building_proposal_filter.py b/scripts/train_building_proposal_filter.py index de90aac7..4c02f184 100644 --- a/scripts/train_building_proposal_filter.py +++ b/scripts/train_building_proposal_filter.py @@ -8,9 +8,23 @@ import hashlib import json import random import shutil +import sys from pathlib import Path from typing import Any, TypeVar +SCRIPT_DIR = Path(__file__).resolve().parent +if str(SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(SCRIPT_DIR)) + +from training_dataset_eligibility import ( # noqa: E402 + TrainingEligibilityError, + assert_frozen_manifest_training_eligible, +) +from training_release_manifest import ( # noqa: E402 + TrainingReleaseError, + assert_yolo_summary_bound_to_training_release, +) + T = TypeVar("T") @@ -111,6 +125,18 @@ def choose_threshold(probabilities: list[float], labels: list[int]) -> tuple[flo def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--summary", type=Path, required=True) + parser.add_argument( + "--corpus-manifest", + type=Path, + required=True, + help="Immutable governed corpus manifest that produced the tile summary.", + ) + parser.add_argument( + "--train-yaml", + type=Path, + required=True, + help="YOLO dataset YAML with an eligible immutable GeoIntel training release.", + ) parser.add_argument("--proposal-model", type=Path, required=True) parser.add_argument("--output-dir", type=Path, required=True) parser.add_argument("--device", default="cuda:0") @@ -124,7 +150,31 @@ def main() -> int: parser.add_argument("--proposal-chunk-size", type=int, default=16) parser.add_argument("--seed", type=int, default=20260727) parser.add_argument("--force", action="store_true") + parser.add_argument( + "--fixture-mode", + action="store_true", + help="Accept only an explicitly fixture-only corpus manifest; never use for operational training.", + ) args = parser.parse_args() + try: + assert_frozen_manifest_training_eligible( + args.corpus_manifest, + fixture_mode=args.fixture_mode, + verify_live=True, + ) + assert_yolo_summary_bound_to_training_release( + summary_path=args.summary, + train_yaml=args.train_yaml, + corpus_manifest=args.corpus_manifest, + fixture_mode=args.fixture_mode, + ) + except (TrainingEligibilityError, TrainingReleaseError) as exc: + raise SystemExit(str(exc)) from exc + summary = json.loads(args.summary.read_text(encoding="utf-8")) + if summary.get("source_manifest_sha256") != sha256(args.corpus_manifest): + raise SystemExit( + "Proposal-filter tile summary is not bound to the supplied governed corpus manifest" + ) if args.output_dir.exists(): if not args.force: raise SystemExit(f"Output exists: {args.output_dir}") @@ -133,12 +183,11 @@ def main() -> int: import torch from torch import nn from torch.utils.data import DataLoader - from torchvision import datasets, transforms + from torchvision import datasets from torchvision.models import ResNet18_Weights, resnet18 from ultralytics import YOLO torch.manual_seed(args.seed) - summary = json.loads(args.summary.read_text(encoding="utf-8")) split_tiles = { split: [tile for tile in summary["tiles"] if tile.get("kept", True) and tile["split"] == split] for split in ("train", "val") @@ -214,6 +263,9 @@ def main() -> int: evidence = { "schema_version": 1, "status": "ok", "architecture": "resnet18_binary_proposal_filter", "summary": str(args.summary), "summary_sha256": sha256(args.summary), + "corpus_manifest": str(args.corpus_manifest), + "corpus_manifest_sha256": sha256(args.corpus_manifest), + "fixture_mode": bool(args.fixture_mode), "proposal_model": str(args.proposal_model), "proposal_model_sha256": sha256(args.proposal_model), "device": args.device, "proposal_confidence": args.proposal_confidence, "crop_scale": args.crop_scale, "proposal_chunk_size": args.proposal_chunk_size, diff --git a/scripts/train_operator_yolo_detector.sh b/scripts/train_operator_yolo_detector.sh index 36cfcceb..76a042ce 100644 --- a/scripts/train_operator_yolo_detector.sh +++ b/scripts/train_operator_yolo_detector.sh @@ -53,6 +53,8 @@ if [[ -z "${PYTHON_BIN:-}" ]]; then fi fi +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + DATASET_YAML="${OPERATOR_YOLO_DATASET_DIR%/}/dataset.yaml" SUMMARY_PATH="${TRAIN_OUTPUT_DIR%/}/${TRAIN_RUN_NAME}/training_summary.json" export DATASET_YAML @@ -79,6 +81,11 @@ if [[ ! -f "${YOLO_BASE_MODEL_PATH}" ]]; then exit 1 fi +# A filename is not a training identity. Verify the immutable YAML/corpus/ +# asset/review release before importing Ultralytics or allocating CUDA work. +"${PYTHON_BIN}" "${SCRIPT_DIR}/training_release_manifest.py" verify \ + --train-yaml "${DATASET_YAML}" + mkdir -p "${TRAIN_OUTPUT_DIR}" "$(dirname "${TRAIN_MODEL_OUTPUT_PATH}")" "${PYTHON_BIN}" - <<'PY' diff --git a/scripts/training_dataset_eligibility.py b/scripts/training_dataset_eligibility.py new file mode 100644 index 00000000..eccdbfed --- /dev/null +++ b/scripts/training_dataset_eligibility.py @@ -0,0 +1,545 @@ +"""Fail-closed provenance gate for Belgian building-training inputs. + +The database data-contract validator decides whether a dataset can be stored. +This module is intentionally a second, independent gate at the point where +persisted datasets become irreversible training pairs. It has no database +side effects, making the decision reproducible in a corpus manifest and easy +to re-check before CUDA training starts. + +Legacy data may only pass through this module in explicit fixture mode. That +mode is deliberately limited to records marked as fixtures and must never be +used for an operational corpus. +""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections.abc import Mapping +from pathlib import Path +from typing import Any, Callable, Literal +from uuid import UUID + + +TRAINING_ELIGIBILITY_POLICY_VERSION = "geointel-training-source-eligibility/v1" +DatasetRole = Literal["raster", "reference"] + +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE) +_ALLOWED_SOURCE_CLASSIFICATIONS = { + "authoritative", + "corroborative", + "contextual", + "derived", +} +_FIXTURE_SOURCES = {"fixture", "test", "test_fixture", "test-fixture"} + + +class TrainingEligibilityError(ValueError): + """Raised when persisted inputs cannot be used in an operational corpus.""" + + +class TrainingEligibilityResult: + """Stable, JSON-serializable decision for one persisted dataset.""" + + __slots__ = ("role", "eligible", "fixture_mode", "reasons", "evidence") + + def __init__( + self, + *, + role: DatasetRole, + eligible: bool, + fixture_mode: bool, + reasons: tuple[str, ...], + evidence: dict[str, Any], + ) -> None: + self.role = role + self.eligible = eligible + self.fixture_mode = fixture_mode + self.reasons = reasons + self.evidence = evidence + + def as_dict(self) -> dict[str, Any]: + return { + "policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION, + "role": self.role, + "eligible": self.eligible, + "fixture_mode": self.fixture_mode, + "reasons": list(self.reasons), + "evidence": self.evidence, + } + + +def _value(record: Any, field: str, default: Any = None) -> Any: + if isinstance(record, Mapping): + return record.get(field, default) + return getattr(record, field, default) + + +def _normalise_status(value: Any) -> str: + return str(value or "").strip().lower() + + +def _normalise_mapping(value: Any) -> dict[str, Any]: + return dict(value) if isinstance(value, Mapping) else {} + + +def _source_identity(dataset: Any) -> tuple[str, str]: + return ( + _normalise_status(_value(dataset, "source")), + _normalise_status(_value(dataset, "source_name")), + ) + + +def _is_explicit_fixture(dataset: Any, registry: Any) -> bool: + source, source_name = _source_identity(dataset) + metadata = _normalise_mapping(_value(dataset, "metadata_json")) + provenance = _normalise_mapping(_value(dataset, "provenance_metadata")) + usage_policy = _normalise_mapping(_value(registry, "usage_policy_json")) + return bool( + source in _FIXTURE_SOURCES + or source_name in _FIXTURE_SOURCES + or metadata.get("fixture") is True + or provenance.get("fixture") is True + or usage_policy.get("fixture_only") is True + ) + + +def _dataset_evidence(dataset: Any, registry: Any, snapshot: Any) -> dict[str, Any]: + usage_policy = _normalise_mapping(_value(registry, "usage_policy_json")) + allowed_tasks = usage_policy.get("allowed_tasks") + validation_authority = _normalise_mapping(usage_policy.get("validation_authority")) + return { + "dataset_id": str(_value(dataset, "id") or ""), + "dataset_type": _normalise_status(_value(dataset, "dataset_type")), + "dataset_role": _normalise_status(_value(dataset, "dataset_role")), + "source": _normalise_status(_value(dataset, "source")), + "source_name": _normalise_status(_value(dataset, "source_name")), + "checksum_sha256": _value(dataset, "checksum_sha256"), + "data_contract_key": _value(dataset, "data_contract_key"), + "data_contract_version": _value(dataset, "data_contract_version"), + "validation_status": _normalise_status(_value(dataset, "validation_status")), + "provenance_status": _normalise_status(_value(dataset, "provenance_status")), + "lineage_status": _normalise_status(_value(dataset, "lineage_status")), + "quarantine_status": _normalise_status(_value(dataset, "quarantine_status")), + "dataset_status": _normalise_status(_value(dataset, "status")), + "dataset_source_registry_id": str(_value(dataset, "source_registry_id") or ""), + "dataset_source_snapshot_id": str(_value(dataset, "source_snapshot_id") or ""), + "source_registry_id": str(_value(registry, "id") or ""), + "source_key": _normalise_status(_value(registry, "source_key")), + "source_classification": _normalise_status(_value(registry, "classification")), + "source_freshness_status": _normalise_status(_value(registry, "freshness_status")), + "source_ingest_status": _normalise_status(_value(registry, "ingest_status")), + "source_training_allowed": usage_policy.get("training_allowed"), + "source_ground_truth_allowed": usage_policy.get("ground_truth_allowed"), + "source_allowed_tasks": list(allowed_tasks) if isinstance(allowed_tasks, list) else [], + "source_validation_authority": validation_authority, + "source_building_validation_authority": validation_authority.get("building_validation"), + "source_snapshot_id": str(_value(snapshot, "id") or _value(dataset, "source_snapshot_id") or ""), + "snapshot_source_registry_id": str(_value(snapshot, "source_registry_id") or ""), + "snapshot_key": _value(snapshot, "snapshot_key"), + "snapshot_checksum_sha256": _value(snapshot, "checksum_sha256"), + "snapshot_freshness_status": _normalise_status(_value(snapshot, "freshness_status")), + "snapshot_ingest_status": _normalise_status(_value(snapshot, "ingest_status")), + } + + +def evaluate_dataset_training_eligibility( + dataset: Any, + *, + role: DatasetRole, + fixture_mode: bool = False, +) -> TrainingEligibilityResult: + """Evaluate a source Dataset without mutating it. + + Operational calls require server-attested source registry and snapshot + relationships. ``fixture_mode`` can relax only legacy provenance fields, + and only for an explicitly marked fixture. A failed validation or a + quarantine is never relaxed. + """ + + registry = _value(dataset, "source_registry") + snapshot = _value(dataset, "source_snapshot") + evidence = _dataset_evidence(dataset, registry, snapshot) + reasons: list[str] = [] + + dataset_type = evidence["dataset_type"] + dataset_role = evidence["dataset_role"] + if role == "raster" and dataset_type != "raster": + reasons.append("dataset_type_not_raster") + if role == "reference": + if dataset_type != "vector": + reasons.append("dataset_type_not_vector") + if dataset_role != "reference": + reasons.append("dataset_role_not_reference") + + if evidence["dataset_status"] != "ready": + reasons.append("dataset_not_ready") + if evidence["quarantine_status"] == "quarantined": + reasons.append("dataset_quarantined") + + explicit_fixture = _is_explicit_fixture(dataset, registry) + if fixture_mode and not explicit_fixture: + reasons.append("fixture_mode_requires_explicit_fixture") + validation_status = evidence["validation_status"] + if validation_status == "failed": + reasons.append("validation_failed") + elif validation_status != "passed" and not (fixture_mode and explicit_fixture): + reasons.append("validation_not_passed") + + if not fixture_mode: + if evidence["provenance_status"] != "complete": + reasons.append("provenance_not_complete") + if evidence["lineage_status"] not in {"complete", "not_applicable"}: + reasons.append("lineage_not_complete") + if not evidence["data_contract_key"] or not evidence["data_contract_version"]: + reasons.append("data_contract_not_versioned") + checksum = str(evidence["checksum_sha256"] or "") + if not _SHA256_RE.fullmatch(checksum): + reasons.append("dataset_checksum_invalid") + if registry is None: + reasons.append("source_registry_missing") + if snapshot is None: + reasons.append("source_snapshot_missing") + if registry is not None: + if ( + evidence["dataset_source_registry_id"] + and evidence["dataset_source_registry_id"] != evidence["source_registry_id"] + ): + reasons.append("dataset_source_registry_binding_mismatch") + classification = evidence["source_classification"] + if classification not in _ALLOWED_SOURCE_CLASSIFICATIONS: + reasons.append("source_classification_not_allowed") + if evidence["source_training_allowed"] is not True: + reasons.append("source_not_allowed_for_training") + if evidence["source_ingest_status"] == "quarantined": + reasons.append("source_registry_quarantined") + if snapshot is not None: + if ( + evidence["dataset_source_snapshot_id"] + and evidence["dataset_source_snapshot_id"] != evidence["source_snapshot_id"] + ): + reasons.append("dataset_source_snapshot_binding_mismatch") + if ( + registry is not None + and evidence["snapshot_source_registry_id"] + and evidence["snapshot_source_registry_id"] != evidence["source_registry_id"] + ): + reasons.append("source_snapshot_registry_mismatch") + if evidence["snapshot_ingest_status"] != "ingested": + reasons.append("source_snapshot_not_ingested") + if evidence["snapshot_freshness_status"] not in {"current", "not_applicable"}: + reasons.append("source_snapshot_freshness_not_approved") + snapshot_checksum = str(evidence["snapshot_checksum_sha256"] or "") + if not _SHA256_RE.fullmatch(snapshot_checksum): + reasons.append("source_snapshot_checksum_invalid") + elif snapshot_checksum.lower() != checksum.lower(): + reasons.append("source_snapshot_checksum_mismatch") + if role == "reference": + if evidence["source_classification"] != "authoritative": + reasons.append("reference_source_not_authoritative") + if evidence["source_ground_truth_allowed"] is not True: + reasons.append("reference_source_not_ground_truth_allowed") + if "building_validation" not in evidence["source_allowed_tasks"]: + reasons.append("reference_source_not_approved_for_building_validation") + if evidence["source_building_validation_authority"] != "primary": + reasons.append("reference_building_validation_not_primary") + + return TrainingEligibilityResult( + role=role, + eligible=not reasons, + fixture_mode=fixture_mode, + reasons=tuple(sorted(set(reasons))), + evidence=evidence, + ) + + +def assert_dataset_training_eligible( + dataset: Any, + *, + role: DatasetRole, + fixture_mode: bool = False, + sample_slug: str | None = None, +) -> TrainingEligibilityResult: + result = evaluate_dataset_training_eligibility(dataset, role=role, fixture_mode=fixture_mode) + if result.eligible: + return result + label = f" for sample {sample_slug!r}" if sample_slug else "" + raise TrainingEligibilityError( + f"{role} dataset is not eligible for training{label}: {', '.join(result.reasons)}" + ) + + +def training_pair_evidence( + *, + raster: Any, + reference: Any, + fixture_mode: bool = False, +) -> dict[str, Any]: + """Return manifest-ready evidence for one raster/reference pair.""" + + raster_result = evaluate_dataset_training_eligibility( + raster, + role="raster", + fixture_mode=fixture_mode, + ) + reference_result = evaluate_dataset_training_eligibility( + reference, + role="reference", + fixture_mode=fixture_mode, + ) + return { + "policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION, + "eligible": raster_result.eligible and reference_result.eligible, + "fixture_mode": fixture_mode, + "raster": raster_result.as_dict(), + "reference": reference_result.as_dict(), + } + + +def manifest_training_eligibility_failures( + manifest: Mapping[str, Any], + *, + fixture_mode: bool = False, +) -> list[str]: + """Re-check immutable eligibility evidence before an actual training run.""" + + failures: list[str] = [] + eligibility = manifest.get("training_eligibility") + if not isinstance(eligibility, Mapping): + return ["manifest_training_eligibility_missing"] + if eligibility.get("policy_version") != TRAINING_ELIGIBILITY_POLICY_VERSION: + failures.append("manifest_training_eligibility_policy_invalid") + if eligibility.get("status") != "eligible": + failures.append("manifest_training_eligibility_not_eligible") + manifest_fixture_mode = eligibility.get("fixture_mode") is True + if manifest_fixture_mode != fixture_mode: + failures.append("manifest_fixture_mode_mismatch") + + samples = manifest.get("samples") + if not isinstance(samples, list) or not samples: + failures.append("manifest_samples_missing") + return failures + for sample in samples: + if not isinstance(sample, Mapping): + failures.append("manifest_sample_invalid") + continue + slug = str(sample.get("sample_slug") or "") + pair = sample.get("training_eligibility") + if not isinstance(pair, Mapping): + failures.append(f"{slug}:training_eligibility_missing") + continue + if pair.get("policy_version") != TRAINING_ELIGIBILITY_POLICY_VERSION: + failures.append(f"{slug}:training_eligibility_policy_invalid") + if pair.get("fixture_mode") is not fixture_mode: + failures.append(f"{slug}:fixture_mode_mismatch") + if pair.get("eligible") is not True: + failures.append(f"{slug}:training_pair_not_eligible") + for role in ("raster", "reference"): + decision = pair.get(role) + if not isinstance(decision, Mapping): + failures.append(f"{slug}:{role}_eligibility_missing") + continue + if decision.get("eligible") is not True: + failures.append(f"{slug}:{role}_not_eligible") + reasons = decision.get("reasons") + if isinstance(reasons, list) and reasons: + failures.append(f"{slug}:{role}_has_rejection_reasons") + return sorted(set(failures)) + + +def _default_live_dataset_access() -> tuple[Callable[[], Any], type[Any]]: + """Load the database dependencies only for an operational live re-check. + + This module is also imported by pure filesystem tooling and fixture tests. + Keeping the import lazy means those paths do not accidentally open a + database connection, while a normal release/verify invocation still fails + closed if the live registry cannot be checked. + """ + + import sys + + repo_root = Path(__file__).resolve().parents[1] + app_root = repo_root / "backend" + if str(app_root) not in sys.path: + sys.path.insert(0, str(app_root)) + from app.db.session import SessionLocal + from app.models import Dataset + + return SessionLocal, Dataset + + +def live_manifest_training_eligibility_failures( + manifest: Mapping[str, Any], + *, + fixture_mode: bool = False, + session_factory: Callable[[], Any] | None = None, + dataset_model: type[Any] | None = None, +) -> list[str]: + """Re-evaluate every frozen corpus parent against the live database. + + A frozen manifest proves what was eligible when it was created, not what + remains eligible now. Operational releases therefore resolve the exact + raster/reference Dataset ids again immediately before seal, retry, resume + and CUDA training. A later quarantine, failed contract, snapshot change + or missing record is a revocation and cannot be masked by the old manifest. + + Fixture-only corpora deliberately have no operational authority and are + never used for a production release; their isolated tests may skip this + live database boundary. + """ + + if fixture_mode: + return [] + failures: list[str] = [] + samples = manifest.get("samples") + if not isinstance(samples, list) or not samples: + return ["live_manifest_samples_missing"] + if session_factory is None or dataset_model is None: + try: + default_factory, default_model = _default_live_dataset_access() + except Exception: + return ["live_training_dataset_access_unavailable"] + session_factory = session_factory or default_factory + dataset_model = dataset_model or default_model + + try: + db = session_factory() + except Exception: + return ["live_training_dataset_access_unavailable"] + try: + for sample in samples: + if not isinstance(sample, Mapping): + failures.append("live_manifest_sample_invalid") + continue + slug = str(sample.get("sample_slug") or "") + recorded_pair = sample.get("training_eligibility") + for role, field_name in (("raster", "raster_dataset_id"), ("reference", "reference_dataset_id")): + raw_id = sample.get(field_name) + if not isinstance(raw_id, str) or not raw_id.strip(): + failures.append(f"{slug}:{role}_dataset_id_missing_for_live_check") + continue + try: + dataset_id = UUID(raw_id) + except (TypeError, ValueError, AttributeError): + failures.append(f"{slug}:{role}_dataset_id_invalid_for_live_check") + continue + try: + dataset = db.get(dataset_model, dataset_id) + except Exception: + failures.append(f"{slug}:{role}_live_lookup_failed") + continue + if dataset is None: + failures.append(f"{slug}:{role}_live_dataset_missing") + continue + result = evaluate_dataset_training_eligibility( + dataset, + role=role, # type: ignore[arg-type] + fixture_mode=False, + ) + if not result.eligible: + for reason in result.reasons: + failures.append(f"{slug}:{role}_live_revoked:{reason}") + if isinstance(recorded_pair, Mapping): + recorded_role = recorded_pair.get(role) + if isinstance(recorded_role, Mapping): + recorded_evidence = recorded_role.get("evidence") + if isinstance(recorded_evidence, Mapping) and str(recorded_evidence.get("dataset_id") or "") != raw_id: + failures.append(f"{slug}:{role}_manifest_dataset_binding_mismatch") + finally: + close = getattr(db, "close", None) + if callable(close): + close() + return sorted(set(failures)) + + +def assert_manifest_training_eligible( + manifest: Mapping[str, Any], + *, + fixture_mode: bool = False, +) -> None: + failures = manifest_training_eligibility_failures(manifest, fixture_mode=fixture_mode) + if failures: + raise TrainingEligibilityError( + "Corpus manifest is not eligible for training: " + ", ".join(failures) + ) + + +def _file_sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def frozen_manifest_training_eligibility_failures( + manifest_path: Path, + *, + fixture_mode: bool = False, + verify_live: bool = False, + session_factory: Callable[[], Any] | None = None, + dataset_model: type[Any] | None = None, +) -> list[str]: + """Verify an immutable manifest and its source-eligibility decision together.""" + + failures: list[str] = [] + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return ["corpus_manifest_unreadable"] + if not isinstance(manifest, Mapping): + return ["corpus_manifest_invalid"] + failures.extend(manifest_training_eligibility_failures(manifest, fixture_mode=fixture_mode)) + + freeze_path = manifest_path.parent / "corpus-freeze.json" + try: + freeze = json.loads(freeze_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return sorted(set(failures + ["corpus_freeze_missing_or_invalid"])) + if not isinstance(freeze, Mapping): + return sorted(set(failures + ["corpus_freeze_missing_or_invalid"])) + if freeze.get("immutable") is not True: + failures.append("corpus_freeze_not_immutable") + if freeze.get("manifest_sha256") != _file_sha256(manifest_path): + failures.append("corpus_manifest_checksum_mismatch") + if freeze.get("training_eligibility_policy") != TRAINING_ELIGIBILITY_POLICY_VERSION: + failures.append("corpus_freeze_policy_invalid") + if bool(freeze.get("fixture_mode")) != fixture_mode: + failures.append("corpus_freeze_fixture_mode_mismatch") + if verify_live: + failures.extend( + live_manifest_training_eligibility_failures( + manifest, + fixture_mode=fixture_mode, + session_factory=session_factory, + dataset_model=dataset_model, + ) + ) + return sorted(set(failures)) + + +def assert_frozen_manifest_training_eligible( + manifest_path: Path, + *, + fixture_mode: bool = False, + verify_live: bool = False, + session_factory: Callable[[], Any] | None = None, + dataset_model: type[Any] | None = None, +) -> dict[str, Any]: + """Load only a frozen, policy-valid corpus manifest for a training entrypoint.""" + + failures = frozen_manifest_training_eligibility_failures( + manifest_path, + fixture_mode=fixture_mode, + verify_live=verify_live, + session_factory=session_factory, + dataset_model=dataset_model, + ) + if failures: + raise TrainingEligibilityError( + "Corpus manifest is not eligible for training: " + ", ".join(failures) + ) + payload = json.loads(manifest_path.read_text(encoding="utf-8")) + assert isinstance(payload, dict) + return payload diff --git a/scripts/training_release_manifest.py b/scripts/training_release_manifest.py new file mode 100644 index 00000000..ab85f0a7 --- /dev/null +++ b/scripts/training_release_manifest.py @@ -0,0 +1,1166 @@ +#!/usr/bin/env python3 +"""Seal and verify immutable YOLO training-release inputs. + +Training data is deliberately treated as a release artifact rather than a +mutable ``dataset.yaml`` file. The sidecars written by this module bind the +exact YAML bytes, every train/validation image and matching label, the frozen +corpus manifest and the human-review decision evidence. Verification re-hashes +all of those inputs immediately before a training or resume command can run. + +The module avoids a YAML dependency on purpose. GeoIntel-generated YOLO YAML +files use a small, auditable top-level scalar subset (``path``, ``train`` and +``val``); unsupported YAML shapes fail closed instead of being guessed. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from collections.abc import Mapping +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable + +from training_dataset_eligibility import ( + TrainingEligibilityError, + assert_frozen_manifest_training_eligible, + frozen_manifest_training_eligibility_failures, +) + + +TRAINING_RELEASE_CONTRACT_VERSION = "geointel-training-release/v1" +TRAINING_RELEASE_FREEZE_VERSION = "geointel-training-release-freeze/v1" +TRAINING_ASSET_MANIFEST_VERSION = "geointel-yolo-training-assets/v1" +TRAINING_LABEL_CONTRACT_MANIFEST_VERSION = "geointel-yolo-training-label-contracts/v1" +_SHA256_RE = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE) +_IMAGE_SUFFIXES = {".bmp", ".jpeg", ".jpg", ".png", ".tif", ".tiff", ".webp"} + + +class TrainingReleaseError(ValueError): + """Raised when a training release is absent, mutable or incomplete.""" + + +def file_sha256(path: Path) -> str: + """Return the SHA-256 of one regular file.""" + + digest = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _canonical_path(path: Path) -> str: + return str(path.expanduser().resolve(strict=False)) + + +def _canonical_json_bytes(value: Any) -> bytes: + return (json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode( + "utf-8" + ) + + +def _write_immutable_json(path: Path, payload: Mapping[str, Any]) -> None: + """Create an immutable sidecar, or accept an identical idempotent rerun.""" + + encoded = json.dumps(dict(payload), ensure_ascii=False, indent=2, sort_keys=True) + "\n" + if path.exists(): + if path.read_text(encoding="utf-8") != encoded: + raise TrainingReleaseError( + f"Immutable training-release artifact already exists with different content: {path}" + ) + return + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(encoded, encoding="utf-8") + temporary.replace(path) + + +def training_release_paths(train_yaml: Path) -> dict[str, Path]: + """Return the only accepted sidecar locations for a dataset YAML.""" + + yaml_path = train_yaml.resolve(strict=False) + return { + "release_manifest": yaml_path.with_name(yaml_path.name + ".geointel-training-release.json"), + "release_freeze": yaml_path.with_name(yaml_path.name + ".geointel-training-release-freeze.json"), + "asset_manifest": yaml_path.with_name(yaml_path.name + ".geointel-training-assets.json"), + "label_contract_manifest": yaml_path.with_name(yaml_path.name + ".geointel-training-label-contracts.json"), + } + + +def _yaml_scalar(raw: str, *, field: str, yaml_path: Path) -> str: + value = raw.strip() + if not value: + raise TrainingReleaseError(f"YOLO YAML field {field!r} is empty: {yaml_path}") + if value.startswith(("[", "{", "|", ">", "&", "*", "!")): + raise TrainingReleaseError( + f"YOLO YAML field {field!r} uses an unsupported non-scalar form: {yaml_path}" + ) + if len(value) >= 2 and value[0] == value[-1] and value[0] in {"'", '"'}: + value = value[1:-1] + if " #" in value: + value = value.split(" #", 1)[0].rstrip() + if not value: + raise TrainingReleaseError(f"YOLO YAML field {field!r} is empty: {yaml_path}") + return value + + +def parse_yolo_dataset_yaml(train_yaml: Path) -> dict[str, str]: + """Parse the intentionally small, generated YOLO YAML contract. + + The parser accepts only direct top-level scalar ``path``, ``train`` and + ``val`` keys. It never tries to infer a list, anchor or nested YAML shape. + Such a file must be converted to an explicit generated release first. + """ + + try: + raw_text = train_yaml.read_text(encoding="utf-8") + except OSError as exc: + raise TrainingReleaseError(f"Training YAML is unreadable: {train_yaml}") from exc + fields: dict[str, str] = {} + for line_number, raw_line in enumerate(raw_text.splitlines(), start=1): + if not raw_line.strip() or raw_line.lstrip().startswith("#"): + continue + if raw_line[:1].isspace(): + continue + key, separator, value = raw_line.partition(":") + if not separator: + continue + field = key.strip() + if field not in {"path", "train", "val"}: + continue + if field in fields: + raise TrainingReleaseError( + f"YOLO YAML field {field!r} occurs more than once at line {line_number}: {train_yaml}" + ) + fields[field] = _yaml_scalar(value, field=field, yaml_path=train_yaml) + missing = sorted({"train", "val"} - fields.keys()) + if missing: + raise TrainingReleaseError( + f"YOLO YAML is missing required field(s) {', '.join(missing)}: {train_yaml}" + ) + return fields + + +def _resolve_dataset_reference(raw: str, *, yaml_path: Path, dataset_root: Path) -> Path: + path = Path(raw).expanduser() + if path.is_absolute(): + return path.resolve(strict=False) + return (dataset_root / path).resolve(strict=False) + + +def _resolve_list_image(raw: str, *, list_path: Path, dataset_root: Path) -> Path: + candidate = Path(raw).expanduser() + if candidate.is_absolute(): + return candidate.resolve(strict=False) + list_relative = (list_path.parent / candidate).resolve(strict=False) + root_relative = (dataset_root / candidate).resolve(strict=False) + if list_relative.is_file(): + return list_relative + if root_relative.is_file(): + return root_relative + return list_relative + + +def _label_path_for_image(image_path: Path) -> Path: + parts = list(image_path.parts) + image_index = next( + (index for index in range(len(parts) - 1, -1, -1) if parts[index].lower() == "images"), + None, + ) + if image_index is None: + return image_path.with_suffix(".txt") + parts[image_index] = "labels" + return Path(*parts).with_suffix(".txt") + + +def _image_paths_for_split( + *, + split: str, + source_path: Path, + dataset_root: Path, +) -> tuple[list[Path], dict[str, Any]]: + if source_path.is_dir(): + images = sorted( + (candidate.resolve(strict=False) for candidate in source_path.rglob("*") if candidate.is_file() and candidate.suffix.lower() in _IMAGE_SUFFIXES), + key=lambda candidate: _canonical_path(candidate), + ) + source = {"kind": "directory", "path": _canonical_path(source_path)} + elif source_path.is_file() and source_path.suffix.lower() == ".txt": + image_lines = [line.strip() for line in source_path.read_text(encoding="utf-8").splitlines() if line.strip()] + images = [ + _resolve_list_image(line, list_path=source_path, dataset_root=dataset_root) + for line in image_lines + ] + source = { + "kind": "list", + "path": _canonical_path(source_path), + "sha256": file_sha256(source_path), + "entry_count": len(images), + } + elif source_path.is_file() and source_path.suffix.lower() in _IMAGE_SUFFIXES: + images = [source_path.resolve(strict=False)] + source = {"kind": "image", "path": _canonical_path(source_path)} + else: + raise TrainingReleaseError( + f"YOLO YAML {split!r} source is neither an image directory, a list nor an image: {source_path}" + ) + if not images: + raise TrainingReleaseError(f"YOLO YAML {split!r} source contains no images: {source_path}") + return images, source + + +def build_yolo_asset_manifest(train_yaml: Path) -> dict[str, Any]: + """Return deterministic hashes for every exact train/validation pair.""" + + yaml_path = train_yaml.resolve(strict=False) + if not yaml_path.is_file(): + raise TrainingReleaseError(f"Training YAML does not exist: {yaml_path}") + yaml_fields = parse_yolo_dataset_yaml(yaml_path) + root_field = yaml_fields.get("path") + dataset_root = ( + _resolve_dataset_reference(root_field, yaml_path=yaml_path, dataset_root=yaml_path.parent) + if root_field is not None + else yaml_path.parent.resolve(strict=False) + ) + entries: list[dict[str, Any]] = [] + split_sources: dict[str, dict[str, Any]] = {} + counts: dict[str, int] = {} + for split in ("train", "val"): + source_path = _resolve_dataset_reference( + yaml_fields[split], + yaml_path=yaml_path, + dataset_root=dataset_root, + ) + images, source = _image_paths_for_split( + split=split, + source_path=source_path, + dataset_root=dataset_root, + ) + split_sources[split] = source + counts[split] = len(images) + for entry_index, image_path in enumerate(images): + if not image_path.is_file() or image_path.suffix.lower() not in _IMAGE_SUFFIXES: + raise TrainingReleaseError( + f"YOLO {split!r} entry is not a supported readable image: {image_path}" + ) + label_path = _label_path_for_image(image_path) + if not label_path.is_file(): + raise TrainingReleaseError( + f"YOLO {split!r} image has no matching label file: {image_path} -> {label_path}" + ) + entries.append( + { + "split": split, + "entry_index": entry_index, + "image_path": _canonical_path(image_path), + "image_sha256": file_sha256(image_path), + "label_path": _canonical_path(label_path), + "label_sha256": file_sha256(label_path), + } + ) + payload: dict[str, Any] = { + "schema_version": 1, + "contract_version": TRAINING_ASSET_MANIFEST_VERSION, + "dataset_yaml_path": _canonical_path(yaml_path), + "dataset_yaml_sha256": file_sha256(yaml_path), + "dataset_root": _canonical_path(dataset_root), + "split_sources": split_sources, + "counts": counts, + "entries": entries, + } + payload["asset_manifest_sha256"] = hashlib.sha256(_canonical_json_bytes(payload)).hexdigest() + return payload + + +def _label_contract_dependencies() -> tuple[Any, Any, Any, Any]: + """Import backend-only contract helpers only when a release is sealed. + + The script is invoked from both the repository and the container where the + backend package can live at a different relative path. A lazy import + keeps the small YAML/hash inspection functions usable without opening a + backend dependency, while operational releases cannot bypass validation. + """ + + import sys + + repo_root = Path(__file__).resolve().parents[1] + app_root = repo_root if (repo_root / "app").is_dir() else repo_root / "backend" + if str(app_root) not in sys.path: + sys.path.insert(0, str(app_root)) + try: + from app.services.data_contract_validation import ( + LineageEvidence, + TransformationEvidence, + build_label_validation_input, + validate_registered_asset, + ) + except Exception as exc: # pragma: no cover - deployment dependency failure + raise TrainingReleaseError("Versioned label-contract validator is unavailable") from exc + return LineageEvidence, TransformationEvidence, build_label_validation_input, validate_registered_asset + + +def _parse_yolo_label_records(label_path: Path) -> tuple[bytes, tuple[dict[str, Any], ...]]: + """Parse one YOLO text label exactly; blank content is a candidate negative.""" + + try: + raw = label_path.read_bytes() + text = raw.decode("utf-8") + except (OSError, UnicodeDecodeError) as exc: + raise TrainingReleaseError(f"YOLO label is unreadable UTF-8: {label_path}") from exc + records: list[dict[str, Any]] = [] + for line_number, line in enumerate(text.splitlines(), start=1): + stripped = line.strip() + if not stripped: + continue + fields = stripped.split() + if len(fields) != 5: + raise TrainingReleaseError( + f"YOLO label line must contain class and four coordinates: {label_path}:{line_number}" + ) + try: + class_id = int(fields[0]) + except ValueError as exc: + raise TrainingReleaseError(f"YOLO class id is invalid: {label_path}:{line_number}") from exc + try: + coordinates = [float(value) for value in fields[1:]] + except ValueError as exc: + raise TrainingReleaseError(f"YOLO coordinate is invalid: {label_path}:{line_number}") from exc + records.append( + { + "class_id": class_id, + "x_center": coordinates[0], + "y_center": coordinates[1], + "width": coordinates[2], + "height": coordinates[3], + } + ) + return raw, tuple(records) + + +def _sample_for_label_asset( + *, + label_path: Path, + split: str, + samples: list[Mapping[str, Any]], +) -> Mapping[str, Any]: + """Bind every exported label path to exactly one frozen corpus sample.""" + + stem = label_path.stem + candidates: list[Mapping[str, Any]] = [] + for sample in samples: + slug = str(sample.get("sample_slug") or "").strip() + if slug and (stem == slug or stem.startswith(f"{slug}_")): + candidates.append(sample) + if len(candidates) != 1: + raise TrainingReleaseError( + f"YOLO label cannot be bound to exactly one frozen corpus sample: {label_path}" + ) + sample = candidates[0] + sample_split = str(sample.get("split") or "").strip().lower() + if sample_split != split: + raise TrainingReleaseError( + f"YOLO label split does not match its frozen corpus sample: {label_path} ({split!r} != {sample_split!r})" + ) + return sample + + +def _accepted_review_metadata(sample_slug: str, review: Mapping[str, Any]) -> dict[str, Any]: + """Return sample-specific accepted review evidence for a pure negative.""" + + if review.get("status") != "accepted" or review.get("review_complete") is not True: + raise TrainingReleaseError( + f"Pure-background label requires accepted human review: {sample_slug}" + ) + evidence = review.get("evidence") + if not isinstance(evidence, Mapping): + raise TrainingReleaseError(f"Pure-background label review evidence is missing: {sample_slug}") + accepted = evidence.get("accepted_sample_slugs") + if not isinstance(accepted, list) or sample_slug not in {str(value) for value in accepted}: + raise TrainingReleaseError(f"Pure-background label sample was not accepted by review: {sample_slug}") + reviewers = evidence.get("reviewer_ids") + timestamps = evidence.get("reviewed_at_by_sample") + artifacts = evidence.get("reviewed_artifact_sha256_by_sample") + if ( + not isinstance(reviewers, list) + or not reviewers + or not isinstance(timestamps, Mapping) + or not isinstance(artifacts, Mapping) + ): + raise TrainingReleaseError(f"Pure-background label review evidence is incomplete: {sample_slug}") + reviewer = next((str(value).strip() for value in reviewers if isinstance(value, str) and value.strip()), "") + reviewed_at = timestamps.get(sample_slug) + artifact_sha256 = artifacts.get(sample_slug) + if not reviewer or not isinstance(reviewed_at, str) or not _is_sha256(artifact_sha256): + raise TrainingReleaseError(f"Pure-background label review evidence is invalid: {sample_slug}") + return { + "review_decision": "accepted", + "reviewer_id": reviewer, + "reviewed_at": reviewed_at, + "review_artifact_sha256": artifact_sha256, + } + + +def _source_evidence_for_label(sample: Mapping[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: + """Extract immutable parent identities from the frozen pair decision.""" + + pair = sample.get("training_eligibility") + if not isinstance(pair, Mapping) or pair.get("eligible") is not True: + raise TrainingReleaseError(f"Label sample has no eligible frozen training pair: {sample.get('sample_slug')}") + parent_evidence: dict[str, dict[str, Any]] = {} + for role, field_name in (("raster", "raster_dataset_id"), ("reference", "reference_dataset_id")): + decision = pair.get(role) + if not isinstance(decision, Mapping) or decision.get("eligible") is not True: + raise TrainingReleaseError(f"Label sample has no eligible {role} parent: {sample.get('sample_slug')}") + evidence = decision.get("evidence") + if not isinstance(evidence, Mapping): + raise TrainingReleaseError(f"Label sample has no {role} provenance evidence: {sample.get('sample_slug')}") + expected_id = str(sample.get(field_name) or "") + if not expected_id or str(evidence.get("dataset_id") or "") != expected_id: + raise TrainingReleaseError(f"Label sample {role} parent identity is not bound: {sample.get('sample_slug')}") + checksum = evidence.get("checksum_sha256") + if not _is_sha256(checksum): + raise TrainingReleaseError(f"Label sample {role} parent checksum is invalid: {sample.get('sample_slug')}") + parent_evidence[role] = dict(evidence) + return parent_evidence["raster"], parent_evidence["reference"] + + +def build_training_label_contract_manifest( + *, + corpus_manifest_path: Path, + asset_manifest: Mapping[str, Any], + review: Mapping[str, Any], + fixture_mode: bool, +) -> dict[str, Any]: + """Validate every release label under a versioned contract before training. + + Empty label text is never inferred as a negative. It receives the + `pure_background` mode only after its exact frozen sample, source parents, + split and accepted review evidence have been bound into this sidecar. + """ + + corpus = _read_json(corpus_manifest_path, label="Frozen corpus manifest") + samples_raw = corpus.get("samples") + if not isinstance(samples_raw, list) or not samples_raw: + raise TrainingReleaseError("Frozen corpus manifest contains no samples for label-contract validation") + samples = [sample for sample in samples_raw if isinstance(sample, Mapping)] + if len(samples) != len(samples_raw): + raise TrainingReleaseError("Frozen corpus manifest contains an invalid sample for label-contract validation") + entries_raw = asset_manifest.get("entries") + if not isinstance(entries_raw, list) or not entries_raw: + raise TrainingReleaseError("Training asset manifest contains no labels for label-contract validation") + asset_manifest_sha256 = asset_manifest.get("asset_manifest_sha256") + if not _is_sha256(asset_manifest_sha256): + raise TrainingReleaseError("Training asset manifest checksum is invalid for label-contract validation") + corpus_sha256 = file_sha256(corpus_manifest_path) + LineageEvidence, TransformationEvidence, build_label_validation_input, validate_registered_asset = _label_contract_dependencies() + entries: list[dict[str, Any]] = [] + for asset_entry in entries_raw: + if not isinstance(asset_entry, Mapping): + raise TrainingReleaseError("Training asset manifest contains an invalid label entry") + split = str(asset_entry.get("split") or "").strip().lower() + label_path_raw = asset_entry.get("label_path") + image_sha256 = asset_entry.get("image_sha256") + if split not in {"train", "val"} or not isinstance(label_path_raw, str) or not _is_sha256(image_sha256): + raise TrainingReleaseError("Training asset manifest label entry is incomplete") + label_path = Path(label_path_raw).expanduser().resolve(strict=False) + raw, records = _parse_yolo_label_records(label_path) + sample = _sample_for_label_asset(label_path=label_path, split=split, samples=samples) + sample_slug = str(sample.get("sample_slug") or "").strip() + raster, reference = _source_evidence_for_label(sample) + source_registry_id = reference.get("source_registry_id") + source_snapshot_id = reference.get("source_snapshot_id") + if not isinstance(source_registry_id, str) or not source_registry_id or not isinstance(source_snapshot_id, str) or not source_snapshot_id: + raise TrainingReleaseError(f"Label source registry/snapshot is not bound: {sample_slug}") + label_mode = "objects" if records else "pure_background" + metadata: dict[str, Any] = { + "image_checksum_sha256": image_sha256, + "class_ontology_version": "geointel-building-yolo/v1", + "source_corpus_manifest_sha256": corpus_sha256, + "label_mode": label_mode, + } + if label_mode == "pure_background": + metadata.update( + { + "sample_slug": sample_slug, + "split": split, + "raster_dataset_id": sample.get("raster_dataset_id"), + "reference_dataset_id": sample.get("reference_dataset_id"), + **_accepted_review_metadata(sample_slug, review), + } + ) + label_input = build_label_validation_input( + asset_id=f"yolo-label:{_canonical_path(label_path)}", + label_records=records, + label_mode=label_mode, + checksum_sha256=file_sha256(label_path), + computed_checksum_sha256=file_sha256(label_path), + content=raw, + source_registry_id=source_registry_id, + source_snapshot_id=source_snapshot_id, + imported_at=datetime.now(timezone.utc), + metadata=metadata, + temporal_unknown_reason="YOLO label inherits the governed raster/reference temporal assessment.", + source_version_unknown_reason="The immutable corpus manifest is the label release version.", + lineage=LineageEvidence( + upstream_asset_ids=(str(sample.get("raster_dataset_id")), str(sample.get("reference_dataset_id"))), + upstream_checksums_sha256=(str(raster["checksum_sha256"]), str(reference["checksum_sha256"])), + transformations=( + TransformationEvidence( + name="geointel-yolo-label-export", + version="1.0.0", + checksum_sha256=asset_manifest_sha256, + ), + ), + ), + ) + report = validate_registered_asset(label_input) + if report.validation_status.value != "passed": + issue_codes = ",".join(sorted(issue.code for issue in report.issues)) + raise TrainingReleaseError(f"YOLO label contract failed for {label_path}: {issue_codes}") + entries.append( + { + "split": split, + "entry_index": asset_entry.get("entry_index"), + "image_path": asset_entry.get("image_path"), + "image_sha256": image_sha256, + "label_path": _canonical_path(label_path), + "label_sha256": file_sha256(label_path), + "sample_slug": sample_slug, + "label_mode": label_mode, + "data_contract": { + "key": report.data_contract_key, + "version": report.data_contract_version, + "fingerprint_sha256": report.contract_fingerprint_sha256, + "validation_status": report.validation_status.value, + }, + } + ) + payload: dict[str, Any] = { + "schema_version": 1, + "contract_version": TRAINING_LABEL_CONTRACT_MANIFEST_VERSION, + "corpus_manifest_path": _canonical_path(corpus_manifest_path), + "corpus_manifest_sha256": corpus_sha256, + "asset_manifest_content_sha256": asset_manifest_sha256, + "fixture_mode": fixture_mode, + "entries": entries, + "counts": { + "total": len(entries), + "objects": sum(entry["label_mode"] == "objects" for entry in entries), + "pure_background": sum(entry["label_mode"] == "pure_background" for entry in entries), + }, + } + payload["label_contract_manifest_sha256"] = hashlib.sha256(_canonical_json_bytes(payload)).hexdigest() + return payload + + +def _is_sha256(value: Any) -> bool: + return isinstance(value, str) and _SHA256_RE.fullmatch(value) is not None + + +def _parse_utc_timestamp(value: Any) -> bool: + if not isinstance(value, str) or not value.strip(): + return False + try: + parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00")) + except ValueError: + return False + return parsed.tzinfo is not None + + +def human_review_audit_failures( + audit: Mapping[str, Any], + *, + corpus_manifest_path: Path | None = None, + corpus_manifest_sha256: str | None = None, +) -> list[str]: + """Validate the persisted, accepted human-review evidence required for training.""" + + failures: list[str] = [] + if audit.get("status") != "ok": + failures.append("review_audit_status_not_ok") + if audit.get("review_complete") is not True: + failures.append("review_complete_not_true") + if audit.get("manifest_immutable") is not True: + failures.append("review_audit_manifest_not_immutable") + if audit.get("spatial_leakage_status") != "ok": + failures.append("review_audit_spatial_leakage_not_ok") + if corpus_manifest_path is not None and audit.get("corpus_manifest_path") != _canonical_path(corpus_manifest_path): + failures.append("review_audit_corpus_manifest_path_mismatch") + if corpus_manifest_sha256 is not None and audit.get("corpus_manifest_sha256") != corpus_manifest_sha256: + failures.append("review_audit_corpus_manifest_checksum_mismatch") + evidence = audit.get("human_review_evidence") + if not isinstance(evidence, Mapping): + return sorted(set(failures + ["accepted_human_review_evidence_missing"])) + if not isinstance(evidence.get("review_decisions_path"), str) or not evidence["review_decisions_path"]: + failures.append("review_decisions_path_missing") + if not _is_sha256(evidence.get("review_decisions_sha256")): + failures.append("review_decisions_checksum_invalid") + elif isinstance(evidence.get("review_decisions_path"), str) and evidence["review_decisions_path"]: + decision_path = Path(evidence["review_decisions_path"]).expanduser().resolve(strict=False) + if not decision_path.is_file() or evidence["review_decisions_sha256"] != file_sha256(decision_path): + failures.append("review_decisions_artifact_checksum_mismatch") + required_count = evidence.get("required_sample_count") + accepted_count = evidence.get("accepted_sample_count") + if not isinstance(required_count, int) or required_count < 1: + failures.append("review_required_sample_count_invalid") + if not isinstance(accepted_count, int) or not isinstance(required_count, int) or accepted_count != required_count: + failures.append("review_accepted_sample_count_incomplete") + accepted_slugs = evidence.get("accepted_sample_slugs") + if not isinstance(accepted_slugs, list) or not isinstance(required_count, int) or len(accepted_slugs) != required_count: + failures.append("review_accepted_sample_evidence_incomplete") + reviewers = evidence.get("reviewer_ids") + if not isinstance(reviewers, list) or not reviewers or not all(isinstance(value, str) and value for value in reviewers): + failures.append("reviewer_identity_evidence_missing") + timestamps = evidence.get("reviewed_at_by_sample") + if not isinstance(timestamps, Mapping) or not isinstance(accepted_slugs, list): + failures.append("review_timestamp_evidence_missing") + elif any(not _parse_utc_timestamp(timestamps.get(str(slug))) for slug in accepted_slugs): + failures.append("review_timestamp_evidence_invalid") + artifacts = evidence.get("reviewed_artifact_sha256_by_sample") + artifact_paths = evidence.get("reviewed_artifact_path_by_sample") + if ( + not isinstance(artifacts, Mapping) + or not isinstance(artifact_paths, Mapping) + or not isinstance(accepted_slugs, list) + ): + failures.append("reviewed_artifact_evidence_missing") + else: + for slug in accepted_slugs: + artifact_hash = artifacts.get(str(slug)) + artifact_path_raw = artifact_paths.get(str(slug)) + if not _is_sha256(artifact_hash) or not isinstance(artifact_path_raw, str) or not artifact_path_raw: + failures.append("reviewed_artifact_evidence_invalid") + continue + artifact_path = Path(artifact_path_raw).expanduser().resolve(strict=False) + if not artifact_path.is_file() or artifact_hash != file_sha256(artifact_path): + failures.append("reviewed_artifact_checksum_mismatch") + return sorted(set(failures)) + + +def _read_json(path: Path, *, label: str) -> dict[str, Any]: + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise TrainingReleaseError(f"{label} is unreadable: {path}") from exc + if not isinstance(payload, dict): + raise TrainingReleaseError(f"{label} must be a JSON object: {path}") + return payload + + +def _review_evidence_for_release( + *, + review_audit_path: Path | None, + corpus_manifest_path: Path, + corpus_manifest_sha256: str, + fixture_mode: bool, +) -> dict[str, Any]: + if fixture_mode: + return { + "status": "fixture_relaxed", + "fixture_only": True, + "review_complete": False, + "reason": "fixture-only frozen corpus explicitly permits test-only review relaxation", + } + if review_audit_path is None: + raise TrainingReleaseError( + "Operational training release requires --review-audit with accepted human-review evidence" + ) + audit_path = review_audit_path.resolve(strict=False) + audit = _read_json(audit_path, label="Human-review audit") + failures = human_review_audit_failures( + audit, + corpus_manifest_path=corpus_manifest_path, + corpus_manifest_sha256=corpus_manifest_sha256, + ) + if failures: + raise TrainingReleaseError( + "Human-review audit is not eligible for operational training: " + ", ".join(failures) + ) + evidence = audit["human_review_evidence"] + assert isinstance(evidence, Mapping) + return { + "status": "accepted", + "fixture_only": False, + "review_complete": True, + "audit_path": _canonical_path(audit_path), + "audit_sha256": file_sha256(audit_path), + "evidence": dict(evidence), + } + + +def create_training_release_manifest( + *, + train_yaml: Path, + corpus_manifest: Path, + review_audit_path: Path | None = None, + fixture_mode: bool = False, + live_session_factory: Callable[[], Any] | None = None, + live_dataset_model: type[Any] | None = None, +) -> dict[str, Path]: + """Seal one exact training YAML against a frozen corpus and review decision. + + A normal release cannot be created without a passed human-review audit. + Fixture mode is accepted only after the corpus eligibility gate proves that + the source manifest *and* its freeze sidecar are explicitly fixture-only. + """ + + yaml_path = train_yaml.resolve(strict=False) + corpus_path = corpus_manifest.resolve(strict=False) + try: + assert_frozen_manifest_training_eligible( + corpus_path, + fixture_mode=fixture_mode, + verify_live=True, + session_factory=live_session_factory, + dataset_model=live_dataset_model, + ) + except TrainingEligibilityError as exc: + raise TrainingReleaseError(str(exc)) from exc + if not yaml_path.is_file(): + raise TrainingReleaseError(f"Training YAML does not exist: {yaml_path}") + paths = training_release_paths(yaml_path) + asset_payload = build_yolo_asset_manifest(yaml_path) + corpus_freeze_path = corpus_path.parent / "corpus-freeze.json" + if not corpus_freeze_path.is_file(): + raise TrainingReleaseError(f"Frozen corpus sidecar is missing: {corpus_freeze_path}") + corpus_hash = file_sha256(corpus_path) + review = _review_evidence_for_release( + review_audit_path=review_audit_path, + corpus_manifest_path=corpus_path, + corpus_manifest_sha256=corpus_hash, + fixture_mode=fixture_mode, + ) + label_contract_payload = build_training_label_contract_manifest( + corpus_manifest_path=corpus_path, + asset_manifest=asset_payload, + review=review, + fixture_mode=fixture_mode, + ) + _write_immutable_json(paths["asset_manifest"], asset_payload) + _write_immutable_json(paths["label_contract_manifest"], label_contract_payload) + release: dict[str, Any] = { + "schema_version": 1, + "contract_version": TRAINING_RELEASE_CONTRACT_VERSION, + "immutable": True, + "status": "eligible", + "fixture_mode": fixture_mode, + "dataset_yaml": { + "path": _canonical_path(yaml_path), + "sha256": file_sha256(yaml_path), + }, + "corpus": { + "manifest_path": _canonical_path(corpus_path), + "manifest_sha256": corpus_hash, + "freeze_path": _canonical_path(corpus_freeze_path), + "freeze_sha256": file_sha256(corpus_freeze_path), + }, + "asset_manifest": { + "path": _canonical_path(paths["asset_manifest"]), + "sha256": file_sha256(paths["asset_manifest"]), + "content_sha256": asset_payload["asset_manifest_sha256"], + "train_entry_count": asset_payload["counts"]["train"], + "val_entry_count": asset_payload["counts"]["val"], + }, + "label_contract_manifest": { + "path": _canonical_path(paths["label_contract_manifest"]), + "sha256": file_sha256(paths["label_contract_manifest"]), + "content_sha256": label_contract_payload["label_contract_manifest_sha256"], + "entry_count": label_contract_payload["counts"]["total"], + "pure_background_entry_count": label_contract_payload["counts"]["pure_background"], + }, + "human_review": review, + } + _write_immutable_json(paths["release_manifest"], release) + freeze = { + "schema_version": 1, + "contract_version": TRAINING_RELEASE_FREEZE_VERSION, + "immutable": True, + "release_manifest_sha256": file_sha256(paths["release_manifest"]), + "fixture_mode": fixture_mode, + "dataset_yaml_sha256": release["dataset_yaml"]["sha256"], + "corpus_manifest_sha256": corpus_hash, + "asset_manifest_sha256": release["asset_manifest"]["sha256"], + "label_contract_manifest_sha256": release["label_contract_manifest"]["sha256"], + } + _write_immutable_json(paths["release_freeze"], freeze) + return paths + + +def training_release_failures( + *, + train_yaml: Path, + corpus_manifest: Path | None = None, + fixture_mode: bool = False, + live_session_factory: Callable[[], Any] | None = None, + live_dataset_model: type[Any] | None = None, +) -> list[str]: + """Return every deterministic reason a YAML cannot enter training now.""" + + yaml_path = train_yaml.resolve(strict=False) + paths = training_release_paths(yaml_path) + failures: list[str] = [] + if not paths["release_manifest"].is_file(): + return ["training_release_manifest_missing"] + if not paths["release_freeze"].is_file(): + return ["training_release_freeze_missing"] + try: + release = _read_json(paths["release_manifest"], label="Training release manifest") + freeze = _read_json(paths["release_freeze"], label="Training release freeze") + except TrainingReleaseError as exc: + return [str(exc)] + if release.get("contract_version") != TRAINING_RELEASE_CONTRACT_VERSION: + failures.append("training_release_contract_invalid") + if release.get("immutable") is not True: + failures.append("training_release_not_immutable") + if release.get("status") != "eligible": + failures.append("training_release_not_eligible") + if release.get("fixture_mode") is not fixture_mode: + failures.append("training_release_fixture_mode_mismatch") + if freeze.get("contract_version") != TRAINING_RELEASE_FREEZE_VERSION: + failures.append("training_release_freeze_contract_invalid") + if freeze.get("immutable") is not True: + failures.append("training_release_freeze_not_immutable") + if freeze.get("fixture_mode") is not fixture_mode: + failures.append("training_release_freeze_fixture_mode_mismatch") + if freeze.get("release_manifest_sha256") != file_sha256(paths["release_manifest"]): + failures.append("training_release_manifest_checksum_mismatch") + + yaml_evidence = release.get("dataset_yaml") + if not isinstance(yaml_evidence, Mapping): + failures.append("training_release_yaml_evidence_missing") + else: + if yaml_evidence.get("path") != _canonical_path(yaml_path): + failures.append("training_release_yaml_path_mismatch") + if not yaml_path.is_file() or yaml_evidence.get("sha256") != file_sha256(yaml_path): + failures.append("training_release_yaml_checksum_mismatch") + if freeze.get("dataset_yaml_sha256") != yaml_evidence.get("sha256"): + failures.append("training_release_freeze_yaml_binding_mismatch") + + corpus = release.get("corpus") + corpus_path: Path | None = None + if not isinstance(corpus, Mapping): + failures.append("training_release_corpus_evidence_missing") + else: + declared = corpus.get("manifest_path") + if not isinstance(declared, str) or not declared: + failures.append("training_release_corpus_path_missing") + else: + corpus_path = Path(declared).expanduser().resolve(strict=False) + if corpus_manifest is not None and corpus_path != corpus_manifest.resolve(strict=False): + failures.append("training_release_corpus_path_mismatch") + if not corpus_path.is_file(): + failures.append("training_release_corpus_missing") + else: + corpus_hash = file_sha256(corpus_path) + if corpus.get("manifest_sha256") != corpus_hash: + failures.append("training_release_corpus_checksum_mismatch") + expected_freeze = corpus_path.parent / "corpus-freeze.json" + if corpus.get("freeze_path") != _canonical_path(expected_freeze): + failures.append("training_release_corpus_freeze_path_mismatch") + if not expected_freeze.is_file() or corpus.get("freeze_sha256") != file_sha256(expected_freeze): + failures.append("training_release_corpus_freeze_checksum_mismatch") + if freeze.get("corpus_manifest_sha256") != corpus_hash: + failures.append("training_release_freeze_corpus_binding_mismatch") + failures.extend( + frozen_manifest_training_eligibility_failures( + corpus_path, + fixture_mode=fixture_mode, + verify_live=True, + session_factory=live_session_factory, + dataset_model=live_dataset_model, + ) + ) + + asset_evidence = release.get("asset_manifest") + if not isinstance(asset_evidence, Mapping): + failures.append("training_release_asset_manifest_evidence_missing") + else: + if asset_evidence.get("path") != _canonical_path(paths["asset_manifest"]): + failures.append("training_release_asset_manifest_path_mismatch") + if not paths["asset_manifest"].is_file(): + failures.append("training_release_asset_manifest_missing") + else: + stored_assets = _read_json(paths["asset_manifest"], label="Training asset manifest") + asset_file_hash = file_sha256(paths["asset_manifest"]) + if asset_evidence.get("sha256") != asset_file_hash: + failures.append("training_release_asset_manifest_checksum_mismatch") + if freeze.get("asset_manifest_sha256") != asset_file_hash: + failures.append("training_release_freeze_asset_binding_mismatch") + try: + expected_assets = build_yolo_asset_manifest(yaml_path) + except TrainingReleaseError as exc: + failures.append(f"training_release_assets_unreadable:{exc}") + else: + if stored_assets != expected_assets: + failures.append("training_release_asset_manifest_content_mismatch") + if asset_evidence.get("content_sha256") != expected_assets["asset_manifest_sha256"]: + failures.append("training_release_asset_content_checksum_mismatch") + + review = release.get("human_review") + if not isinstance(review, Mapping): + failures.append("training_release_human_review_missing") + elif fixture_mode: + if review.get("status") != "fixture_relaxed" or review.get("fixture_only") is not True: + failures.append("training_release_fixture_review_relaxation_invalid") + else: + if review.get("status") != "accepted" or review.get("review_complete") is not True: + failures.append("training_release_human_review_not_accepted") + audit_path_raw = review.get("audit_path") + if not isinstance(audit_path_raw, str) or not audit_path_raw: + failures.append("training_release_review_audit_path_missing") + else: + audit_path = Path(audit_path_raw).expanduser().resolve(strict=False) + if not audit_path.is_file() or review.get("audit_sha256") != file_sha256(audit_path): + failures.append("training_release_review_audit_checksum_mismatch") + else: + audit = _read_json(audit_path, label="Human-review audit") + corpus_hash = file_sha256(corpus_path) if corpus_path and corpus_path.is_file() else None + failures.extend( + human_review_audit_failures( + audit, + corpus_manifest_path=corpus_path, + corpus_manifest_sha256=corpus_hash, + ) + ) + if review.get("evidence") != audit.get("human_review_evidence"): + failures.append("training_release_human_review_evidence_mismatch") + + label_contract_evidence = release.get("label_contract_manifest") + if not isinstance(label_contract_evidence, Mapping): + failures.append("training_release_label_contract_manifest_evidence_missing") + else: + if label_contract_evidence.get("path") != _canonical_path(paths["label_contract_manifest"]): + failures.append("training_release_label_contract_manifest_path_mismatch") + if not paths["label_contract_manifest"].is_file(): + failures.append("training_release_label_contract_manifest_missing") + else: + label_file_hash = file_sha256(paths["label_contract_manifest"]) + if label_contract_evidence.get("sha256") != label_file_hash: + failures.append("training_release_label_contract_manifest_checksum_mismatch") + if freeze.get("label_contract_manifest_sha256") != label_file_hash: + failures.append("training_release_freeze_label_contract_binding_mismatch") + try: + stored_label_contracts = _read_json( + paths["label_contract_manifest"], + label="Training label-contract manifest", + ) + except TrainingReleaseError as exc: + failures.append(f"training_release_label_contract_manifest_unreadable:{exc}") + else: + if ( + corpus_path is None + or not corpus_path.is_file() + or not isinstance(review, Mapping) + or not yaml_path.is_file() + ): + failures.append("training_release_label_contract_inputs_unavailable") + else: + try: + expected_label_contracts = build_training_label_contract_manifest( + corpus_manifest_path=corpus_path, + asset_manifest=build_yolo_asset_manifest(yaml_path), + review=review, + fixture_mode=fixture_mode, + ) + except TrainingReleaseError as exc: + failures.append(f"training_release_label_contract_invalid:{exc}") + else: + if stored_label_contracts != expected_label_contracts: + failures.append("training_release_label_contract_manifest_content_mismatch") + if ( + label_contract_evidence.get("content_sha256") + != expected_label_contracts.get("label_contract_manifest_sha256") + ): + failures.append("training_release_label_contract_content_checksum_mismatch") + counts = expected_label_contracts.get("counts") + if not isinstance(counts, Mapping): + failures.append("training_release_label_contract_counts_invalid") + else: + if label_contract_evidence.get("entry_count") != counts.get("total"): + failures.append("training_release_label_contract_entry_count_mismatch") + if label_contract_evidence.get("pure_background_entry_count") != counts.get("pure_background"): + failures.append("training_release_label_contract_background_count_mismatch") + return sorted(set(failures)) + + +def assert_training_release_eligible( + *, + train_yaml: Path, + corpus_manifest: Path | None = None, + fixture_mode: bool = False, + live_session_factory: Callable[[], Any] | None = None, + live_dataset_model: type[Any] | None = None, +) -> dict[str, Any]: + """Fail closed unless the exact YAML/corpus/assets/review release is valid.""" + + failures = training_release_failures( + train_yaml=train_yaml, + corpus_manifest=corpus_manifest, + fixture_mode=fixture_mode, + live_session_factory=live_session_factory, + live_dataset_model=live_dataset_model, + ) + if failures: + raise TrainingReleaseError( + "Training release is not eligible: " + ", ".join(failures) + ) + return _read_json(training_release_paths(train_yaml)["release_manifest"], label="Training release manifest") + + +def assert_yolo_summary_bound_to_training_release( + *, + summary_path: Path, + train_yaml: Path, + corpus_manifest: Path | None = None, + fixture_mode: bool = False, + live_session_factory: Callable[[], Any] | None = None, + live_dataset_model: type[Any] | None = None, +) -> dict[str, Any]: + """Verify that a tile summary is an exact view of an eligible release. + + Derived training tools must not accept a merely similarly named JSON + summary. This gate binds every selected image/label pair to the already + re-hashed asset manifest and requires the same immutable corpus/release. + It is deliberately strict: a transform that changes image bytes needs a + new governed corpus and release instead of inheriting production authority. + """ + + yaml_path = train_yaml.resolve(strict=False) + release = assert_training_release_eligible( + train_yaml=yaml_path, + corpus_manifest=corpus_manifest, + fixture_mode=fixture_mode, + live_session_factory=live_session_factory, + live_dataset_model=live_dataset_model, + ) + summary = _read_json(summary_path.resolve(strict=False), label="YOLO tile summary") + release_paths = training_release_paths(yaml_path) + release_manifest_path = release_paths["release_manifest"] + corpus = release.get("corpus") + assets = release.get("asset_manifest") + yaml_evidence = release.get("dataset_yaml") + if not isinstance(corpus, Mapping) or not isinstance(assets, Mapping) or not isinstance(yaml_evidence, Mapping): + raise TrainingReleaseError("Training release lacks corpus, asset or YAML evidence for summary binding") + + required_bindings = { + "dataset_yaml": _canonical_path(yaml_path), + "training_release_manifest": _canonical_path(release_manifest_path), + "training_release_manifest_sha256": file_sha256(release_manifest_path), + "source_manifest_sha256": corpus.get("manifest_sha256"), + } + for field_name, expected in required_bindings.items(): + if summary.get(field_name) != expected: + raise TrainingReleaseError( + f"YOLO tile summary {field_name!r} is not bound to the eligible training release" + ) + if yaml_evidence.get("path") != _canonical_path(yaml_path): + raise TrainingReleaseError("Training release YAML evidence does not match the supplied tile summary YAML") + if summary.get("training_asset_manifest") != _canonical_path(release_paths["asset_manifest"]): + raise TrainingReleaseError("YOLO tile summary points to a different training asset manifest") + + stored_assets = _read_json(release_paths["asset_manifest"], label="Training asset manifest") + entries = stored_assets.get("entries") + tiles = summary.get("tiles") + if not isinstance(entries, list) or not entries or not isinstance(tiles, list) or not tiles: + raise TrainingReleaseError("YOLO tile summary or training asset manifest has no entries") + expected_pairs: set[tuple[str, str, str]] = set() + for asset in entries: + if not isinstance(asset, Mapping): + raise TrainingReleaseError("Training asset manifest has an invalid entry") + split = str(asset.get("split") or "").strip().lower() + image_path = asset.get("image_path") + label_path = asset.get("label_path") + if split not in {"train", "val"} or not isinstance(image_path, str) or not isinstance(label_path, str): + raise TrainingReleaseError("Training asset manifest entry is incomplete") + expected_pairs.add((split, _canonical_path(Path(image_path)), _canonical_path(Path(label_path)))) + if len(expected_pairs) != len(entries): + raise TrainingReleaseError("Training asset manifest contains duplicate image/label pairs") + + observed_pairs: set[tuple[str, str, str]] = set() + for index, tile in enumerate(tiles): + if not isinstance(tile, Mapping): + raise TrainingReleaseError(f"YOLO tile summary entry {index} is invalid") + split = str(tile.get("split") or "").strip().lower() + image_path = tile.get("image_path") + label_path = tile.get("label_path") + if split not in {"train", "val"} or not isinstance(image_path, str) or not isinstance(label_path, str): + raise TrainingReleaseError(f"YOLO tile summary entry {index} is incomplete") + pair = (split, _canonical_path(Path(image_path)), _canonical_path(Path(label_path))) + if pair not in expected_pairs: + raise TrainingReleaseError(f"YOLO tile summary entry {index} is not in the immutable training assets") + if pair in observed_pairs: + raise TrainingReleaseError(f"YOLO tile summary repeats immutable training asset {index}") + observed_pairs.add(pair) + if observed_pairs != expected_pairs: + raise TrainingReleaseError("YOLO tile summary is not a complete immutable view of the training assets") + return release + + +def assert_yolo_summary_bound_to_embedded_training_release( + *, + summary_path: Path, + corpus_manifest: Path | None = None, + fixture_mode: bool = False, + live_session_factory: Callable[[], Any] | None = None, + live_dataset_model: type[Any] | None = None, +) -> dict[str, Any]: + """Resolve a generated summary's YAML only long enough to verify its release. + + The embedded path has no authority on its own. The delegated exact-binding + check below requires it to match the immutable release manifest and asset + manifest before a derived sampler may read any tile from the summary. + """ + + summary = _read_json(summary_path.resolve(strict=False), label="YOLO tile summary") + raw_yaml = summary.get("dataset_yaml") + if not isinstance(raw_yaml, str) or not raw_yaml.strip(): + raise TrainingReleaseError("YOLO tile summary has no dataset YAML binding") + return assert_yolo_summary_bound_to_training_release( + summary_path=summary_path, + train_yaml=Path(raw_yaml), + corpus_manifest=corpus_manifest, + fixture_mode=fixture_mode, + live_session_factory=live_session_factory, + live_dataset_model=live_dataset_model, + ) + + +def _main() -> int: + parser = argparse.ArgumentParser(description="Seal or verify an immutable GeoIntel YOLO training release.") + subparsers = parser.add_subparsers(dest="command", required=True) + create = subparsers.add_parser("create", help="Create immutable sidecars for one frozen training release.") + create.add_argument("--train-yaml", type=Path, required=True) + create.add_argument("--corpus-manifest", type=Path, required=True) + create.add_argument("--review-audit", type=Path) + create.add_argument("--fixture-mode", action="store_true") + verify = subparsers.add_parser("verify", help="Verify a release immediately before training or resume.") + verify.add_argument("--train-yaml", type=Path, required=True) + verify.add_argument("--corpus-manifest", type=Path) + verify.add_argument("--fixture-mode", action="store_true") + args = parser.parse_args() + try: + if args.command == "create": + paths = create_training_release_manifest( + train_yaml=args.train_yaml, + corpus_manifest=args.corpus_manifest, + review_audit_path=args.review_audit, + fixture_mode=args.fixture_mode, + ) + print(json.dumps({key: _canonical_path(value) for key, value in paths.items()}, indent=2)) + else: + release = assert_training_release_eligible( + train_yaml=args.train_yaml, + corpus_manifest=args.corpus_manifest, + fixture_mode=args.fixture_mode, + ) + print(json.dumps({"status": "eligible", "release": release}, indent=2)) + except (TrainingReleaseError, TrainingEligibilityError) as exc: + raise SystemExit(str(exc)) from exc + return 0 + + +if __name__ == "__main__": + raise SystemExit(_main()) diff --git a/scripts/verify_phase2_postgres_guards.py b/scripts/verify_phase2_postgres_guards.py new file mode 100644 index 00000000..a3b7a956 --- /dev/null +++ b/scripts/verify_phase2_postgres_guards.py @@ -0,0 +1,738 @@ +#!/usr/bin/env python3 +"""Exercise Phase-2 PostgreSQL migration guards on a disposable database. + +This is intentionally *not* a general migration runner. It refuses every +database whose name does not start with ``geointel_phase2_`` so it cannot be +pointed accidentally at a developer, staging or production database. The +script upgrades the complete Alembic chain, proves the new trigger guards with +real DML, and optionally downgrades again. +""" + +from __future__ import annotations + +import argparse +from contextlib import contextmanager +from datetime import UTC, datetime +import json +import os +from pathlib import Path +import subprocess +import sys +from typing import Any, Iterator +from uuid import UUID, uuid4 + +from sqlalchemy import create_engine, text +from sqlalchemy.engine import Connection, Engine, make_url +from sqlalchemy.exc import IntegrityError + + +ROOT = Path(__file__).resolve().parents[1] +BACKEND = ROOT / "backend" +SAFE_DATABASE_PREFIX = "geointel_phase2_" + + +def _json_dump(path: Path, value: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8" + ) + + +def _require_disposable_database(database_url: str) -> None: + parsed = make_url(database_url) + database_name = str(parsed.database or "") + if not database_name.startswith(SAFE_DATABASE_PREFIX): + raise SystemExit( + "Refusing migration guard test: database name must start with " + f"{SAFE_DATABASE_PREFIX!r}, received {database_name!r}." + ) + + +def _run_alembic(database_url: str, *arguments: str) -> str: + environment = dict(os.environ) + environment["DATABASE_URL"] = database_url + completed = subprocess.run( + [sys.executable, "-m", "alembic", *arguments], + cwd=BACKEND, + env=environment, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + if completed.returncode != 0: + raise RuntimeError( + f"Alembic {' '.join(arguments)} failed with exit code {completed.returncode}:\n{completed.stdout}" + ) + return completed.stdout + + +@contextmanager +def _transaction(engine: Engine) -> Iterator[Connection]: + with engine.begin() as connection: + yield connection + + +def _insert_project(connection: Connection) -> UUID: + project_id = uuid4() + connection.execute( + text("INSERT INTO projects (id, name) VALUES (:id, :name)"), + {"id": project_id, "name": "Phase 2 migration guard fixture"}, + ) + return project_id + + +def _source_id(connection: Connection, source_key: str) -> UUID: + value = connection.execute( + text("SELECT id FROM source_registry WHERE source_key = :source_key"), + {"source_key": source_key}, + ).scalar_one() + return UUID(str(value)) + + +def _insert_snapshot( + connection: Connection, source_registry_id: UUID, *, key: str, checksum: str +) -> UUID: + snapshot_id = uuid4() + connection.execute( + text( + """ + INSERT INTO source_snapshots ( + id, source_registry_id, snapshot_key, checksum_sha256, + crs, units, freshness_status, ingest_status + ) VALUES ( + :id, :source_registry_id, :snapshot_key, :checksum_sha256, + 'EPSG:4326', 'metres', 'current', 'ingested' + ) + """ + ), + { + "id": snapshot_id, + "source_registry_id": source_registry_id, + "snapshot_key": key, + "checksum_sha256": checksum, + }, + ) + return snapshot_id + + +def _accepted_report( + *, contract_key: str = "geointel.vector.geojson", contract_version: str = "1.0.0" +) -> str: + """Produce a structurally complete persisted validation report fixture.""" + + return json.dumps( + { + "asset_id": "phase2-guard-fixture", + "data_contract_key": contract_key, + "data_contract_version": contract_version, + "contract_fingerprint_sha256": "d" * 64, + "validation_status": "passed", + "provenance_status": "complete", + "lineage_status": "complete", + "quarantine_status": "not_quarantined", + "validation_scope": ["contract"], + "report_sha256": "e" * 64, + } + ) + + +def _insert_dataset( + connection: Connection, + *, + project_id: UUID, + source_registry_id: UUID, + source_snapshot_id: UUID, + suffix: str, + checksum_sha256: str, +) -> UUID: + dataset_id = uuid4() + connection.execute( + text( + """ + INSERT INTO datasets ( + id, project_id, name, dataset_type, source, source_name, + status, source_registry_id, source_snapshot_id, + data_contract_key, data_contract_version, validation_report_json, + validation_status, provenance_status, lineage_status, quarantine_status, + checksum_sha256 + ) VALUES ( + :id, :project_id, :name, 'vector', 'governed', 'grb', + 'ready', :source_registry_id, :source_snapshot_id, + 'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json), + 'passed', 'complete', 'complete', 'not_quarantined', :checksum_sha256 + ) + """ + ), + { + "id": dataset_id, + "project_id": project_id, + "name": f"phase2-{suffix}.geojson", + "source_registry_id": source_registry_id, + "source_snapshot_id": source_snapshot_id, + "validation_report_json": _accepted_report(), + "checksum_sha256": checksum_sha256, + }, + ) + return dataset_id + + +def _assert_constraint_rejection( + engine: Engine, statement: str, parameters: dict[str, Any], *, label: str +) -> None: + try: + with _transaction(engine) as connection: + connection.execute(text(statement), parameters) + except IntegrityError as exc: + sqlstate = getattr(getattr(exc, "orig", None), "sqlstate", None) + if sqlstate == "23514": + return + raise AssertionError( + f"{label} failed with unexpected SQLSTATE {sqlstate!r}" + ) from exc + raise AssertionError(f"{label} was accepted unexpectedly") + + +def verify(database_url: str, *, verify_downgrade: bool) -> dict[str, Any]: + _require_disposable_database(database_url) + started_at = datetime.now(UTC).isoformat() + upgrade_output = _run_alembic(database_url, "upgrade", "head") + engine = create_engine(database_url) + try: + edge_a = uuid4() + edge_b = uuid4() + with _transaction(engine) as connection: + project_id = _insert_project(connection) + grb_id = _source_id(connection, "grb") + osm_id = _source_id(connection, "osm") + grb_snapshot_id = _insert_snapshot( + connection, grb_id, key="phase2-grb", checksum="a" * 64 + ) + osm_snapshot_id = _insert_snapshot( + connection, osm_id, key="phase2-osm", checksum="b" * 64 + ) + derived_b_snapshot_id = _insert_snapshot( + connection, grb_id, key="phase2-derived-b", checksum="c" * 64 + ) + derived_c_snapshot_id = _insert_snapshot( + connection, grb_id, key="phase2-derived-c", checksum="d" * 64 + ) + mutation_snapshot_id = _insert_snapshot( + connection, grb_id, key="phase2-mutation", checksum="e" * 64 + ) + dataset_a = _insert_dataset( + connection, + project_id=project_id, + source_registry_id=grb_id, + source_snapshot_id=grb_snapshot_id, + suffix="a", + checksum_sha256="a" * 64, + ) + dataset_b = _insert_dataset( + connection, + project_id=project_id, + source_registry_id=grb_id, + source_snapshot_id=derived_b_snapshot_id, + suffix="b", + checksum_sha256="c" * 64, + ) + dataset_c = _insert_dataset( + connection, + project_id=project_id, + source_registry_id=grb_id, + source_snapshot_id=derived_c_snapshot_id, + suffix="c", + checksum_sha256="d" * 64, + ) + shared_dataset = _insert_dataset( + connection, + project_id=project_id, + source_registry_id=grb_id, + source_snapshot_id=grb_snapshot_id, + suffix="shared", + checksum_sha256="a" * 64, + ) + mutation_dataset = _insert_dataset( + connection, + project_id=project_id, + source_registry_id=grb_id, + source_snapshot_id=mutation_snapshot_id, + suffix="mutation", + checksum_sha256="e" * 64, + ) + version_id = uuid4() + connection.execute( + text( + """ + INSERT INTO dataset_versions ( + id, dataset_id, version, source_registry_id, source_snapshot_id, + data_contract_key, data_contract_version, validation_report_json, + validation_status, provenance_status, lineage_status, checksum_sha256 + ) VALUES ( + :id, :dataset_id, 1, :source_registry_id, :source_snapshot_id, + 'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json), + 'passed', 'complete', 'complete', :checksum_sha256 + ) + """ + ), + { + "id": version_id, + "dataset_id": dataset_a, + "source_registry_id": grb_id, + "source_snapshot_id": grb_snapshot_id, + "validation_report_json": _accepted_report(), + "checksum_sha256": "a" * 64, + }, + ) + connection.execute( + text( + """ + INSERT INTO dataset_lineage_edges ( + id, parent_dataset_id, child_dataset_id, relation_type, transformation_name + ) VALUES + (:edge_a, :dataset_a, :dataset_b, 'derived_from', 'clip'), + (:edge_b, :dataset_b, :dataset_c, 'derived_from', 'buffer') + """ + ), + { + "edge_a": edge_a, + "edge_b": edge_b, + "dataset_a": dataset_a, + "dataset_b": dataset_b, + "dataset_c": dataset_c, + }, + ) + + _assert_constraint_rejection( + engine, + """ + INSERT INTO datasets ( + id, project_id, name, dataset_type, source, source_name, status, + source_registry_id, source_snapshot_id, validation_status, + data_contract_key, data_contract_version, validation_report_json, + provenance_status, lineage_status, quarantine_status, checksum_sha256 + ) VALUES ( + :id, :project_id, 'mismatched.geojson', 'vector', 'governed', 'grb', 'ready', + :source_registry_id, :source_snapshot_id, 'passed', + 'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json), + 'complete', 'complete', 'not_quarantined', :checksum_sha256 + ) + """, + { + "id": uuid4(), + "project_id": project_id, + "source_registry_id": grb_id, + "source_snapshot_id": osm_snapshot_id, + "validation_report_json": _accepted_report(), + "checksum_sha256": "b" * 64, + }, + label="snapshot-registry mismatch guard", + ) + _assert_constraint_rejection( + engine, + """ + INSERT INTO datasets ( + id, project_id, name, dataset_type, source, source_name, status, + source_registry_id, source_snapshot_id, validation_status, + provenance_status, lineage_status, quarantine_status, checksum_sha256 + ) VALUES ( + :id, :project_id, 'unreported.geojson', 'vector', 'governed', 'grb', 'ready', + :source_registry_id, :source_snapshot_id, 'passed', 'complete', 'complete', 'not_quarantined', :checksum_sha256 + ) + """, + { + "id": uuid4(), + "project_id": project_id, + "source_registry_id": grb_id, + "source_snapshot_id": grb_snapshot_id, + "checksum_sha256": "a" * 64, + }, + label="passed contract report guard", + ) + _assert_constraint_rejection( + engine, + """ + INSERT INTO datasets ( + id, project_id, name, dataset_type, source, source_name, status, + source_registry_id, source_snapshot_id, validation_status, + data_contract_key, data_contract_version, validation_report_json, + provenance_status, lineage_status, quarantine_status + ) VALUES ( + :id, :project_id, 'missing-checksum.geojson', 'vector', 'governed', 'grb', 'ready', + :source_registry_id, :source_snapshot_id, 'passed', + 'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json), + 'complete', 'complete', 'not_quarantined' + ) + """, + { + "id": uuid4(), + "project_id": project_id, + "source_registry_id": grb_id, + "source_snapshot_id": grb_snapshot_id, + "validation_report_json": _accepted_report(), + }, + label="passed dataset checksum required guard", + ) + _assert_constraint_rejection( + engine, + """ + INSERT INTO datasets ( + id, project_id, name, dataset_type, source, source_name, status, + source_registry_id, source_snapshot_id, validation_status, + data_contract_key, data_contract_version, validation_report_json, + provenance_status, lineage_status, quarantine_status, checksum_sha256 + ) VALUES ( + :id, :project_id, 'checksum-mismatch.geojson', 'vector', 'governed', 'grb', 'ready', + :source_registry_id, :source_snapshot_id, 'passed', + 'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json), + 'complete', 'complete', 'not_quarantined', :checksum_sha256 + ) + """, + { + "id": uuid4(), + "project_id": project_id, + "source_registry_id": grb_id, + "source_snapshot_id": grb_snapshot_id, + "validation_report_json": _accepted_report(), + "checksum_sha256": "f" * 64, + }, + label="passed dataset snapshot checksum binding guard", + ) + _assert_constraint_rejection( + engine, + """ + INSERT INTO dataset_versions ( + id, dataset_id, version, source_registry_id, source_snapshot_id, + data_contract_key, data_contract_version, validation_report_json, + validation_status, provenance_status, lineage_status + ) VALUES ( + :id, :dataset_id, 2, :source_registry_id, :source_snapshot_id, + 'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json), + 'passed', 'complete', 'complete' + ) + """, + { + "id": uuid4(), + "dataset_id": dataset_a, + "source_registry_id": grb_id, + "source_snapshot_id": grb_snapshot_id, + "validation_report_json": _accepted_report(), + }, + label="passed dataset version checksum required guard", + ) + _assert_constraint_rejection( + engine, + """ + INSERT INTO dataset_versions ( + id, dataset_id, version, source_registry_id, source_snapshot_id, + data_contract_key, data_contract_version, validation_report_json, + validation_status, provenance_status, lineage_status, checksum_sha256 + ) VALUES ( + :id, :dataset_id, 2, :source_registry_id, :source_snapshot_id, + 'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json), + 'passed', 'complete', 'complete', :checksum_sha256 + ) + """, + { + "id": uuid4(), + "dataset_id": dataset_a, + "source_registry_id": grb_id, + "source_snapshot_id": grb_snapshot_id, + "validation_report_json": _accepted_report(), + "checksum_sha256": "f" * 64, + }, + label="passed dataset version snapshot checksum binding guard", + ) + _assert_constraint_rejection( + engine, + "UPDATE datasets SET data_contract_version = '9.9.9' WHERE id = :dataset_id", + {"dataset_id": dataset_b}, + label="accepted contract evidence immutability guard", + ) + _assert_constraint_rejection( + engine, + "UPDATE dataset_versions SET data_contract_version = '9.9.9' WHERE id = :dataset_version_id", + {"dataset_version_id": version_id}, + label="accepted dataset-version contract evidence immutability guard", + ) + _assert_constraint_rejection( + engine, + "UPDATE datasets SET storage_path = '/tampered/asset.geojson' WHERE id = :dataset_id", + {"dataset_id": mutation_dataset}, + label="accepted dataset artifact immutability guard", + ) + _assert_constraint_rejection( + engine, + "UPDATE datasets SET observed_at = CURRENT_TIMESTAMP WHERE id = :dataset_id", + {"dataset_id": mutation_dataset}, + label="accepted dataset temporal evidence immutability guard", + ) + _assert_constraint_rejection( + engine, + "UPDATE dataset_versions SET storage_path = '/tampered/version.geojson' WHERE id = :dataset_version_id", + {"dataset_version_id": version_id}, + label="accepted dataset-version artifact immutability guard", + ) + _assert_constraint_rejection( + engine, + """ + UPDATE datasets + SET validation_status = 'failed', checksum_sha256 = 'f' || repeat('0', 63) + WHERE id = :dataset_id + """, + {"dataset_id": mutation_dataset}, + label="accepted dataset requires invalidation before artifact replacement", + ) + with _transaction(engine) as connection: + connection.execute( + text( + "UPDATE datasets SET validation_status = 'failed' WHERE id = :dataset_id" + ), + {"dataset_id": mutation_dataset}, + ) + connection.execute( + text( + "UPDATE datasets SET storage_path = '/replacement/asset.geojson' WHERE id = :dataset_id" + ), + {"dataset_id": mutation_dataset}, + ) + _assert_constraint_rejection( + engine, + "UPDATE source_snapshots SET source_registry_id = :registry_id WHERE id = :snapshot_id", + {"registry_id": osm_id, "snapshot_id": grb_snapshot_id}, + label="immutable snapshot registry guard", + ) + _assert_constraint_rejection( + engine, + "UPDATE source_registry SET display_name = 'tampered' WHERE id = :registry_id", + {"registry_id": grb_id}, + label="server-owned source registry guard", + ) + _assert_constraint_rejection( + engine, + "UPDATE source_snapshots SET checksum_sha256 = :checksum_sha256 WHERE id = :snapshot_id", + {"checksum_sha256": "c" * 64, "snapshot_id": grb_snapshot_id}, + label="immutable snapshot evidence guard", + ) + _assert_constraint_rejection( + engine, + """ + INSERT INTO dataset_lineage_edges ( + id, parent_dataset_id, child_dataset_id, relation_type, transformation_name + ) VALUES (:id, :parent_dataset_id, :child_dataset_id, 'derived_from', 'cycle') + """, + { + "id": uuid4(), + "parent_dataset_id": dataset_c, + "child_dataset_id": dataset_a, + }, + label="lineage cycle guard", + ) + _assert_constraint_rejection( + engine, + "UPDATE dataset_lineage_edges SET transformation_name = 'tampered' WHERE id = :id", + {"id": edge_a}, + label="lineage edge immutability update guard", + ) + _assert_constraint_rejection( + engine, + "DELETE FROM dataset_lineage_edges WHERE id = :id", + {"id": edge_b}, + label="lineage edge immutability delete guard", + ) + with _transaction(engine) as connection: + connection.execute( + text( + """ + INSERT INTO dataset_quarantines ( + id, dataset_version_id, stage, reason_code, status + ) VALUES (:id, :dataset_version_id, 'integration_test', 'CHECKSUM_MISMATCH', 'quarantined') + """ + ), + {"id": uuid4(), "dataset_version_id": version_id}, + ) + dataset_state = ( + connection.execute( + text( + """ + SELECT status, quarantine_status, validation_status, provenance_status, lineage_status + FROM datasets WHERE id = :dataset_id + """ + ), + {"dataset_id": dataset_a}, + ) + .mappings() + .one() + ) + version_state = ( + connection.execute( + text( + """ + SELECT validation_status, provenance_status, lineage_status + FROM dataset_versions WHERE id = :dataset_version_id + """ + ), + {"dataset_version_id": version_id}, + ) + .mappings() + .one() + ) + snapshot_state = connection.execute( + text( + "SELECT ingest_status FROM source_snapshots WHERE id = :snapshot_id" + ), + {"snapshot_id": grb_snapshot_id}, + ).scalar_one() + child_dataset_state = ( + connection.execute( + text( + """ + SELECT status, quarantine_status, validation_status, provenance_status, lineage_status + FROM datasets WHERE id = :dataset_id + """ + ), + {"dataset_id": dataset_b}, + ) + .mappings() + .one() + ) + grandchild_dataset_state = ( + connection.execute( + text( + """ + SELECT status, quarantine_status, validation_status, provenance_status, lineage_status + FROM datasets WHERE id = :dataset_id + """ + ), + {"dataset_id": dataset_c}, + ) + .mappings() + .one() + ) + shared_dataset_state = ( + connection.execute( + text( + """ + SELECT status, quarantine_status, validation_status, provenance_status, lineage_status + FROM datasets WHERE id = :dataset_id + """ + ), + {"dataset_id": shared_dataset}, + ) + .mappings() + .one() + ) + expected_dataset_state = { + "status": "quarantined", + "quarantine_status": "quarantined", + "validation_status": "failed", + "provenance_status": "incomplete", + "lineage_status": "incomplete", + } + if dict(dataset_state) != expected_dataset_state: + raise AssertionError( + f"quarantine parent propagation mismatch: {dict(dataset_state)}" + ) + expected_version_state = { + "validation_status": "failed", + "provenance_status": "incomplete", + "lineage_status": "incomplete", + } + if dict(version_state) != expected_version_state: + raise AssertionError( + f"quarantine version propagation mismatch: {dict(version_state)}" + ) + if snapshot_state != "quarantined": + raise AssertionError( + f"quarantine snapshot propagation mismatch: {snapshot_state}" + ) + if dict(child_dataset_state) != expected_dataset_state: + raise AssertionError( + "quarantine transitive-child propagation mismatch: " + f"{dict(child_dataset_state)}" + ) + if dict(grandchild_dataset_state) != expected_dataset_state: + raise AssertionError( + "quarantine transitive-grandchild propagation mismatch: " + f"{dict(grandchild_dataset_state)}" + ) + if dict(shared_dataset_state) != expected_dataset_state: + raise AssertionError( + "quarantine shared-snapshot propagation mismatch: " + f"{dict(shared_dataset_state)}" + ) + finally: + engine.dispose() + + downgrade_output = ( + _run_alembic(database_url, "downgrade", "base") + if verify_downgrade + else "not requested" + ) + return { + "schema_version": 1, + "phase": "P2", + "started_at": started_at, + "completed_at": datetime.now(UTC).isoformat(), + "migration_revision": "202608010001", + "database_name": str(make_url(database_url).database), + "result": "passed", + "guards": { + "snapshot_registry_pairing": "passed", + "snapshot_registry_immutable": "passed", + "source_registry_immutable": "passed", + "snapshot_evidence_immutable": "passed", + "lineage_cycle": "passed", + "lineage_edge_immutable": "passed", + "version_quarantine_propagation": "passed", + "snapshot_quarantine_fanout": "passed", + "transitive_lineage_quarantine": "passed", + "passed_contract_report_required": "passed", + "passed_dataset_checksum_required_and_snapshot_bound": "passed", + "passed_dataset_version_checksum_required_and_snapshot_bound": "passed", + "accepted_contract_evidence_immutable": "passed", + "accepted_dataset_version_contract_evidence_immutable": "passed", + "accepted_artifact_and_temporal_evidence_immutable": "passed", + }, + "upgrade_output_tail": upgrade_output.splitlines()[-8:], + "downgrade_output_tail": downgrade_output.splitlines()[-8:], + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--database-url", required=True) + parser.add_argument( + "--output", + type=Path, + default=ROOT + / "artifacts" + / "evidence" + / "accuracy" + / "P2" + / "postgres-migration-guards.json", + ) + parser.add_argument("--skip-downgrade", action="store_true") + args = parser.parse_args() + try: + result = verify(args.database_url, verify_downgrade=not args.skip_downgrade) + except Exception as exc: + result = { + "schema_version": 1, + "phase": "P2", + "completed_at": datetime.now(UTC).isoformat(), + "migration_revision": "202608010001", + "result": "failed", + "error_type": type(exc).__name__, + "error": str(exc), + } + _json_dump(args.output, result) + print(json.dumps(result, indent=2)) + return 1 + _json_dump(args.output, result) + print(json.dumps(result, indent=2)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/fixtures/data-contracts/labels-invalid.json b/tests/fixtures/data-contracts/labels-invalid.json new file mode 100644 index 00000000..fc213ab1 --- /dev/null +++ b/tests/fixtures/data-contracts/labels-invalid.json @@ -0,0 +1,9 @@ +[ + { + "class_id": 2, + "x_center": 0.95, + "y_center": 0.5, + "width": 0.2, + "height": -0.1 + } +] diff --git a/tests/fixtures/data-contracts/labels-valid.json b/tests/fixtures/data-contracts/labels-valid.json new file mode 100644 index 00000000..a64b0a60 --- /dev/null +++ b/tests/fixtures/data-contracts/labels-valid.json @@ -0,0 +1,9 @@ +[ + { + "class_id": 0, + "x_center": 0.5, + "y_center": 0.5, + "width": 0.2, + "height": 0.3 + } +] diff --git a/tests/fixtures/data-contracts/vector-building-valid.geojson b/tests/fixtures/data-contracts/vector-building-valid.geojson new file mode 100644 index 00000000..dd7228a5 --- /dev/null +++ b/tests/fixtures/data-contracts/vector-building-valid.geojson @@ -0,0 +1,24 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "native_id": "GRB-GBG-001", + "feature_status": "active" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [4.8470, 51.1550], + [4.8480, 51.1550], + [4.8480, 51.1560], + [4.8470, 51.1560], + [4.8470, 51.1550] + ] + ] + } + } + ] +} diff --git a/tests/fixtures/data-contracts/vector-lambert-mislabelled-as-4326.geojson b/tests/fixtures/data-contracts/vector-lambert-mislabelled-as-4326.geojson new file mode 100644 index 00000000..44f7fc8a --- /dev/null +++ b/tests/fixtures/data-contracts/vector-lambert-mislabelled-as-4326.geojson @@ -0,0 +1,24 @@ +{ + "type": "FeatureCollection", + "features": [ + { + "type": "Feature", + "properties": { + "native_id": "GRB-GBG-002", + "feature_status": "active" + }, + "geometry": { + "type": "Polygon", + "coordinates": [ + [ + [193277.5, 205708.3], + [193377.5, 205708.3], + [193377.5, 205808.3], + [193277.5, 205808.3], + [193277.5, 205708.3] + ] + ] + } + } + ] +} diff --git a/tests/test_building_proposal_classifier.py b/tests/test_building_proposal_classifier.py index 3eb9206b..d58c8987 100644 --- a/tests/test_building_proposal_classifier.py +++ b/tests/test_building_proposal_classifier.py @@ -1,9 +1,14 @@ from __future__ import annotations +import hashlib import importlib.util +import json +import sys +import types from pathlib import Path import pytest +from PIL import Image ROOT = Path(__file__).resolve().parents[1] @@ -22,6 +27,87 @@ miner = load("build_building_proposal_classifier_dataset") trainer = load("train_building_proposal_classifier") +def _sha256(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def _write_json(path: Path, payload: dict) -> None: + path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") + + +def _governed_proposal_crop_fixture(tmp_path: Path) -> tuple[Path, Path]: + """Create a complete tiny crop release without importing Torch or YOLO.""" + + corpus_manifest = tmp_path / "corpus-manifest.json" + _write_json(corpus_manifest, {"samples": []}) + corpus_sha256 = _sha256(corpus_manifest) + corpus_freeze = tmp_path / "corpus-freeze.json" + _write_json(corpus_freeze, {"immutable": True}) + summary = tmp_path / "source-summary.json" + _write_json(summary, {"source_manifest_sha256": corpus_sha256, "tiles": []}) + + dataset_dir = tmp_path / "proposal-crops" + entries: list[dict] = [] + for split in ("train", "val"): + for label, colour in (("negative", (0, 0, 0)), ("positive", (255, 255, 255))): + crop_path = dataset_dir / split / label / f"{split}-{label}.jpg" + crop_path.parent.mkdir(parents=True, exist_ok=True) + Image.new("RGB", (2, 2), colour).save(crop_path) + relative_path = crop_path.relative_to(dataset_dir).as_posix() + entries.append( + { + "relative_path": relative_path, + "sha256": _sha256(crop_path), + "size_bytes": crop_path.stat().st_size, + "split": split, + "label": label, + "sample_slug": f"{split}-{label}", + "proposal_index": 0, + "proposal_score": 0.8, + "source_box_xyxy": [0.0, 0.0, 1.0, 1.0], + "source_image_path": str(tmp_path / "source-image.tif"), + "source_image_sha256": "a" * 64, + "source_label_path": str(tmp_path / "source-label.txt"), + "source_label_sha256": "b" * 64, + } + ) + entries.sort(key=lambda item: item["relative_path"]) + counts = {f"{split}/{label}": 1 for split in ("train", "val") for label in ("negative", "positive")} + payload: dict = { + "schema_version": 1, + "status": "ok", + "immutable": True, + "dataset_kind": "building_proposal_classifier_crops", + "fixture_mode": False, + "governed_corpus_live_recheck": True, + "source": { + "corpus_manifest": {"path": str(corpus_manifest), "sha256": corpus_sha256}, + "corpus_freeze": {"path": str(corpus_freeze), "sha256": _sha256(corpus_freeze)}, + "summary": { + "path": str(summary), + "sha256": _sha256(summary), + "source_manifest_sha256": corpus_sha256, + }, + "training_release": { + "dataset_yaml_path": "fixture-dataset.yaml", + "dataset_yaml_sha256": "d" * 64, + "corpus_manifest_sha256": corpus_sha256, + }, + "proposal_model": {"path": str(tmp_path / "proposal.pt"), "sha256": "c" * 64}, + }, + "parameters": {"crop_scale": 1.4}, + "counts": counts, + "sample_counts": {item["sample_slug"]: 1 for item in entries}, + "tile_count": 2, + "crop_count": len(entries), + "crops_sha256": hashlib.sha256(trainer._canonical_json_bytes({"crops": entries})).hexdigest(), + "crops": entries, + } + payload["manifest_sha256"] = trainer._payload_sha256(payload) + _write_json(dataset_dir / trainer.PROPOSAL_DATASET_PROVENANCE_NAME, payload) + return dataset_dir, corpus_manifest + + def test_classify_proposals_consumes_reference_once() -> None: reference = [(0.0, 0.0, 10.0, 10.0)] proposals = [((0.0, 0.0, 10.0, 10.0), 0.9), ((0.0, 0.0, 10.0, 10.0), 0.8)] @@ -38,3 +124,166 @@ def test_eligible_tiles_rejects_protected_manifest_split() -> None: def test_binary_metrics() -> None: result = trainer.binary_metrics([0.9, 0.8, 0.2, 0.1], [1, 0, 1, 0]) assert result == {"tp": 1, "fp": 1, "fn": 1, "precision": 0.5, "recall": 0.5, "f1": 0.5} + + +def test_builder_rechecks_frozen_corpus_against_live_governed_state(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + observed: dict[str, object] = {} + + def fake_assert(path: Path, **kwargs: object) -> dict: + observed["path"] = path + observed.update(kwargs) + return {"samples": []} + + monkeypatch.setattr(miner, "assert_frozen_manifest_training_eligible", fake_assert) + manifest_path = tmp_path / "corpus.json" + manifest_path.write_text("{}", encoding="utf-8") + assert miner.load_governed_corpus_manifest(manifest_path, fixture_mode=False) == {"samples": []} + assert observed == {"path": manifest_path, "fixture_mode": False, "verify_live": True} + + +def test_builder_rejects_summary_not_bound_to_exact_corpus_manifest(tmp_path: Path) -> None: + manifest_path = tmp_path / "corpus.json" + manifest_path.write_text('{"samples": []}', encoding="utf-8") + with pytest.raises(ValueError, match="not bound"): + miner.assert_summary_source_manifest_binding({"source_manifest_sha256": "0" * 64}, manifest_path) + + +def test_builder_emits_immutable_checksum_bound_crop_provenance( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + class FakeTensor: + def __init__(self, values: list[list[float]] | list[float]) -> None: + self._values = values + + def cpu(self) -> "FakeTensor": + return self + + def tolist(self) -> list[list[float]] | list[float]: + return self._values + + class FakeBoxes: + xyxy = FakeTensor([[40.0, 40.0, 60.0, 60.0], [0.0, 0.0, 10.0, 10.0]]) + conf = FakeTensor([0.9, 0.8]) + + class FakeResult: + boxes = FakeBoxes() + + class FakeYOLO: + def __init__(self, model: str) -> None: + self.model = model + + def predict(self, sources: list[str], **_kwargs: object) -> list[FakeResult]: + return [FakeResult() for _source in sources] + + corpus_manifest = tmp_path / "corpus-manifest.json" + manifest = { + "samples": [ + {"sample_slug": "train-a", "split": "train", "region": "flanders"}, + {"sample_slug": "val-b", "split": "val", "region": "flanders"}, + ] + } + _write_json(corpus_manifest, manifest) + _write_json(tmp_path / "corpus-freeze.json", {"immutable": True}) + tiles: list[dict[str, object]] = [] + for sample_slug, split in (("train-a", "train"), ("val-b", "val")): + image_path = tmp_path / f"{sample_slug}.png" + Image.new("RGB", (100, 100), (50, 100, 150)).save(image_path) + label_path = tmp_path / f"{sample_slug}.txt" + label_path.write_text("0 0.5 0.5 0.2 0.2\n", encoding="utf-8") + tiles.append( + { + "sample_slug": sample_slug, + "split": split, + "kept": True, + "image_path": str(image_path), + "label_path": str(label_path), + } + ) + summary_path = tmp_path / "source-summary.json" + _write_json( + summary_path, + { + "source_manifest_sha256": _sha256(corpus_manifest), + "training_release_manifest": "fixture-training-release.json", + "training_release_manifest_sha256": "a" * 64, + "training_asset_manifest": "fixture-training-assets.json", + "tiles": tiles, + }, + ) + model_path = tmp_path / "proposal-model.pt" + model_path.write_bytes(b"proposal-model") + output_dir = tmp_path / "proposal-crops" + + monkeypatch.setattr(miner, "load_governed_corpus_manifest", lambda *_args, **_kwargs: manifest) + monkeypatch.setattr( + miner, + "assert_yolo_summary_bound_to_embedded_training_release", + lambda **_kwargs: { + "dataset_yaml": {"path": "fixture-dataset.yaml", "sha256": "d" * 64}, + "corpus": {"manifest_sha256": _sha256(corpus_manifest)}, + }, + ) + monkeypatch.setitem(sys.modules, "ultralytics", types.SimpleNamespace(YOLO=FakeYOLO)) + monkeypatch.setattr( + miner.sys, + "argv", + [ + "build_building_proposal_classifier_dataset.py", + "--model", + str(model_path), + "--summary", + str(summary_path), + "--corpus-manifest", + str(corpus_manifest), + "--output-dir", + str(output_dir), + ], + ) + + assert miner.main() == 0 + provenance_path = output_dir / miner.PROPOSAL_DATASET_PROVENANCE_NAME + provenance = json.loads(provenance_path.read_text(encoding="utf-8")) + assert provenance["immutable"] is True + assert provenance["crop_count"] == 4 + assert provenance["manifest_sha256"] == miner._immutable_payload_sha256(provenance) + assert all(_sha256(output_dir / item["relative_path"]) == item["sha256"] for item in provenance["crops"]) + with pytest.raises(RuntimeError, match="immutable provenance"): + miner._write_immutable_json(provenance_path, {"different": True}) + + +def test_trainer_validates_every_crop_and_source_binding_before_torch( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + dataset_dir, corpus_manifest = _governed_proposal_crop_fixture(tmp_path) + monkeypatch.setattr( + trainer, + "assert_yolo_summary_bound_to_embedded_training_release", + lambda **_kwargs: { + "dataset_yaml": {"path": "fixture-dataset.yaml", "sha256": "d" * 64}, + "corpus": {"manifest_sha256": _sha256(corpus_manifest)}, + }, + ) + payload = trainer.validate_proposal_dataset_provenance(dataset_dir, corpus_manifest, fixture_mode=False) + assert payload["crop_count"] == 4 + + crop = dataset_dir / "train" / "positive" / "train-positive.jpg" + crop.write_bytes(b"tampered") + with pytest.raises(trainer.ProposalDatasetProvenanceError, match="checksum"): + trainer.validate_proposal_dataset_provenance(dataset_dir, corpus_manifest, fixture_mode=False) + + +def test_trainer_rechecks_live_governed_corpus_before_pytorch(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + observed: dict[str, object] = {} + + def fake_assert(path: Path, **kwargs: object) -> dict: + observed["path"] = path + observed.update(kwargs) + return {"samples": []} + + monkeypatch.setattr(trainer, "assert_frozen_manifest_training_eligible", fake_assert) + manifest_path = tmp_path / "corpus.json" + manifest_path.write_text("{}", encoding="utf-8") + assert trainer.load_governed_corpus_manifest(manifest_path, fixture_mode=False) == {"samples": []} + assert observed == {"path": manifest_path, "fixture_mode": False, "verify_live": True}