Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(provenance): govern source snapshots and data inputs

Browse Source
This commit is contained in:
Jens
2026-08-01 23:46:17 +02:00
parent cebeb5f3b4
commit 5b3c17b494
96 changed files with 20156 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/models/__init__.py
+25 -1
View File
@@ -1,4 +1,24 @@
from .entities import AoiOperation, AoiOperationPartition, AnalysisRun, Area, Dataset, DatasetVersion, Detection, DetectionReview, Export, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature
from .entities import (
AoiOperation,
AoiOperationPartition,
AnalysisRun,
Area,
Dataset,
DatasetLineageEdge,
DatasetQuarantine,
DatasetVersion,
Detection,
DetectionReview,
Export,
Job,
Metric,
Project,
QualityCheck,
Segmentation,
SourceRegistry,
SourceSnapshot,
VectorFeature,
)
__all__ = [
"AnalysisRun",
@@ -6,6 +26,8 @@ __all__ = [
"AoiOperationPartition",
"Area",
"Dataset",
"DatasetLineageEdge",
"DatasetQuarantine",
"DatasetVersion",
"Detection",
"DetectionReview",
@@ -15,5 +37,7 @@ __all__ = [
"Project",
"QualityCheck",
"Segmentation",
"SourceRegistry",
"SourceSnapshot",
"VectorFeature",
]
backend/app/models/entities.py
+365
View File
@@ -12,6 +12,37 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.db.base import Base
SOURCE_CLASSIFICATIONS = (
"authoritative",
"corroborative",
"contextual",
"derived",
"experimental",
)
SOURCE_FRESHNESS_STATUSES = (
"unknown",
"current",
"due",
"stale",
"not_applicable",
"review_required",
)
SOURCE_INGEST_STATUSES = (
"registered",
"configured",
"not_configured",
"available",
"ingested",
"failed",
"quarantined",
"legacy_unverified",
)
PROVENANCE_STATUSES = ("complete", "incomplete", "not_applicable")
LINEAGE_STATUSES = ("complete", "incomplete", "not_applicable")
VALIDATION_STATUSES = ("not_validated", "passed", "failed")
QUARANTINE_STATUSES = ("not_quarantined", "quarantined")
class Project(Base):
__tablename__ = "projects"
@@ -42,6 +73,123 @@ class Area(Base):
project: Mapped[Project] = relationship("Project", back_populates="areas")
class SourceRegistry(Base):
"""Server-owned source identity and authority contract.
Dataset metadata remains descriptive until a governed importer binds a
dataset to both this registry entry and an immutable SourceSnapshot.
"""
__tablename__ = "source_registry"
__table_args__ = (
UniqueConstraint("source_key", name="uq_source_registry_source_key"),
CheckConstraint(
"classification IN ('authoritative', 'corroborative', 'contextual', 'derived', 'experimental')",
name="ck_source_registry_classification",
),
CheckConstraint(
"freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')",
name="ck_source_registry_freshness_status",
),
CheckConstraint(
"ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', "
"'failed', 'quarantined', 'legacy_unverified')",
name="ck_source_registry_ingest_status",
),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
source_key: Mapped[str] = mapped_column(String(120), nullable=False)
display_name: Mapped[str] = mapped_column(String(255), nullable=False)
classification: Mapped[str] = mapped_column(String(32), nullable=False)
authority_name: Mapped[str] = mapped_column(String(255), nullable=False, default="unknown", server_default="unknown")
authority_scope_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
provider_adapter_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
license_name: Mapped[str] = mapped_column(String(255), nullable=False, default="unknown", server_default="unknown")
license_url: Mapped[str | None] = mapped_column(Text, nullable=True)
usage_restrictions: Mapped[str] = mapped_column(Text, nullable=False, default="unknown", server_default="unknown")
default_crs: Mapped[str] = mapped_column(String(64), nullable=False, default="unknown", server_default="unknown")
default_units: Mapped[str] = mapped_column(String(120), nullable=False, default="unknown", server_default="unknown")
spatial_resolution_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
temporal_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
geographic_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
expected_geometry_types_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
expected_attributes_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
usage_policy_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
freshness_status: Mapped[str] = mapped_column(
String(32), nullable=False, default="unknown", server_default="unknown"
)
ingest_status: Mapped[str] = mapped_column(
String(32), nullable=False, default="registered", server_default="registered"
)
known_limitations_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
registry_metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
snapshots: Mapped[list["SourceSnapshot"]] = relationship(
"SourceSnapshot", back_populates="source_registry", cascade="all, delete-orphan"
)
datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="source_registry")
dataset_versions: Mapped[list["DatasetVersion"]] = relationship("DatasetVersion", back_populates="source_registry")
class SourceSnapshot(Base):
"""Immutable source-version evidence recorded by governed ingestion."""
__tablename__ = "source_snapshots"
__table_args__ = (
UniqueConstraint("source_registry_id", "snapshot_key", name="uq_source_snapshots_registry_key"),
CheckConstraint(
"freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')",
name="ck_source_snapshots_freshness_status",
),
CheckConstraint(
"ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', "
"'failed', 'quarantined', 'legacy_unverified')",
name="ck_source_snapshots_ingest_status",
),
CheckConstraint(
"checksum_sha256 = lower(checksum_sha256) AND checksum_sha256 ~ '^[0-9a-f]{64}$'",
name="ck_source_snapshots_checksum_sha256",
),
Index("ix_source_snapshots_registry_fetched", "source_registry_id", "fetched_at"),
Index("ix_source_snapshots_checksum", "checksum_sha256"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
source_registry_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="CASCADE"), nullable=False
)
snapshot_key: Mapped[str] = mapped_column(String(255), nullable=False)
source_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
snapshot_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
crs: Mapped[str | None] = mapped_column(String(64), nullable=True)
units: Mapped[str | None] = mapped_column(String(120), nullable=True)
spatial_resolution_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
temporal_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
geographic_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
observed_schema_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
freshness_status: Mapped[str] = mapped_column(
String(32), nullable=False, default="unknown", server_default="unknown"
)
ingest_status: Mapped[str] = mapped_column(
String(32), nullable=False, default="registered", server_default="registered"
)
known_limitations_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
snapshot_metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
source_registry: Mapped[SourceRegistry] = relationship("SourceRegistry", back_populates="snapshots")
datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="source_snapshot")
dataset_versions: Mapped[list["DatasetVersion"]] = relationship("DatasetVersion", back_populates="source_snapshot")
quarantines: Mapped[list["DatasetQuarantine"]] = relationship("DatasetQuarantine", back_populates="source_snapshot")
class Dataset(Base):
__tablename__ = "datasets"
__table_args__ = (
@@ -49,6 +197,27 @@ class Dataset(Base):
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
name="ck_datasets_temporal_valid_range",
),
CheckConstraint(
"validation_status IN ('not_validated', 'passed', 'failed')",
name="ck_datasets_validation_status",
),
CheckConstraint(
"provenance_status IN ('complete', 'incomplete', 'not_applicable')",
name="ck_datasets_provenance_status",
),
CheckConstraint(
"lineage_status IN ('complete', 'incomplete', 'not_applicable')",
name="ck_datasets_lineage_status",
),
CheckConstraint(
"quarantine_status IN ('not_quarantined', 'quarantined')",
name="ck_datasets_quarantine_status",
),
CheckConstraint(
"ingest_key IS NULL OR btrim(ingest_key) <> ''",
name="ck_datasets_ingest_key_not_blank",
),
UniqueConstraint("project_id", "ingest_key", name="uq_datasets_project_ingest_key"),
Index(
"ix_datasets_project_temporal_series_observed",
"project_id",
@@ -69,6 +238,7 @@ class Dataset(Base):
content_type: Mapped[str | None] = mapped_column(String(120), nullable=True)
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
ingest_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
derived_from_dataset_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("datasets.id", ondelete="SET NULL"),
@@ -84,6 +254,43 @@ class Dataset(Base):
reference_layer_name: Mapped[str | None] = mapped_column(String(120), nullable=True)
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
source_registry_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="SET NULL"), nullable=True
)
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
)
data_contract_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
data_contract_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
validation_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="not_validated",
server_default="not_validated",
comment="not_validated | passed | failed",
)
validation_report_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="incomplete",
server_default="incomplete",
comment="complete | incomplete | not_applicable",
)
lineage_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="incomplete",
server_default="incomplete",
comment="complete | incomplete | not_applicable",
)
quarantine_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="not_quarantined",
server_default="not_quarantined",
comment="not_quarantined | quarantined",
)
imported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
temporal_series_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
@@ -106,6 +313,23 @@ class Dataset(Base):
back_populates="dataset",
cascade="all, delete-orphan",
)
source_registry: Mapped[SourceRegistry | None] = relationship("SourceRegistry", back_populates="datasets")
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="datasets")
parent_lineage_edges: Mapped[list["DatasetLineageEdge"]] = relationship(
"DatasetLineageEdge",
foreign_keys="DatasetLineageEdge.parent_dataset_id",
back_populates="parent_dataset",
cascade="all, delete-orphan",
)
child_lineage_edges: Mapped[list["DatasetLineageEdge"]] = relationship(
"DatasetLineageEdge",
foreign_keys="DatasetLineageEdge.child_dataset_id",
back_populates="child_dataset",
cascade="all, delete-orphan",
)
quarantines: Mapped[list["DatasetQuarantine"]] = relationship(
"DatasetQuarantine", back_populates="dataset", cascade="all, delete-orphan"
)
class DatasetVersion(Base):
@@ -115,7 +339,24 @@ class DatasetVersion(Base):
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
name="ck_dataset_versions_temporal_valid_range",
),
CheckConstraint(
"validation_status IN ('not_validated', 'passed', 'failed')",
name="ck_dataset_versions_validation_status",
),
CheckConstraint(
"provenance_status IN ('complete', 'incomplete', 'not_applicable')",
name="ck_dataset_versions_provenance_status",
),
CheckConstraint(
"lineage_status IN ('complete', 'incomplete', 'not_applicable')",
name="ck_dataset_versions_lineage_status",
),
CheckConstraint(
"ingest_key IS NULL OR btrim(ingest_key) <> ''",
name="ck_dataset_versions_ingest_key_not_blank",
),
Index("ix_dataset_versions_dataset_version", "dataset_id", "version", unique=True),
UniqueConstraint("dataset_id", "ingest_key", name="uq_dataset_versions_dataset_ingest_key"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
@@ -127,11 +368,135 @@ class DatasetVersion(Base):
valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
ingest_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
source_registry_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="SET NULL"), nullable=True
)
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
)
data_contract_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
data_contract_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
validation_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="not_validated",
server_default="not_validated",
comment="not_validated | passed | failed",
)
validation_report_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="incomplete",
server_default="incomplete",
comment="complete | incomplete | not_applicable",
)
lineage_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="incomplete",
server_default="incomplete",
comment="complete | incomplete | not_applicable",
)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
dataset: Mapped[Dataset] = relationship("Dataset", back_populates="versions")
source_registry: Mapped[SourceRegistry | None] = relationship("SourceRegistry", back_populates="dataset_versions")
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="dataset_versions")
quarantines: Mapped[list["DatasetQuarantine"]] = relationship("DatasetQuarantine", back_populates="dataset_version")
class DatasetLineageEdge(Base):
"""Immutable relationship between input/output datasets and transforms."""
__tablename__ = "dataset_lineage_edges"
__table_args__ = (
CheckConstraint("parent_dataset_id <> child_dataset_id", name="ck_dataset_lineage_edges_distinct_datasets"),
UniqueConstraint(
"parent_dataset_id",
"child_dataset_id",
"relation_type",
"transformation_name",
name="uq_dataset_lineage_edges_relation",
),
Index("ix_dataset_lineage_edges_parent", "parent_dataset_id"),
Index("ix_dataset_lineage_edges_child", "child_dataset_id"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
parent_dataset_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False
)
child_dataset_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False
)
parent_dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
)
child_dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
)
relation_type: Mapped[str] = mapped_column(String(64), nullable=False)
transformation_name: Mapped[str] = mapped_column(String(255), nullable=False)
transformation_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
parameters_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
input_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
output_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
parent_dataset: Mapped[Dataset] = relationship(
"Dataset", foreign_keys=[parent_dataset_id], back_populates="parent_lineage_edges"
)
child_dataset: Mapped[Dataset] = relationship(
"Dataset", foreign_keys=[child_dataset_id], back_populates="child_lineage_edges"
)
class DatasetQuarantine(Base):
"""Durable fail-closed record for rejected or doubtful source artifacts."""
__tablename__ = "dataset_quarantines"
__table_args__ = (
CheckConstraint(
"dataset_id IS NOT NULL OR dataset_version_id IS NOT NULL OR source_snapshot_id IS NOT NULL",
name="ck_dataset_quarantines_target_present",
),
CheckConstraint(
"status IN ('quarantined', 'released', 'rejected')",
name="ck_dataset_quarantines_status",
),
Index("ix_dataset_quarantines_dataset_status", "dataset_id", "status"),
Index("ix_dataset_quarantines_snapshot_status", "source_snapshot_id", "status"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
dataset_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True
)
dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
)
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
)
stage: Mapped[str] = mapped_column(String(64), nullable=False)
reason_code: Mapped[str] = mapped_column(String(120), nullable=False)
details_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
artifact_path: Mapped[str | None] = mapped_column(Text, nullable=True)
artifact_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
status: Mapped[str] = mapped_column(
String(32), nullable=False, default="quarantined", server_default="quarantined"
)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
resolved_by: Mapped[str | None] = mapped_column(String(120), nullable=True)
dataset: Mapped[Dataset | None] = relationship("Dataset", back_populates="quarantines")
dataset_version: Mapped[DatasetVersion | None] = relationship("DatasetVersion", back_populates="quarantines")
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="quarantines")
class VectorFeature(Base):