Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(provenance): govern source snapshots and data inputs

Browse Source
This commit is contained in:
Jens
2026-08-01 23:46:17 +02:00
parent cebeb5f3b4
commit 5b3c17b494
96 changed files with 20156 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app/api/routes/__init__.py
+15 -1
View File
@@ -1 +1,15 @@
__all__ = ["analysis", "areas", "assistant", "auth", "datasets", "health", "projects", "exports", "jobs", "external", "qa", "temporal"]
__all__ = [
"analysis",
"areas",
"assistant",
"auth",
"datasets",
"exports",
"external",
"health",
"jobs",
"projects",
"qa",
"source_registry",
"temporal",
]
backend/app/api/routes/source_registry.py
+119
View File
@@ -0,0 +1,119 @@
from __future__ import annotations
from uuid import UUID
from fastapi import APIRouter, Depends
from sqlalchemy import func, or_
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.db.session import get_db
from app.models import Dataset, DatasetLineageEdge, DatasetQuarantine, Project, SourceRegistry, SourceSnapshot
from app.schemas import (
DatasetLineageEdgeRead,
DatasetProvenanceRead,
DatasetQuarantineRead,
Envelope,
ItemList,
SourceRegistryDetailRead,
SourceRegistryRead,
SourceSnapshotRead,
)
from app.utils.response import envelope
router = APIRouter(tags=["source-registry"])
def _source_read(source: SourceRegistry, *, snapshot_count: int = 0) -> SourceRegistryRead:
return SourceRegistryRead.model_validate(source).model_copy(update={"snapshot_count": int(snapshot_count)})
@router.get("/source-registry", response_model=Envelope[ItemList[SourceRegistryRead]])
def list_source_registry(
classification: str | None = None,
db: Session = Depends(get_db),
) -> dict:
query = (
db.query(SourceRegistry, func.count(SourceSnapshot.id).label("snapshot_count"))
.outerjoin(SourceSnapshot, SourceSnapshot.source_registry_id == SourceRegistry.id)
)
if classification:
query = query.filter(SourceRegistry.classification == classification.strip().lower())
rows = (
query.group_by(SourceRegistry.id)
.order_by(SourceRegistry.classification.asc(), SourceRegistry.display_name.asc())
.all()
)
items = [_source_read(source, snapshot_count=count) for source, count in rows]
return envelope({"items": items, "total": len(items)})
@router.get("/source-registry/{source_key}", response_model=Envelope[SourceRegistryDetailRead])
def get_source_registry_entry(source_key: str, db: Session = Depends(get_db)) -> dict:
normalized_key = source_key.strip().lower()
source = db.query(SourceRegistry).filter(SourceRegistry.source_key == normalized_key).one_or_none()
if source is None:
raise AppError(code="SOURCE_REGISTRY_ENTRY_NOT_FOUND", message="Source registry entry was not found", status_code=404)
snapshots = (
db.query(SourceSnapshot)
.filter(SourceSnapshot.source_registry_id == source.id)
.order_by(SourceSnapshot.fetched_at.desc(), SourceSnapshot.created_at.desc())
.all()
)
detail = SourceRegistryDetailRead(
source=_source_read(source, snapshot_count=len(snapshots)),
snapshots=[SourceSnapshotRead.model_validate(snapshot) for snapshot in snapshots],
)
return envelope(detail)
@router.get(
"/projects/{project_id}/datasets/{dataset_id}/provenance",
response_model=Envelope[DatasetProvenanceRead],
)
def get_dataset_provenance(
project_id: UUID,
dataset_id: UUID,
db: Session = Depends(get_db),
) -> dict:
if db.get(Project, project_id) is None:
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
dataset = db.get(Dataset, dataset_id)
if dataset is None or dataset.project_id != project_id:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
source = db.get(SourceRegistry, dataset.source_registry_id) if dataset.source_registry_id else None
snapshot = db.get(SourceSnapshot, dataset.source_snapshot_id) if dataset.source_snapshot_id else None
lineage = (
db.query(DatasetLineageEdge)
.filter(
or_(
DatasetLineageEdge.parent_dataset_id == dataset.id,
DatasetLineageEdge.child_dataset_id == dataset.id,
)
)
.order_by(DatasetLineageEdge.created_at.asc(), DatasetLineageEdge.id.asc())
.all()
)
quarantines = (
db.query(DatasetQuarantine)
.filter(DatasetQuarantine.dataset_id == dataset.id)
.order_by(DatasetQuarantine.created_at.desc(), DatasetQuarantine.id.desc())
.all()
)
result = DatasetProvenanceRead(
dataset_id=dataset.id,
source=_source_read(source) if source else None,
snapshot=SourceSnapshotRead.model_validate(snapshot) if snapshot else None,
data_contract_key=dataset.data_contract_key,
data_contract_version=dataset.data_contract_version,
validation_status=dataset.validation_status,
validation_report_json=dataset.validation_report_json,
provenance_status=dataset.provenance_status,
lineage_status=dataset.lineage_status,
quarantine_status=dataset.quarantine_status,
lineage=[DatasetLineageEdgeRead.model_validate(item) for item in lineage],
quarantines=[DatasetQuarantineRead.model_validate(item) for item in quarantines],
)
return envelope(result)
backend/app/main.py
+2 -1
View File
@@ -12,7 +12,7 @@ from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from app.api.routes import analysis, aoi_operations, areas, assistant, auth, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation, selection_partitions, temporal
from app.api.routes import analysis, aoi_operations, areas, assistant, auth, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation, selection_partitions, source_registry, temporal
from app.core.config import get_settings
from app.core.errors import AppError
from app.core.logging import configure_logging
@@ -103,6 +103,7 @@ def create_app() -> FastAPI:
app.include_router(quality_checks.router, prefix=settings.api_prefix)
app.include_router(exports.router, prefix=settings.api_prefix)
app.include_router(external.router, prefix=settings.api_prefix)
app.include_router(source_registry.router, prefix=settings.api_prefix)
app.include_router(demo.router, prefix=settings.api_prefix)
app.include_router(qa.router, prefix=settings.api_prefix)
app.include_router(detection.router, prefix=settings.api_prefix)
backend/app/models/__init__.py
+25 -1
View File
@@ -1,4 +1,24 @@
from .entities import AoiOperation, AoiOperationPartition, AnalysisRun, Area, Dataset, DatasetVersion, Detection, DetectionReview, Export, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature
from .entities import (
AoiOperation,
AoiOperationPartition,
AnalysisRun,
Area,
Dataset,
DatasetLineageEdge,
DatasetQuarantine,
DatasetVersion,
Detection,
DetectionReview,
Export,
Job,
Metric,
Project,
QualityCheck,
Segmentation,
SourceRegistry,
SourceSnapshot,
VectorFeature,
)
__all__ = [
"AnalysisRun",
@@ -6,6 +26,8 @@ __all__ = [
"AoiOperationPartition",
"Area",
"Dataset",
"DatasetLineageEdge",
"DatasetQuarantine",
"DatasetVersion",
"Detection",
"DetectionReview",
@@ -15,5 +37,7 @@ __all__ = [
"Project",
"QualityCheck",
"Segmentation",
"SourceRegistry",
"SourceSnapshot",
"VectorFeature",
]
backend/app/models/entities.py
+365
View File
@@ -12,6 +12,37 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.db.base import Base
SOURCE_CLASSIFICATIONS = (
"authoritative",
"corroborative",
"contextual",
"derived",
"experimental",
)
SOURCE_FRESHNESS_STATUSES = (
"unknown",
"current",
"due",
"stale",
"not_applicable",
"review_required",
)
SOURCE_INGEST_STATUSES = (
"registered",
"configured",
"not_configured",
"available",
"ingested",
"failed",
"quarantined",
"legacy_unverified",
)
PROVENANCE_STATUSES = ("complete", "incomplete", "not_applicable")
LINEAGE_STATUSES = ("complete", "incomplete", "not_applicable")
VALIDATION_STATUSES = ("not_validated", "passed", "failed")
QUARANTINE_STATUSES = ("not_quarantined", "quarantined")
class Project(Base):
__tablename__ = "projects"
@@ -42,6 +73,123 @@ class Area(Base):
project: Mapped[Project] = relationship("Project", back_populates="areas")
class SourceRegistry(Base):
"""Server-owned source identity and authority contract.
Dataset metadata remains descriptive until a governed importer binds a
dataset to both this registry entry and an immutable SourceSnapshot.
"""
__tablename__ = "source_registry"
__table_args__ = (
UniqueConstraint("source_key", name="uq_source_registry_source_key"),
CheckConstraint(
"classification IN ('authoritative', 'corroborative', 'contextual', 'derived', 'experimental')",
name="ck_source_registry_classification",
),
CheckConstraint(
"freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')",
name="ck_source_registry_freshness_status",
),
CheckConstraint(
"ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', "
"'failed', 'quarantined', 'legacy_unverified')",
name="ck_source_registry_ingest_status",
),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
source_key: Mapped[str] = mapped_column(String(120), nullable=False)
display_name: Mapped[str] = mapped_column(String(255), nullable=False)
classification: Mapped[str] = mapped_column(String(32), nullable=False)
authority_name: Mapped[str] = mapped_column(String(255), nullable=False, default="unknown", server_default="unknown")
authority_scope_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
provider_adapter_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
license_name: Mapped[str] = mapped_column(String(255), nullable=False, default="unknown", server_default="unknown")
license_url: Mapped[str | None] = mapped_column(Text, nullable=True)
usage_restrictions: Mapped[str] = mapped_column(Text, nullable=False, default="unknown", server_default="unknown")
default_crs: Mapped[str] = mapped_column(String(64), nullable=False, default="unknown", server_default="unknown")
default_units: Mapped[str] = mapped_column(String(120), nullable=False, default="unknown", server_default="unknown")
spatial_resolution_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
temporal_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
geographic_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
expected_geometry_types_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
expected_attributes_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
usage_policy_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
freshness_status: Mapped[str] = mapped_column(
String(32), nullable=False, default="unknown", server_default="unknown"
)
ingest_status: Mapped[str] = mapped_column(
String(32), nullable=False, default="registered", server_default="registered"
)
known_limitations_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
registry_metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
snapshots: Mapped[list["SourceSnapshot"]] = relationship(
"SourceSnapshot", back_populates="source_registry", cascade="all, delete-orphan"
)
datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="source_registry")
dataset_versions: Mapped[list["DatasetVersion"]] = relationship("DatasetVersion", back_populates="source_registry")
class SourceSnapshot(Base):
"""Immutable source-version evidence recorded by governed ingestion."""
__tablename__ = "source_snapshots"
__table_args__ = (
UniqueConstraint("source_registry_id", "snapshot_key", name="uq_source_snapshots_registry_key"),
CheckConstraint(
"freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')",
name="ck_source_snapshots_freshness_status",
),
CheckConstraint(
"ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', "
"'failed', 'quarantined', 'legacy_unverified')",
name="ck_source_snapshots_ingest_status",
),
CheckConstraint(
"checksum_sha256 = lower(checksum_sha256) AND checksum_sha256 ~ '^[0-9a-f]{64}$'",
name="ck_source_snapshots_checksum_sha256",
),
Index("ix_source_snapshots_registry_fetched", "source_registry_id", "fetched_at"),
Index("ix_source_snapshots_checksum", "checksum_sha256"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
source_registry_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="CASCADE"), nullable=False
)
snapshot_key: Mapped[str] = mapped_column(String(255), nullable=False)
source_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
snapshot_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
crs: Mapped[str | None] = mapped_column(String(64), nullable=True)
units: Mapped[str | None] = mapped_column(String(120), nullable=True)
spatial_resolution_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
temporal_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
geographic_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
observed_schema_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
freshness_status: Mapped[str] = mapped_column(
String(32), nullable=False, default="unknown", server_default="unknown"
)
ingest_status: Mapped[str] = mapped_column(
String(32), nullable=False, default="registered", server_default="registered"
)
known_limitations_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
snapshot_metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
source_registry: Mapped[SourceRegistry] = relationship("SourceRegistry", back_populates="snapshots")
datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="source_snapshot")
dataset_versions: Mapped[list["DatasetVersion"]] = relationship("DatasetVersion", back_populates="source_snapshot")
quarantines: Mapped[list["DatasetQuarantine"]] = relationship("DatasetQuarantine", back_populates="source_snapshot")
class Dataset(Base):
__tablename__ = "datasets"
__table_args__ = (
@@ -49,6 +197,27 @@ class Dataset(Base):
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
name="ck_datasets_temporal_valid_range",
),
CheckConstraint(
"validation_status IN ('not_validated', 'passed', 'failed')",
name="ck_datasets_validation_status",
),
CheckConstraint(
"provenance_status IN ('complete', 'incomplete', 'not_applicable')",
name="ck_datasets_provenance_status",
),
CheckConstraint(
"lineage_status IN ('complete', 'incomplete', 'not_applicable')",
name="ck_datasets_lineage_status",
),
CheckConstraint(
"quarantine_status IN ('not_quarantined', 'quarantined')",
name="ck_datasets_quarantine_status",
),
CheckConstraint(
"ingest_key IS NULL OR btrim(ingest_key) <> ''",
name="ck_datasets_ingest_key_not_blank",
),
UniqueConstraint("project_id", "ingest_key", name="uq_datasets_project_ingest_key"),
Index(
"ix_datasets_project_temporal_series_observed",
"project_id",
@@ -69,6 +238,7 @@ class Dataset(Base):
content_type: Mapped[str | None] = mapped_column(String(120), nullable=True)
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
ingest_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
derived_from_dataset_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("datasets.id", ondelete="SET NULL"),
@@ -84,6 +254,43 @@ class Dataset(Base):
reference_layer_name: Mapped[str | None] = mapped_column(String(120), nullable=True)
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
source_registry_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="SET NULL"), nullable=True
)
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
)
data_contract_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
data_contract_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
validation_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="not_validated",
server_default="not_validated",
comment="not_validated | passed | failed",
)
validation_report_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="incomplete",
server_default="incomplete",
comment="complete | incomplete | not_applicable",
)
lineage_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="incomplete",
server_default="incomplete",
comment="complete | incomplete | not_applicable",
)
quarantine_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="not_quarantined",
server_default="not_quarantined",
comment="not_quarantined | quarantined",
)
imported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
temporal_series_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
@@ -106,6 +313,23 @@ class Dataset(Base):
back_populates="dataset",
cascade="all, delete-orphan",
)
source_registry: Mapped[SourceRegistry | None] = relationship("SourceRegistry", back_populates="datasets")
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="datasets")
parent_lineage_edges: Mapped[list["DatasetLineageEdge"]] = relationship(
"DatasetLineageEdge",
foreign_keys="DatasetLineageEdge.parent_dataset_id",
back_populates="parent_dataset",
cascade="all, delete-orphan",
)
child_lineage_edges: Mapped[list["DatasetLineageEdge"]] = relationship(
"DatasetLineageEdge",
foreign_keys="DatasetLineageEdge.child_dataset_id",
back_populates="child_dataset",
cascade="all, delete-orphan",
)
quarantines: Mapped[list["DatasetQuarantine"]] = relationship(
"DatasetQuarantine", back_populates="dataset", cascade="all, delete-orphan"
)
class DatasetVersion(Base):
@@ -115,7 +339,24 @@ class DatasetVersion(Base):
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
name="ck_dataset_versions_temporal_valid_range",
),
CheckConstraint(
"validation_status IN ('not_validated', 'passed', 'failed')",
name="ck_dataset_versions_validation_status",
),
CheckConstraint(
"provenance_status IN ('complete', 'incomplete', 'not_applicable')",
name="ck_dataset_versions_provenance_status",
),
CheckConstraint(
"lineage_status IN ('complete', 'incomplete', 'not_applicable')",
name="ck_dataset_versions_lineage_status",
),
CheckConstraint(
"ingest_key IS NULL OR btrim(ingest_key) <> ''",
name="ck_dataset_versions_ingest_key_not_blank",
),
Index("ix_dataset_versions_dataset_version", "dataset_id", "version", unique=True),
UniqueConstraint("dataset_id", "ingest_key", name="uq_dataset_versions_dataset_ingest_key"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
@@ -127,11 +368,135 @@ class DatasetVersion(Base):
valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
ingest_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
source_registry_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="SET NULL"), nullable=True
)
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
)
data_contract_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
data_contract_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
validation_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="not_validated",
server_default="not_validated",
comment="not_validated | passed | failed",
)
validation_report_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="incomplete",
server_default="incomplete",
comment="complete | incomplete | not_applicable",
)
lineage_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="incomplete",
server_default="incomplete",
comment="complete | incomplete | not_applicable",
)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
dataset: Mapped[Dataset] = relationship("Dataset", back_populates="versions")
source_registry: Mapped[SourceRegistry | None] = relationship("SourceRegistry", back_populates="dataset_versions")
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="dataset_versions")
quarantines: Mapped[list["DatasetQuarantine"]] = relationship("DatasetQuarantine", back_populates="dataset_version")
class DatasetLineageEdge(Base):
"""Immutable relationship between input/output datasets and transforms."""
__tablename__ = "dataset_lineage_edges"
__table_args__ = (
CheckConstraint("parent_dataset_id <> child_dataset_id", name="ck_dataset_lineage_edges_distinct_datasets"),
UniqueConstraint(
"parent_dataset_id",
"child_dataset_id",
"relation_type",
"transformation_name",
name="uq_dataset_lineage_edges_relation",
),
Index("ix_dataset_lineage_edges_parent", "parent_dataset_id"),
Index("ix_dataset_lineage_edges_child", "child_dataset_id"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
parent_dataset_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False
)
child_dataset_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False
)
parent_dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
)
child_dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
)
relation_type: Mapped[str] = mapped_column(String(64), nullable=False)
transformation_name: Mapped[str] = mapped_column(String(255), nullable=False)
transformation_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
parameters_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
input_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
output_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
parent_dataset: Mapped[Dataset] = relationship(
"Dataset", foreign_keys=[parent_dataset_id], back_populates="parent_lineage_edges"
)
child_dataset: Mapped[Dataset] = relationship(
"Dataset", foreign_keys=[child_dataset_id], back_populates="child_lineage_edges"
)
class DatasetQuarantine(Base):
"""Durable fail-closed record for rejected or doubtful source artifacts."""
__tablename__ = "dataset_quarantines"
__table_args__ = (
CheckConstraint(
"dataset_id IS NOT NULL OR dataset_version_id IS NOT NULL OR source_snapshot_id IS NOT NULL",
name="ck_dataset_quarantines_target_present",
),
CheckConstraint(
"status IN ('quarantined', 'released', 'rejected')",
name="ck_dataset_quarantines_status",
),
Index("ix_dataset_quarantines_dataset_status", "dataset_id", "status"),
Index("ix_dataset_quarantines_snapshot_status", "source_snapshot_id", "status"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
dataset_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True
)
dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
)
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
)
stage: Mapped[str] = mapped_column(String(64), nullable=False)
reason_code: Mapped[str] = mapped_column(String(120), nullable=False)
details_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
artifact_path: Mapped[str | None] = mapped_column(Text, nullable=True)
artifact_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
status: Mapped[str] = mapped_column(
String(32), nullable=False, default="quarantined", server_default="quarantined"
)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
resolved_by: Mapped[str | None] = mapped_column(String(120), nullable=True)
dataset: Mapped[Dataset | None] = relationship("Dataset", back_populates="quarantines")
dataset_version: Mapped[DatasetVersion | None] = relationship("DatasetVersion", back_populates="quarantines")
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="quarantines")
class VectorFeature(Base):
backend/app/schemas/__init__.py
+18
View File
@@ -32,6 +32,14 @@ from .source_catalog import (
SourceCatalogProbeReport,
SourceCatalogProbeSummary,
)
from .source_registry import (
DatasetProvenanceRead,
DatasetLineageEdgeRead,
DatasetQuarantineRead,
SourceRegistryDetailRead,
SourceRegistryRead,
SourceSnapshotRead,
)
from .grb_refresh import GrbRefreshLayerPlan, GrbRefreshPlan, GrbRefreshPlanSummary
from .grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead
from .official_vector import (
@@ -207,6 +215,12 @@ __all__ = [
"SourceCatalogProbeItem",
"SourceCatalogProbeReport",
"SourceCatalogProbeSummary",
"SourceRegistryRead",
"SourceRegistryDetailRead",
"SourceSnapshotRead",
"DatasetLineageEdgeRead",
"DatasetQuarantineRead",
"DatasetProvenanceRead",
"GrbRefreshLayerPlan",
"GrbRefreshPlan",
"GrbRefreshPlanSummary",
@@ -271,6 +285,10 @@ __all__ = [
"FloodHazardSelectionSummary",
"BathymetryProfileAcquireRequest",
"BathymetryProfileAcquisitionResult",
"BathymetryRasterMetric",
"BathymetryRasterSelectionRequest",
"BathymetryRasterSelectionResponse",
"BathymetryRasterSelectionSummary",
"BathymetryPartitionFinalizeRequest",
"BathymetryPartitionFinalizationResult",
"BathymetrySourceProbeRead",
backend/app/schemas/dataset.py
+19
View File
@@ -35,6 +35,16 @@ class DatasetCreateResponse(BaseModel):
reference_layer_name: str | None = None
source_metadata: dict | None = None
provenance_metadata: dict | None = None
ingest_key: str | None = None
source_registry_id: UUID | None = None
source_snapshot_id: UUID | None = None
data_contract_key: str | None = None
data_contract_version: str | None = None
validation_status: str | None = None
validation_report_json: dict | None = None
provenance_status: str | None = None
lineage_status: str | None = None
quarantine_status: str | None = None
imported_at: datetime | None = None
temporal_series_key: str | None = None
observed_at: datetime | None = None
@@ -97,6 +107,15 @@ class DatasetVersionRead(BaseModel):
checksum_sha256: str | None = None
source_metadata: dict | None = None
provenance_metadata: dict | None = None
ingest_key: str | None = None
source_registry_id: UUID | None = None
source_snapshot_id: UUID | None = None
data_contract_key: str | None = None
data_contract_version: str | None = None
validation_status: str | None = None
validation_report_json: dict | None = None
provenance_status: str | None = None
lineage_status: str | None = None
created_at: datetime | None = None
model_config = {"from_attributes": True}
backend/app/schemas/detection.py
+2
View File
@@ -144,6 +144,8 @@ class YoloPreflightChecks(BaseModel):
accelerator_ready: bool | None = None
model_path_set: bool | None = None
model_file_exists: bool | None = None
model_provenance_manifest_path: str | None = None
model_provenance_valid: bool | None = None
model_load_requested: bool
model_load_ok: bool | None = None
manifest_path_set: bool | None = None
backend/app/schemas/source_registry.py
+120
View File
@@ -0,0 +1,120 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from pydantic import BaseModel, Field
class SourceRegistryRead(BaseModel):
"""Read-only, server-owned source-authority definition."""
id: UUID
source_key: str
display_name: str
classification: str
authority_name: str
authority_scope_json: dict
provider_adapter_key: str | None = None
source_url: str | None = None
license_name: str
license_url: str | None = None
usage_restrictions: str
default_crs: str
default_units: str
spatial_resolution_json: dict
temporal_coverage_json: dict
geographic_coverage_json: dict
expected_geometry_types_json: list
expected_attributes_json: dict
usage_policy_json: dict
freshness_status: str
ingest_status: str
known_limitations_json: list
registry_metadata_json: dict
created_at: datetime | None = None
updated_at: datetime | None = None
snapshot_count: int = 0
model_config = {"from_attributes": True}
class SourceSnapshotRead(BaseModel):
"""Immutable version/snapshot evidence attached to an imported dataset."""
id: UUID
source_registry_id: UUID
snapshot_key: str
source_version: str | None = None
snapshot_at: datetime | None = None
fetched_at: datetime | None = None
source_url: str | None = None
checksum_sha256: str | None = None
crs: str | None = None
units: str | None = None
spatial_resolution_json: dict
temporal_coverage_json: dict
geographic_coverage_json: dict
observed_schema_json: dict
freshness_status: str
ingest_status: str
known_limitations_json: list
snapshot_metadata_json: dict
created_at: datetime | None = None
model_config = {"from_attributes": True}
class SourceRegistryDetailRead(BaseModel):
source: SourceRegistryRead
snapshots: list[SourceSnapshotRead]
class DatasetLineageEdgeRead(BaseModel):
id: UUID
parent_dataset_id: UUID
child_dataset_id: UUID
parent_dataset_version_id: UUID | None = None
child_dataset_version_id: UUID | None = None
relation_type: str
transformation_name: str
transformation_version: str | None = None
parameters_json: dict | None = None
input_checksum_sha256: str | None = None
output_checksum_sha256: str | None = None
created_at: datetime | None = None
model_config = {"from_attributes": True}
class DatasetQuarantineRead(BaseModel):
id: UUID
dataset_id: UUID | None = None
dataset_version_id: UUID | None = None
source_snapshot_id: UUID | None = None
stage: str
reason_code: str
details_json: dict | None = None
artifact_path: str | None = None
artifact_checksum_sha256: str | None = None
status: str
created_at: datetime | None = None
resolved_at: datetime | None = None
resolved_by: str | None = None
model_config = {"from_attributes": True}
class DatasetProvenanceRead(BaseModel):
dataset_id: UUID
source: SourceRegistryRead | None = None
snapshot: SourceSnapshotRead | None = None
data_contract_key: str | None = None
data_contract_version: str | None = None
validation_status: str | None = None
validation_report_json: dict | None = None
provenance_status: str | None = None
lineage_status: str | None = None
quarantine_status: str | None = None
lineage: list[DatasetLineageEdgeRead] = Field(default_factory=list)
quarantines: list[DatasetQuarantineRead] = Field(default_factory=list)
backend/app/services/coverage_registry_service.py
+5
View File
@@ -12,6 +12,7 @@ from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.models import Area, Dataset, Project
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
from app.schemas.coverage import (
CoverageBBox,
CoverageCatalogResponse,
@@ -463,6 +464,10 @@ class CoverageRegistryService:
for dataset in datasets:
if dataset.status != "ready" or dataset.source_name not in definition.materialized_source_names:
continue
# A source-name claim alone must not cause an unsafe artifact to
# appear as operational authoritative coverage.
if not DatasetConsumptionGate.eligible_for_authoritative_coverage(dataset):
continue
layer_names = definition.materialized_layer_names
if definition.contract.source_name == "digitaal_vlaanderen":
theme_sources = FLANDERS_THEME_DATASETS.get(theme, {})
backend/app/services/data_contract_validation.py
+2081
View File
File diff suppressed because it is too large Load Diff
backend/app/services/data_quarantine_service.py
+160
View File
@@ -0,0 +1,160 @@
"""Fail-closed quarantine decisions for validated data assets.
Persistence is intentionally delegated to the caller's transaction. This
module derives stable decisions from immutable validation reports and blocks a
quarantined or failed asset from training, production inference and derived
processing until an explicit, separately persisted release action exists.
"""
from __future__ import annotations
from dataclasses import dataclass
from enum import StrEnum
from hashlib import sha256
from typing import Any
import json
from app.core.errors import AppError
from app.services.data_contract_validation import QuarantineStatus, ValidationReport, ValidationStatus
class AssetUse(StrEnum):
TRAINING = "training"
PRODUCTION_INFERENCE = "production_inference"
DERIVED_PROCESSING = "derived_processing"
EXPORT = "export"
@dataclass(frozen=True)
class QuarantineDecision:
"""A deterministic, auditable quarantine decision for one report."""
asset_id: str
data_contract_key: str
data_contract_version: str
validation_report_sha256: str
validation_status: ValidationStatus
quarantine_status: QuarantineStatus
reason_codes: tuple[str, ...]
idempotency_key: str
requires_explicit_release: bool = False
@property
def eligible_for_use(self) -> bool:
return self.validation_status == ValidationStatus.PASSED and self.quarantine_status == QuarantineStatus.NOT_QUARANTINED
def to_dict(self) -> dict[str, Any]:
return {
"asset_id": self.asset_id,
"data_contract_key": self.data_contract_key,
"data_contract_version": self.data_contract_version,
"validation_report_sha256": self.validation_report_sha256,
"validation_status": self.validation_status.value,
"quarantine_status": self.quarantine_status.value,
"reason_codes": list(self.reason_codes),
"idempotency_key": self.idempotency_key,
"requires_explicit_release": self.requires_explicit_release,
"eligible_for_use": self.eligible_for_use,
}
class DataQuarantineService:
"""Derive and enforce safe use decisions from validation results."""
@staticmethod
def decide(
report: ValidationReport,
*,
previous: QuarantineDecision | None = None,
) -> QuarantineDecision:
"""Create a stable decision without silently releasing old quarantines.
A fresh passing validation report can be persisted as a new validated
version by the import transaction. It cannot automatically release an
existing quarantined record: the caller must explicitly record that
reviewed state transition against the new report/version.
"""
is_quarantined = report.validation_status == ValidationStatus.FAILED or report.quarantine_status == QuarantineStatus.QUARANTINED
failure_codes = tuple(
sorted(
{
issue.code
for issue in report.issues
if issue.severity.value == "error" or is_quarantined
}
)
)
requires_explicit_release = False
reason_codes = failure_codes
status = QuarantineStatus.QUARANTINED if is_quarantined else QuarantineStatus.NOT_QUARANTINED
if previous is not None and previous.quarantine_status == QuarantineStatus.QUARANTINED and not is_quarantined:
status = QuarantineStatus.QUARANTINED
requires_explicit_release = True
reason_codes = ("QUARANTINE_RELEASE_REQUIRES_EXPLICIT_PERSISTENCE",)
idempotency_key = _decision_key(
asset_id=report.asset_id,
contract_key=report.data_contract_key,
contract_version=report.data_contract_version,
report_sha256=report.report_sha256,
quarantine_status=status,
reason_codes=reason_codes,
requires_explicit_release=requires_explicit_release,
)
return QuarantineDecision(
asset_id=report.asset_id,
data_contract_key=report.data_contract_key,
data_contract_version=report.data_contract_version,
validation_report_sha256=report.report_sha256,
validation_status=report.validation_status,
quarantine_status=status,
reason_codes=reason_codes,
idempotency_key=idempotency_key,
requires_explicit_release=requires_explicit_release,
)
@staticmethod
def require_eligible(decision: QuarantineDecision, *, use: AssetUse) -> None:
"""Raise a typed error before a non-eligible artifact reaches a pipeline."""
if decision.eligible_for_use:
return
raise AppError(
code="DATASET_QUARANTINED",
message="Dataset is quarantined or failed validation and cannot enter this pipeline.",
status_code=409,
details={
"asset_id": decision.asset_id,
"use": use.value,
"quarantine_status": decision.quarantine_status.value,
"validation_status": decision.validation_status.value,
"reason_codes": list(decision.reason_codes),
"validation_report_sha256": decision.validation_report_sha256,
"idempotency_key": decision.idempotency_key,
"requires_explicit_release": decision.requires_explicit_release,
},
)
def _decision_key(
*,
asset_id: str,
contract_key: str,
contract_version: str,
report_sha256: str,
quarantine_status: QuarantineStatus,
reason_codes: tuple[str, ...],
requires_explicit_release: bool,
) -> str:
payload = {
"asset_id": asset_id,
"contract_key": contract_key,
"contract_version": contract_version,
"report_sha256": report_sha256,
"quarantine_status": quarantine_status.value,
"reason_codes": list(reason_codes),
"requires_explicit_release": requires_explicit_release,
}
return sha256(json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")).hexdigest()
backend/app/services/dataset_consumption_gate_service.py
+440
View File
@@ -0,0 +1,440 @@
"""Fail-closed provenance gates at data-consumption boundaries.
Import validation protects newly staged assets, but a persisted record can
subsequently become quarantined or have its provenance marked incomplete. The
callers of this service therefore re-check the durable Dataset state directly
before production inference, QA, derived processing, export, or authoritative
coverage reporting.
The only legacy relaxation is deliberately narrow: an *explicitly tagged*
fixture with no Phase-2 state can be used for fixture QA. A caller-provided
``fixture_mode`` flag alone never creates that trust claim. Fixture data can
never become a production inference, derived-processing, export or
authoritative-coverage input, and it never relaxes a recorded failed,
incomplete, or quarantined state.
"""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
import re
from typing import Any, Literal
from sqlalchemy import inspect as sa_inspect
from app.core.errors import AppError
from app.models import Dataset
DatasetConsumptionPurpose = Literal[
"production_inference",
"quality_assessment",
"reference_validation",
"derived_processing",
"authoritative_coverage",
"export",
]
_FIXTURE_SOURCE_KEYS = {"fixture", "test", "test_fixture", "test-fixture", "unit-test-fixture"}
_UNTRUSTED_SOURCE_KEYS = {"manual", "fixture", "experimental", "legacy_unknown"}
_CHECKSUM_SHA256 = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE)
_CONSUMABLE_SNAPSHOT_FRESHNESS = {"current", "not_applicable"}
_VALID_PURPOSES = {
"production_inference",
"quality_assessment",
"reference_validation",
"derived_processing",
"authoritative_coverage",
"export",
}
@dataclass(frozen=True)
class DatasetConsumptionDecision:
"""Auditable decision returned by the consumption gate."""
eligible: bool
purpose: DatasetConsumptionPurpose
fixture_legacy_exception: bool
reasons: tuple[str, ...]
evidence: dict[str, Any]
class DatasetConsumptionGate:
"""Evaluate durable provenance before a Dataset is consumed downstream."""
@staticmethod
def _value(dataset: Any, field: str, default: Any = None) -> Any:
if isinstance(dataset, Mapping):
return dataset.get(field, default)
return getattr(dataset, field, default)
@staticmethod
def _mapping(value: Any) -> dict[str, Any]:
return dict(value) if isinstance(value, Mapping) else {}
@classmethod
def _normalise(cls, value: Any) -> str:
return str(value or "").strip().lower()
@classmethod
def is_explicit_fixture(cls, dataset: Any) -> bool:
source = cls._normalise(cls._value(dataset, "source"))
source_name = cls._normalise(cls._value(dataset, "source_name"))
metadata = cls._mapping(cls._value(dataset, "metadata_json"))
source_metadata = cls._mapping(cls._value(dataset, "source_metadata"))
provenance = cls._mapping(cls._value(dataset, "provenance_metadata"))
return bool(
source in _FIXTURE_SOURCE_KEYS
or source_name in _FIXTURE_SOURCE_KEYS
or metadata.get("fixture") is True
or metadata.get("fixture_mode") is True
or source_metadata.get("fixture") is True
or source_metadata.get("fixture_mode") is True
or provenance.get("fixture") is True
or provenance.get("fixture_mode") is True
)
@classmethod
def _phase2_state_is_absent(cls, dataset: Any) -> bool:
fields = (
"data_contract_key",
"data_contract_version",
"validation_status",
"provenance_status",
"lineage_status",
"quarantine_status",
"source_registry_id",
"source_snapshot_id",
)
return all(cls._value(dataset, field) in {None, ""} for field in fields)
@staticmethod
def _is_transient_orm_dataset(dataset: Any) -> bool:
"""Recognize only unpersisted ORM fixtures, never database rows."""
if not isinstance(dataset, Dataset):
return False
try:
return bool(sa_inspect(dataset).transient)
except Exception: # pragma: no cover - defensive for unusual test doubles
return False
@classmethod
def _evidence(
cls,
dataset: Any,
purpose: DatasetConsumptionPurpose,
reference_task: str | None = None,
) -> dict[str, Any]:
source_registry = cls._value(dataset, "source_registry")
source_snapshot = cls._value(dataset, "source_snapshot")
source_policy = cls._mapping(cls._value(source_registry, "usage_policy_json"))
validation_authority = cls._mapping(source_policy.get("validation_authority"))
reference_approvals = cls._mapping(source_policy.get("reference_validation_approvals"))
return {
"dataset_id": str(cls._value(dataset, "id") or ""),
"purpose": purpose,
"dataset_status": cls._normalise(cls._value(dataset, "status")),
"source": cls._normalise(cls._value(dataset, "source")),
"source_name": cls._normalise(cls._value(dataset, "source_name")),
"data_contract_key": cls._value(dataset, "data_contract_key"),
"data_contract_version": cls._value(dataset, "data_contract_version"),
"validation_status": cls._normalise(cls._value(dataset, "validation_status")),
"provenance_status": cls._normalise(cls._value(dataset, "provenance_status")),
"lineage_status": cls._normalise(cls._value(dataset, "lineage_status")),
"quarantine_status": cls._normalise(cls._value(dataset, "quarantine_status")),
"checksum_sha256": cls._value(dataset, "checksum_sha256"),
"source_registry_id": str(cls._value(dataset, "source_registry_id") or ""),
"source_snapshot_id": str(cls._value(dataset, "source_snapshot_id") or ""),
"source_classification": cls._normalise(cls._value(source_registry, "classification")),
"source_key": cls._normalise(cls._value(source_registry, "source_key")),
"source_ground_truth_allowed": source_policy.get("ground_truth_allowed") is True,
"source_validation_authority": validation_authority,
"source_reference_validation_approvals": reference_approvals,
"source_authority_scope": cls._mapping(cls._value(source_registry, "authority_scope_json")),
"reference_task": cls._normalise(reference_task),
"snapshot_source_registry_id": str(cls._value(source_snapshot, "source_registry_id") or ""),
"snapshot_ingest_status": cls._normalise(cls._value(source_snapshot, "ingest_status")),
"snapshot_freshness_status": cls._normalise(cls._value(source_snapshot, "freshness_status")),
"snapshot_checksum_sha256": cls._value(source_snapshot, "checksum_sha256"),
}
@classmethod
def evaluate(
cls,
dataset: Any,
*,
purpose: DatasetConsumptionPurpose,
fixture_mode: bool = False,
reference_task: str | None = None,
) -> DatasetConsumptionDecision:
"""Return a stable decision without mutating the Dataset.
``fixture_mode`` can be supplied only by an explicitly fixture-only
caller. It is evidence for a QA fixture path, never a relaxation for
a production boundary or an explicit unsafe state.
"""
if purpose not in _VALID_PURPOSES:
raise ValueError(f"Unsupported dataset-consumption purpose: {purpose}")
evidence = cls._evidence(dataset, purpose, reference_task)
reasons: list[str] = []
explicit_fixture = cls.is_explicit_fixture(dataset)
phase2_absent = cls._phase2_state_is_absent(dataset)
# These are irrevocable safety states. They are checked before a
# fixture exception, so fixture rows cannot hide a bad recorded state.
if evidence["dataset_status"] in {"failed", "quarantined"}:
reasons.append("dataset_status_unsafe")
if evidence["quarantine_status"] == "quarantined":
reasons.append("dataset_quarantined")
if evidence["validation_status"] == "failed":
reasons.append("validation_failed")
if evidence["provenance_status"] == "incomplete":
reasons.append("provenance_incomplete")
if evidence["lineage_status"] == "incomplete":
reasons.append("lineage_incomplete")
if evidence["snapshot_ingest_status"] in {"failed", "quarantined"}:
reasons.append("source_snapshot_unsafe")
# A source family can be authoritative while an individual snapshot
# remains too old or insufficiently described to trust. Historical
# data that is intentionally valid needs an explicit
# ``not_applicable`` contract policy; an omitted, due or stale status
# cannot silently enter a production boundary.
if evidence["source_snapshot_id"] and evidence["snapshot_freshness_status"] not in _CONSUMABLE_SNAPSHOT_FRESHNESS:
reasons.append("source_snapshot_freshness_not_eligible")
if reasons:
return DatasetConsumptionDecision(
eligible=False,
purpose=purpose,
fixture_legacy_exception=False,
reasons=tuple(sorted(set(reasons))),
evidence=evidence,
)
# Fixtures are evidence for tests and QA only. They cannot become
# production inference, derived processing or export inputs merely by
# presenting a fixture flag at a public service boundary.
if phase2_absent and explicit_fixture:
if purpose == "quality_assessment":
return DatasetConsumptionDecision(
eligible=True,
purpose=purpose,
fixture_legacy_exception=True,
reasons=(),
evidence=evidence,
)
if purpose == "authoritative_coverage":
reasons.append("fixture_not_authoritative_coverage")
else:
reasons.append("fixture_qa_only")
elif phase2_absent and fixture_mode:
# `fixture_mode` is a caller flag, not a source trust claim. A
# manual/unknown production dataset must never self-designate as a
# fixture merely by supplying this parameter.
reasons.append("fixture_source_required")
# Existing service tests construct transient SQLAlchemy Dataset objects
# directly rather than retrieving a persisted row. A production
# `db.get()` result is persistent and never enters this branch. This
# compatibility path is intentionally unavailable to coverage, where
# a fixture must never appear authoritative.
if (
phase2_absent
and not reasons
and cls._is_transient_orm_dataset(dataset)
and purpose == "quality_assessment"
):
return DatasetConsumptionDecision(
eligible=True,
purpose=purpose,
fixture_legacy_exception=True,
reasons=(),
evidence=evidence,
)
# QA unit tests intentionally use projection objects
# instead of persisted ORM Datasets. Those projections cannot enter an
# application API boundary; keep the exception isolated to read-only
# candidate verification. Inference, reference validation, derived
# processing, export and coverage never accept a projection.
if phase2_absent and not reasons and not isinstance(dataset, Dataset) and purpose == "quality_assessment":
return DatasetConsumptionDecision(
eligible=True,
purpose=purpose,
fixture_legacy_exception=True,
reasons=(),
evidence=evidence,
)
if phase2_absent:
reasons.append("phase2_provenance_missing")
if evidence["dataset_status"] != "ready":
reasons.append("dataset_not_ready")
if evidence["validation_status"] != "passed":
reasons.append("validation_not_passed")
if evidence["provenance_status"] != "complete":
reasons.append("provenance_not_complete")
if evidence["lineage_status"] not in {"complete", "not_applicable"}:
reasons.append("lineage_not_complete")
if evidence["quarantine_status"] != "not_quarantined":
reasons.append("quarantine_status_not_clear")
if not evidence["data_contract_key"] or not evidence["data_contract_version"]:
reasons.append("data_contract_not_versioned")
if not _CHECKSUM_SHA256.fullmatch(str(evidence["checksum_sha256"] or "")):
reasons.append("dataset_checksum_invalid")
if not evidence["source_registry_id"]:
reasons.append("source_registry_missing")
if not evidence["source_snapshot_id"]:
reasons.append("source_snapshot_missing")
if evidence["source_registry_id"] and not evidence["source_classification"]:
reasons.append("source_registry_unresolved")
if evidence["source_snapshot_id"] and evidence["snapshot_ingest_status"] != "ingested":
reasons.append("source_snapshot_not_ingested")
if evidence["source_snapshot_id"] and not _CHECKSUM_SHA256.fullmatch(
str(evidence["snapshot_checksum_sha256"] or "")
):
reasons.append("source_snapshot_checksum_invalid")
if (
evidence["source_registry_id"]
and evidence["source_snapshot_id"]
and evidence["snapshot_source_registry_id"]
and evidence["source_registry_id"] != evidence["snapshot_source_registry_id"]
):
reasons.append("source_snapshot_registry_mismatch")
if (
_CHECKSUM_SHA256.fullmatch(str(evidence["checksum_sha256"] or ""))
and _CHECKSUM_SHA256.fullmatch(str(evidence["snapshot_checksum_sha256"] or ""))
and str(evidence["checksum_sha256"]).lower() != str(evidence["snapshot_checksum_sha256"]).lower()
):
reasons.append("source_snapshot_checksum_mismatch")
if evidence["source_key"] and evidence["source_name"] and evidence["source_key"] != evidence["source_name"]:
reasons.append("source_registry_identity_mismatch")
# Contextual, corroborative, authoritative and properly derived
# sources can serve their declared non-ground-truth roles once the
# full governed contract passes. Experimental/manual sources cannot
# cross a production boundary; only an explicitly marked fixture may
# participate in candidate QA.
experimental_source = (
evidence["source_classification"] == "experimental"
or evidence["source_key"] in _UNTRUSTED_SOURCE_KEYS
)
if experimental_source:
if purpose == "quality_assessment" and explicit_fixture:
pass
elif purpose == "quality_assessment":
reasons.append("experimental_source_requires_fixture_qa")
else:
reasons.append("experimental_source_not_allowed_for_purpose")
if purpose == "authoritative_coverage":
if evidence["source_classification"] != "authoritative":
reasons.append("coverage_source_not_authoritative")
if evidence["source_key"] and evidence["source_name"] and evidence["source_key"] != evidence["source_name"]:
reasons.append("coverage_source_identity_mismatch")
if purpose == "reference_validation":
if cls._normalise(cls._value(dataset, "dataset_role")) != "reference":
reasons.append("reference_dataset_role_required")
if evidence["source_classification"] != "authoritative":
reasons.append("reference_source_not_authoritative")
if evidence["source_ground_truth_allowed"] is not True:
reasons.append("reference_source_not_ground_truth_allowed")
if not evidence["reference_task"]:
reasons.append("reference_validation_task_required")
elif not cls._reference_task_is_approved(evidence):
reasons.append("reference_task_authority_not_approved")
return DatasetConsumptionDecision(
eligible=not reasons,
purpose=purpose,
fixture_legacy_exception=False,
reasons=tuple(sorted(set(reasons))),
evidence=evidence,
)
@classmethod
def assert_eligible(
cls,
dataset: Any,
*,
purpose: DatasetConsumptionPurpose,
fixture_mode: bool = False,
reference_task: str | None = None,
) -> DatasetConsumptionDecision:
decision = cls.evaluate(
dataset,
purpose=purpose,
fixture_mode=fixture_mode,
reference_task=reference_task,
)
if decision.eligible:
return decision
code = "DATASET_QUARANTINED" if any(
reason in {"dataset_status_unsafe", "dataset_quarantined", "validation_failed", "source_snapshot_unsafe"}
for reason in decision.reasons
) else "DATASET_PROVENANCE_INCOMPLETE"
raise AppError(
code=code,
message="Dataset cannot be consumed until its provenance and validation gates are satisfied.",
status_code=409,
details={
"dataset_id": decision.evidence["dataset_id"],
"purpose": purpose,
"reasons": list(decision.reasons),
"fixture_legacy_exception": decision.fixture_legacy_exception,
},
)
@classmethod
def eligible_for_authoritative_coverage(cls, dataset: Any) -> bool:
"""Return false instead of raising so coverage can report a gap safely."""
return cls.evaluate(dataset, purpose="authoritative_coverage").eligible
@staticmethod
def _reference_task_is_approved(evidence: Mapping[str, Any]) -> bool:
"""Require task-specific primary authority or an explicit zone approval.
``classification=authoritative`` is deliberately not a blanket
permission to serve as building truth. A source may be authoritative
for an address lifecycle or elevation product while remaining only
corroborative for footprint QA. A regional product that is marked
``*_pending_contract`` similarly remains blocked until an operator
records a narrow product-and-zone approval in its server-owned policy.
"""
task = str(evidence.get("reference_task") or "").strip().lower()
authority = DatasetConsumptionGate._normalise(
DatasetConsumptionGate._mapping(evidence.get("source_validation_authority")).get(task)
)
if authority == "primary":
return True
if authority not in {"approved", "approved_product_zone"}:
return False
approvals = DatasetConsumptionGate._mapping(evidence.get("source_reference_validation_approvals"))
approval = DatasetConsumptionGate._mapping(approvals.get(task))
if approval.get("approved") is not True:
return False
source_key = str(evidence.get("source_key") or "").strip().lower()
source_scope = DatasetConsumptionGate._mapping(evidence.get("source_authority_scope"))
source_zone = str(source_scope.get("zone") or source_scope.get("scope") or "").strip()
approved_keys = approval.get("source_keys")
approved_zones = approval.get("zones")
if not isinstance(approved_keys, list) or source_key not in {
str(value).strip().lower() for value in approved_keys
}:
return False
if not isinstance(approved_zones, list) or source_zone not in {
str(value).strip() for value in approved_zones
}:
return False
return True
backend/app/services/dataset_service.py
+2105 -46
View File
File diff suppressed because it is too large Load Diff
backend/app/services/demo_workflow_service.py
+41 -21
View File
@@ -12,6 +12,7 @@ from sqlalchemy.orm import Session
from app.models import Area, Dataset, DatasetVersion, Metric, Project, QualityCheck
from app.schemas.demo import DemoWorkflowResponse
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
from app.services.geojson_service import parse_geojson_payload
from app.services.qa_service import QaService
from app.services.quality_service import QualityService
@@ -30,17 +31,17 @@ class DemoWorkflowService:
EXPECTED_METRICS_FILENAME = "expected_qa_metrics.json"
@staticmethod
def _add_initial_version(db: Session, dataset: Dataset) -> None:
db.add(
DatasetVersion(
dataset_id=dataset.id,
version=1,
storage_path=dataset.storage_path,
checksum_sha256=dataset.checksum_sha256,
source_metadata=dataset.source_metadata,
provenance_metadata=dataset.provenance_metadata,
)
def _add_initial_version(db: Session, dataset: Dataset) -> DatasetVersion:
version = DatasetVersion(
dataset_id=dataset.id,
version=1,
storage_path=dataset.storage_path,
checksum_sha256=dataset.checksum_sha256,
source_metadata=dataset.source_metadata,
provenance_metadata=dataset.provenance_metadata,
)
db.add(version)
return version
@staticmethod
def _repo_root() -> Path:
@@ -233,24 +234,34 @@ class DemoWorkflowService:
crs=metadata.get("crs"),
bounds_json=metadata.get("bounds_json"),
metadata_json=metadata,
status="ready",
status="validating",
)
db.add(dataset)
DemoWorkflowService._add_initial_version(db, dataset)
version = DemoWorkflowService._add_initial_version(db, dataset)
is_ready = DerivedDatasetGovernanceService.govern_vector(
db,
dataset=dataset,
dataset_version=version,
feature_collection=payload,
source_key="fixture",
operation="demo.fixture_vector",
operation_parameters={"fixture_name": filename, "role": role},
)
if is_ready:
VectorFeatureService.persist_geojson_features(
db=db,
dataset_id=dataset.id,
payload=payload,
feature_class=reference_layer_name or "building",
commit=False,
)
db.commit()
db.refresh(dataset)
VectorFeatureService.persist_geojson_features(
db=db,
dataset_id=dataset.id,
payload=payload,
feature_class=reference_layer_name or "building",
)
return dataset
@staticmethod
def _create_demo_raster_bytes() -> bytes:
numpy = importlib.import_module("numpy")
rasterio = importlib.import_module("rasterio")
rasterio_io = importlib.import_module("rasterio.io")
rasterio_transform = importlib.import_module("rasterio.transform")
@@ -318,10 +329,19 @@ class DemoWorkflowService:
crs=metadata.get("crs"),
bounds_json=bounds_json,
metadata_json=metadata,
status="ready",
status="validating",
)
db.add(dataset)
DemoWorkflowService._add_initial_version(db, dataset)
version = DemoWorkflowService._add_initial_version(db, dataset)
DerivedDatasetGovernanceService.govern_raster(
db,
dataset=dataset,
dataset_version=version,
raster_metadata=metadata,
source_key="fixture",
operation="demo.fixture_raster",
operation_parameters={"fixture_name": DemoWorkflowService.RASTER_FILENAME},
)
db.commit()
db.refresh(dataset)
return dataset
backend/app/services/derived_dataset_governance_service.py
+516
View File
@@ -0,0 +1,516 @@
"""Governance for persisted derived and fixture datasets.
Dataset importers own raw-source ingestion. This small service owns the
other persistence boundary: artifacts produced inside the workbench (vector
and raster operations) and the explicitly local demo fixtures. It is kept
separate from :mod:`dataset_service` so an operation can never create a ready
dataset without a source registry binding, immutable snapshot, validation
report and, for derived results, a durable lineage edge.
The ``Session.query`` capability check deliberately preserves lightweight
unit-test doubles used by pre-Phase-2 tests. Real SQLAlchemy sessions always
take the governed branch; the compatibility branch is not reachable in the
application runtime.
"""
from __future__ import annotations
from datetime import datetime, timezone
from hashlib import sha256
import json
import re
from typing import Any, Mapping
from sqlalchemy.orm import Session
from app.models import Dataset, DatasetVersion
from app.services.data_contract_validation import (
IssueSeverity,
LineageStatus,
LineageEvidence,
ProvenanceStatus,
QuarantineStatus,
TransformationEvidence,
ValidationIssue,
ValidationReport,
ValidationStatus,
build_raster_ingest_input,
build_vector_ingest_input,
validate_registered_asset,
)
from app.services.data_quarantine_service import DataQuarantineService
from app.services.dataset_consumption_gate_service import DatasetConsumptionDecision, DatasetConsumptionGate
from app.services.source_registry_service import SourceRegistryService
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
_CANONICAL_VECTOR_CRS = "EPSG:4326"
class DerivedDatasetGovernanceService:
"""Apply Phase-2 provenance rules to non-importer Dataset creation.
Callers add the dataset and first immutable version, then call one of the
``govern_*`` methods before committing. A failed contract deliberately
leaves the artifact and its durable quarantine record in the transaction;
it is never silently promoted to ``ready``.
"""
@staticmethod
def persistence_available(db: Session) -> bool:
"""Return whether this is a real ORM persistence session.
Historical unit tests use minimal fakes with ``add``/``commit`` only.
Keeping that explicitly isolated avoids pretending a fake test store
has source-registry guarantees while production remains fail-closed.
"""
return callable(getattr(db, "query", None)) and callable(getattr(db, "flush", None))
@classmethod
def govern_vector(
cls,
db: Session,
*,
dataset: Dataset,
dataset_version: DatasetVersion,
feature_collection: Mapping[str, Any],
source_key: str,
operation: str,
parent_dataset: Dataset | None = None,
operation_parameters: Mapping[str, Any] | None = None,
) -> bool:
"""Validate and bind a vector result, returning ``True`` when ready."""
if not cls.persistence_available(db):
# Explicit compatibility for historical minimal test fixtures.
# Production sessions always have query/flush and never take this
# branch.
dataset.status = "ready"
return True
parent_gate = cls._parent_derived_processing_gate(parent_dataset)
metadata = dict(dataset.metadata_json or {})
output_crs = str(metadata.get("crs") or dataset.crs or _CANONICAL_VECTOR_CRS)
source = SourceRegistryService.ensure_server_owned_source(db, source_key)
snapshot = cls._record_snapshot(
db,
source_key=source_key,
dataset=dataset,
operation=operation,
source_crs=output_crs,
spatial_resolution=metadata.get("resolution_json") or {"status": "not_applicable"},
geographic_coverage={"bounds": metadata.get("bounds_json") or dataset.bounds_json},
observed_schema={
"dataset_type": "vector",
"geometry_types": metadata.get("geometry_types") or [],
"feature_count": metadata.get("feature_count"),
},
)
lineage = cls._lineage_evidence(parent_dataset, operation, operation_parameters)
report = validate_registered_asset(
build_vector_ingest_input(
asset_id=str(dataset.id),
source_crs=output_crs,
storage_crs=output_crs,
feature_collection=feature_collection,
checksum_sha256=dataset.checksum_sha256,
computed_checksum_sha256=dataset.checksum_sha256,
source_registry_id=str(source.id),
source_snapshot_id=str(snapshot.id),
imported_at=dataset.imported_at or datetime.now(timezone.utc),
metadata=cls._contract_metadata(metadata, source.license_name),
observed_at=dataset.observed_at,
valid_from=dataset.valid_from,
valid_to=dataset.valid_to,
temporal_unknown_reason=cls._temporal_unknown_reason(dataset),
source_version=dataset.source_version,
source_version_unknown_reason=cls._source_version_unknown_reason(dataset),
lineage=lineage,
)
)
if parent_gate is not None and not parent_gate.eligible:
report = cls._with_parent_gate_failure(report, parent_gate)
return cls._apply(
db,
dataset=dataset,
dataset_version=dataset_version,
source=source,
snapshot=snapshot,
report=report,
stage="derived_vector_validation",
parent_dataset=parent_dataset,
operation=operation,
operation_parameters=operation_parameters,
)
@classmethod
def govern_raster(
cls,
db: Session,
*,
dataset: Dataset,
dataset_version: DatasetVersion,
raster_metadata: Mapping[str, Any],
source_key: str,
operation: str,
parent_dataset: Dataset | None = None,
operation_parameters: Mapping[str, Any] | None = None,
) -> bool:
"""Validate and bind a raster result, returning ``True`` when ready."""
if not cls.persistence_available(db):
dataset.status = "ready"
return True
parent_gate = cls._parent_derived_processing_gate(parent_dataset)
metadata = dict(raster_metadata or {})
output_crs = str(metadata.get("crs") or dataset.crs or "") or None
source = SourceRegistryService.ensure_server_owned_source(db, source_key)
snapshot = cls._record_snapshot(
db,
source_key=source_key,
dataset=dataset,
operation=operation,
source_crs=output_crs,
spatial_resolution=cls._raster_resolution(metadata, output_crs),
geographic_coverage={"bounds": metadata.get("bounds") or dataset.bounds_json},
observed_schema={
"dataset_type": "raster",
"width": metadata.get("width"),
"height": metadata.get("height"),
"band_count": metadata.get("band_count"),
"dtype": metadata.get("dtype"),
},
)
lineage = cls._lineage_evidence(parent_dataset, operation, operation_parameters)
report = validate_registered_asset(
build_raster_ingest_input(
asset_id=str(dataset.id),
source_crs=output_crs,
storage_crs=output_crs,
raster_profile=metadata,
bounds=metadata.get("bounds") or dataset.bounds_json,
resolution=cls._raster_resolution(metadata, output_crs),
checksum_sha256=dataset.checksum_sha256,
computed_checksum_sha256=dataset.checksum_sha256,
source_registry_id=str(source.id),
source_snapshot_id=str(snapshot.id),
imported_at=dataset.imported_at or datetime.now(timezone.utc),
metadata=cls._contract_metadata(metadata, source.license_name),
observed_at=dataset.observed_at,
valid_from=dataset.valid_from,
valid_to=dataset.valid_to,
temporal_unknown_reason=cls._temporal_unknown_reason(dataset),
source_version=dataset.source_version,
source_version_unknown_reason=cls._source_version_unknown_reason(dataset),
lineage=lineage,
)
)
if parent_gate is not None and not parent_gate.eligible:
report = cls._with_parent_gate_failure(report, parent_gate)
return cls._apply(
db,
dataset=dataset,
dataset_version=dataset_version,
source=source,
snapshot=snapshot,
report=report,
stage="derived_raster_validation",
parent_dataset=parent_dataset,
operation=operation,
operation_parameters=operation_parameters,
)
@classmethod
def _record_snapshot(
cls,
db: Session,
*,
source_key: str,
dataset: Dataset,
operation: str,
source_crs: str | None,
spatial_resolution: Mapping[str, Any] | None,
geographic_coverage: Mapping[str, Any] | None,
observed_schema: Mapping[str, Any] | None,
):
checksum = cls._checksum_or_placeholder(dataset.checksum_sha256)
# A derived artifact has a distinct creation event even when its bytes
# equal a prior output. Include the immutable dataset id so source
# snapshots never collide on a different fetched_at timestamp.
snapshot_key = f"{source_key}:{operation}:{dataset.id}:{checksum}"
return SourceRegistryService.record_snapshot(
db,
source_key=source_key,
snapshot_key=snapshot_key,
checksum_sha256=checksum,
source_version=dataset.source_version or f"{operation}:1.0.0",
snapshot_at=dataset.observed_at,
fetched_at=dataset.imported_at or datetime.now(timezone.utc),
crs=source_crs,
units=cls._units_for_crs(source_crs),
spatial_resolution=dict(spatial_resolution or {"status": "unknown"}),
temporal_coverage={
"observed_at": cls._datetime_value(dataset.observed_at),
"valid_from": cls._datetime_value(dataset.valid_from),
"valid_to": cls._datetime_value(dataset.valid_to),
},
geographic_coverage=dict(geographic_coverage or {"status": "unknown"}),
observed_schema=dict(observed_schema or {"status": "unknown"}),
# A transform cannot establish source freshness. With no source
# observation it is not applicable; with one it still needs an
# explicit policy review rather than a fabricated "current" flag.
freshness_status="not_applicable" if dataset.observed_at is None else "review_required",
ingest_status="ingested",
known_limitations=[
"Derived and fixture artifacts inherit no automatic source authority beyond their explicit registry entry and lineage.",
],
snapshot_metadata={
"operation": operation,
"dataset_id": str(dataset.id),
"storage_path": dataset.storage_path,
"artifact_checksum_sha256": dataset.checksum_sha256,
},
)
@classmethod
def _apply(
cls,
db: Session,
*,
dataset: Dataset,
dataset_version: DatasetVersion,
source: Any,
snapshot: Any,
report: ValidationReport,
stage: str,
parent_dataset: Dataset | None,
operation: str,
operation_parameters: Mapping[str, Any] | None,
) -> bool:
fields = report.persistence_fields()
dataset.validation_report_json = fields["validation_report_json"]
dataset_version.validation_report_json = fields["validation_report_json"]
SourceRegistryService.bind_dataset_provenance(
dataset,
source=source,
snapshot=snapshot,
data_contract_key=fields["data_contract_key"],
data_contract_version=fields["data_contract_version"],
validation_status=fields["validation_status"],
provenance_status=fields["provenance_status"],
lineage_status=fields["lineage_status"],
)
SourceRegistryService.bind_dataset_version_provenance(
dataset_version,
source=source,
snapshot=snapshot,
data_contract_key=fields["data_contract_key"],
data_contract_version=fields["data_contract_version"],
validation_status=fields["validation_status"],
provenance_status=fields["provenance_status"],
lineage_status=fields["lineage_status"],
)
# IDs for DatasetVersion defaults exist only after the caller adds and
# flushes both rows. The operation services call this before commit.
db.flush()
if parent_dataset is not None:
SourceRegistryService.record_lineage_edge(
db,
parent_dataset_id=parent_dataset.id,
child_dataset_id=dataset.id,
parent_dataset_version_id=cls._latest_parent_version_id(db, parent_dataset),
child_dataset_version_id=dataset_version.id,
relation_type="derived_from",
transformation_name=operation,
transformation_version="1.0.0",
parameters=dict(operation_parameters or {}),
input_checksum_sha256=cls._valid_checksum(parent_dataset.checksum_sha256),
output_checksum_sha256=cls._valid_checksum(dataset.checksum_sha256),
)
decision = DataQuarantineService.decide(report)
if decision.eligible_for_use:
dataset.status = "ready"
dataset.quarantine_status = "not_quarantined"
return True
SourceRegistryService.quarantine_dataset(
db,
dataset=dataset,
dataset_version=dataset_version,
source_snapshot=snapshot,
stage=stage,
reason_code=(decision.reason_codes[0] if decision.reason_codes else "DATA_CONTRACT_FAILED"),
details={"validation_report": report.to_dict(), "quarantine_decision": decision.to_dict()},
artifact_path=dataset.storage_path,
artifact_checksum_sha256=cls._valid_checksum(dataset.checksum_sha256),
)
return False
@staticmethod
def _contract_metadata(metadata: Mapping[str, Any], license_name: str) -> dict[str, Any]:
values = dict(metadata)
values.setdefault("license", license_name)
return values
@classmethod
def _lineage_evidence(
cls,
parent_dataset: Dataset | None,
operation: str,
operation_parameters: Mapping[str, Any] | None,
) -> LineageEvidence:
upstream_ids: tuple[str, ...] = ()
upstream_checksums: tuple[str, ...] = ()
if parent_dataset is not None:
upstream_ids = (str(parent_dataset.id),)
# An invalid/missing parent checksum intentionally fails the
# derived contract rather than inventing traceability. The same
# applies to a legacy/unvalidated parent: it may remain visible
# as evidence, but cannot create a new ready derived asset.
parent_is_governed = (
parent_dataset.status == "ready"
and parent_dataset.quarantine_status == "not_quarantined"
and parent_dataset.validation_status == "passed"
and parent_dataset.provenance_status == "complete"
and parent_dataset.source_registry_id is not None
and parent_dataset.source_snapshot_id is not None
)
upstream_checksums = (
parent_dataset.checksum_sha256 if parent_is_governed else "parent_dataset_not_governed",
)
transform_checksum = cls._stable_hash(
{"operation": operation, "version": "1.0.0", "parameters": dict(operation_parameters or {})}
)
return LineageEvidence(
upstream_asset_ids=upstream_ids,
upstream_checksums_sha256=upstream_checksums,
transformations=(
TransformationEvidence(
name=operation,
version="1.0.0",
checksum_sha256=transform_checksum,
),
),
)
@staticmethod
def _parent_derived_processing_gate(parent_dataset: Dataset | None) -> DatasetConsumptionDecision | None:
"""Evaluate the durable parent boundary before creating a ready child.
We deliberately turn a rejected parent into a child validation failure
(instead of simply raising): the output is then persisted with its
source snapshot, validation evidence and durable quarantine record.
This makes an attempted derivation from a manual or experimental
dataset observable and prevents a caller from bypassing the boundary
by invoking the governance service directly.
"""
if parent_dataset is None:
return None
return DatasetConsumptionGate.evaluate(parent_dataset, purpose="derived_processing")
@staticmethod
def _with_parent_gate_failure(
report: ValidationReport,
decision: DatasetConsumptionDecision,
) -> ValidationReport:
"""Attach an auditable, fail-closed lineage failure to a report."""
issue = ValidationIssue(
code="PARENT_DATASET_NOT_ELIGIBLE_FOR_DERIVED_PROCESSING",
category="lineage",
field="lineage.parent_dataset",
message="Parent dataset failed the governed derived-processing consumption gate.",
severity=IssueSeverity.ERROR,
expected="eligible governed parent dataset",
observed={
"dataset_id": decision.evidence.get("dataset_id"),
"reasons": list(decision.reasons),
"source_key": decision.evidence.get("source_key"),
"source_classification": decision.evidence.get("source_classification"),
},
)
return ValidationReport(
asset_id=report.asset_id,
data_contract_key=report.data_contract_key,
data_contract_version=report.data_contract_version,
contract_fingerprint_sha256=report.contract_fingerprint_sha256,
validation_status=ValidationStatus.FAILED,
provenance_status=(
ProvenanceStatus.INCOMPLETE
if report.provenance_status == ProvenanceStatus.COMPLETE
else report.provenance_status
),
lineage_status=LineageStatus.INCOMPLETE,
quarantine_status=QuarantineStatus.QUARANTINED,
validation_scope=report.validation_scope,
checked_at=report.checked_at,
issues=(*report.issues, issue),
)
@staticmethod
def _latest_parent_version_id(db: Session, dataset: Dataset):
version = (
db.query(DatasetVersion)
.filter(DatasetVersion.dataset_id == dataset.id)
.order_by(DatasetVersion.version.desc())
.first()
)
return version.id if version is not None else None
@staticmethod
def _raster_resolution(metadata: Mapping[str, Any], crs: str | None) -> dict[str, Any] | None:
values = metadata.get("resolution")
if not isinstance(values, (list, tuple)) or len(values) < 2:
return None
try:
return {"x": abs(float(values[0])), "y": abs(float(values[1])), "unit": DerivedDatasetGovernanceService._resolution_unit(crs)}
except (TypeError, ValueError):
return None
@staticmethod
def _resolution_unit(crs: str | None) -> str:
return "degree" if str(crs or "").upper() == _CANONICAL_VECTOR_CRS else "m"
@staticmethod
def _units_for_crs(crs: str | None) -> str:
return "degrees" if str(crs or "").upper() == _CANONICAL_VECTOR_CRS else "metres"
@staticmethod
def _temporal_unknown_reason(dataset: Dataset) -> str | None:
if dataset.observed_at is not None:
return None
return "Derived or fixture artifact inherits no precise observation timestamp from its input."
@staticmethod
def _source_version_unknown_reason(dataset: Dataset) -> str | None:
if dataset.source_version:
return None
return "Derived or fixture artifact has no source edition; the transform version is recorded separately."
@staticmethod
def _datetime_value(value: datetime | None) -> str | None:
return value.astimezone(timezone.utc).isoformat() if value is not None else None
@staticmethod
def _stable_hash(value: Mapping[str, Any]) -> str:
return sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")).hexdigest()
@staticmethod
def _valid_checksum(value: str | None) -> str | None:
normalized = str(value or "").strip().lower()
return normalized if _SHA256.fullmatch(normalized) else None
@classmethod
def _checksum_or_placeholder(cls, value: str | None) -> str:
checksum = cls._valid_checksum(value)
if checksum is not None:
return checksum
# The contract receives the original invalid/missing checksum and
# quarantines it. A deterministic placeholder only permits storing
# the rejected snapshot without fabricating a valid artifact hash.
return cls._stable_hash({"invalid_dataset_checksum": value or "missing"})
backend/app/services/detection_service.py
+80 -10
View File
@@ -19,10 +19,12 @@ from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project, Vect
from app.schemas.detection import DetectionListResponse, DetectionRead, DetectionRunListResponse, DetectionRunRead, DetectionRunResponse
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
from app.services.detection_qa_service import DetectionQaService
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
from app.services.model_asset_catalog_service import ModelAssetCatalogService
from app.services.model_registry_service import ModelRegistryService
from app.services.qa_service import QaService
from app.services.quality_service import QualityService
from app.services.runtime_model_provenance_service import RuntimeModelProvenance, RuntimeModelProvenanceService
from app.services.temporal_compatibility_service import TemporalCompatibilityService
from app.services.yolo_adapter import YoloDetectionAdapter
@@ -97,6 +99,18 @@ class DetectionService:
if model.model_id == resolved_settings.yolo_model_id and resolved_settings.yolo_enforce_validation_scope:
DetectionService._validate_model_area_scope(db, dataset, resolved_settings)
# Never enter a production inference path with a persisted dataset
# that has failed validation, incomplete provenance, or an active
# quarantine. Fixture detection is a separate QA/test-only path.
if model.model_id == "manual-fixture-detector":
DatasetConsumptionGate.assert_eligible(
dataset,
purpose="quality_assessment",
fixture_mode=True,
)
elif model.model_id == resolved_settings.yolo_model_id and model.configured:
DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
run_parameters = {
"model_id": model.model_id,
"model_asset_id": selected_model_asset.model_asset_id if selected_model_asset else None,
@@ -352,15 +366,6 @@ class DetectionService:
reference_dataset,
)
detections = DetectionService._query_detection_rows(
db,
analysis_run_id=analysis_run_id,
dataset_id=run.dataset_id,
class_name=class_name,
min_confidence=min_confidence,
)
raw_candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in detections]
candidate_geometries = raw_candidate_geometries
run_parameters = run.parameters_json if isinstance(run.parameters_json, dict) else {}
manifest_path = DetectionQaService.tile_manifest_path(run_parameters)
resolved_settings = get_settings()
@@ -375,6 +380,33 @@ class DetectionService:
status_code=422,
)
fixture_parameters = run_parameters.get("parameters_json")
fixture_mode = bool(
run.model_name == "manual-fixture-detector"
and isinstance(fixture_parameters, dict)
and fixture_parameters.get("fixture_mode") is True
)
DatasetConsumptionGate.assert_eligible(
candidate_dataset,
purpose="quality_assessment",
fixture_mode=fixture_mode,
)
DatasetConsumptionGate.assert_eligible(
reference_dataset,
purpose="reference_validation",
reference_task="building_validation",
)
detections = DetectionService._query_detection_rows(
db,
analysis_run_id=analysis_run_id,
dataset_id=run.dataset_id,
class_name=class_name,
min_confidence=min_confidence,
)
raw_candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in detections]
candidate_geometries = raw_candidate_geometries
coverage = None
if manifest_path:
manifest = DetectionService._load_tile_manifest(manifest_path, resolved_settings.yolo_max_tiles)
@@ -728,6 +760,19 @@ class DetectionService:
) -> tuple[list[Detection], dict[str, Any]]:
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles)
model_path = Path(settings.yolo_model_path or "").expanduser()
runtime_model_provenance = RuntimeModelProvenanceService.validate_for_production_runtime(
db=db,
model_path=model_path,
model_id=model_name,
task_type="object_detection",
expected_model_version=model_version,
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
)
DetectionService._attach_runtime_model_provenance(
analysis_run,
job,
runtime_model_provenance,
)
adapter = yolo_adapter_class(settings)
model = adapter.load_model(model_path)
allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)}
@@ -785,7 +830,10 @@ class DetectionService:
"y_max": float(bbox[3]),
},
source_tile_path=candidate["source_tile_path"],
properties_json=candidate["properties"],
properties_json={
**candidate["properties"],
"runtime_model_provenance": runtime_model_provenance.as_dict(),
},
)
db.add(detection)
persisted.append(detection)
@@ -796,8 +844,30 @@ class DetectionService:
"raw_detection_count": len(candidates),
"suppressed_detection_count": len(candidates) - len(filtered_candidates),
"duplicate_iou_threshold": float(settings.yolo_duplicate_iou_threshold),
"runtime_model_provenance": runtime_model_provenance.as_dict(),
}
@staticmethod
def _attach_runtime_model_provenance(
analysis_run: AnalysisRun,
job: Job,
provenance: RuntimeModelProvenance,
) -> None:
"""Persist byte-bound model evidence with the run before adapter loading.
Individual detections retain the same evidence in ``properties_json``;
this run-level copy is the compact audit root for a complete inference.
Assigning fresh dictionaries matters for SQLAlchemy JSON change tracking.
"""
evidence = provenance.as_dict()
analysis_parameters = dict(analysis_run.parameters_json or {})
analysis_parameters["runtime_model_provenance"] = evidence
analysis_run.parameters_json = analysis_parameters
job_parameters = dict(job.parameters_json or {})
job_parameters["runtime_model_provenance"] = evidence
job.parameters_json = job_parameters
@staticmethod
def _canonical_class_name(value: Any) -> str:
return str(value or "").strip().casefold()
backend/app/services/export_service.py
+34
View File
@@ -23,6 +23,7 @@ from app.schemas.flood_hazard import FloodHazardPartitionSelectionRequest, Flood
from app.schemas.temporal import TemporalComparisonRequest
from app.schemas.thematic_raster import ThematicRasterSelectionRequest
from app.services.dataset_service import DatasetService
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
from app.services.detection_service import DetectionService
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
from app.services.segmentation_service import SegmentationService
@@ -34,12 +35,39 @@ from app.services.vector_feature_service import VectorFeatureService
class ExportService:
@staticmethod
def _assert_run_source_dataset_exportable(db: Session, run: AnalysisRun) -> Dataset:
"""Block an output export when its persisted source dataset is unsafe."""
if not run.dataset_id:
raise AppError(
code="DATASET_PROVENANCE_INCOMPLETE",
message="Analysis output cannot be exported without a persisted source dataset.",
status_code=409,
)
dataset = db.get(Dataset, run.dataset_id)
if not dataset or dataset.project_id != run.project_id:
raise AppError(code="DATASET_NOT_FOUND", message="Analysis source dataset not found", status_code=404)
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
return dataset
@staticmethod
def export_map_result(
db: Session,
payload: MapResultExportRequest,
) -> ExportCreateResponse:
if payload.mode == "evolution":
earlier_dataset = db.get(Dataset, payload.earlier_dataset_id)
later_dataset = db.get(Dataset, payload.later_dataset_id)
if (
not earlier_dataset
or not later_dataset
or earlier_dataset.project_id != payload.project_id
or later_dataset.project_id != payload.project_id
):
raise AppError(code="DATASET_NOT_FOUND", message="Temporal export dataset not found", status_code=404)
DatasetConsumptionGate.assert_eligible(earlier_dataset, purpose="export")
DatasetConsumptionGate.assert_eligible(later_dataset, purpose="export")
comparison = TemporalAnalysisService.compare(
db,
project_id=payload.project_id,
@@ -86,6 +114,7 @@ class ExportService:
dataset = db.get(Dataset, payload.dataset_id)
if not dataset or dataset.project_id != payload.project_id:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
if dataset.dataset_type in DatasetService.VECTOR_TYPES:
if payload.partitioned:
return ExportService.export_partitioned_vector_selection_geojson(
@@ -219,6 +248,7 @@ class ExportService:
limit: int = 1000,
name: str | None = None,
) -> ExportCreateResponse:
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
if dataset.source_name != "vmm_vha_bathymetry_profiles" or partition_scope_key != "flanders":
raise AppError(
code="PARTITIONED_VECTOR_EXPORT_UNSUPPORTED",
@@ -308,6 +338,7 @@ class ExportService:
details={"dataset_type": dataset.dataset_type},
status_code=400,
)
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
selection_kwargs: dict[str, Any] = {
"dataset_id": dataset_id,
@@ -374,6 +405,7 @@ class ExportService:
details={"dataset_type": dataset.dataset_type},
status_code=400,
)
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
feature_collection = DatasetService.get_dataset_geojson(db, dataset_id)
filename = ExportService._filename(name, f"{dataset.id}.geojson", ".geojson")
@@ -401,6 +433,7 @@ class ExportService:
run = db.get(AnalysisRun, analysis_run_id)
if not run or run.analysis_type != "detection":
raise AppError(code="DETECTION_RUN_NOT_FOUND", message="Detection run not found", status_code=404)
ExportService._assert_run_source_dataset_exportable(db, run)
feature_collection = DetectionService.detections_to_geojson(db, analysis_run_id=analysis_run_id)
filename = ExportService._filename(name, f"{run.id}-detections.geojson", ".geojson")
@@ -428,6 +461,7 @@ class ExportService:
run = db.get(AnalysisRun, analysis_run_id)
if not run or run.analysis_type != "segmentation":
raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404)
ExportService._assert_run_source_dataset_exportable(db, run)
feature_collection = SegmentationService.segmentations_to_geojson(db, analysis_run_id=analysis_run_id)
filename = ExportService._filename(name, f"{run.id}-segmentations.geojson", ".geojson")
backend/app/services/model_registry_service.py
+55 -9
View File
@@ -9,6 +9,7 @@ from app.services.segmentation_adapter import (
SamSegmentationAdapter,
YoloSegmentationAdapter,
)
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
from app.services.yolo_adapter import YoloDetectionAdapter
from app.core.errors import AppError
@@ -144,9 +145,24 @@ class ModelRegistryService:
elif not model_path.exists() or not model_path.is_file():
limitation = "YOLO_SEG_MODEL_PATH does not point to an existing local model file. GeoIntel will not download segmentation model weights automatically."
else:
configured = True
status = "configured"
limitation = "Configured for local YOLO segmentation inference over an existing raster tile manifest."
try:
RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id=settings.yolo_seg_model_id,
task_type="segmentation",
expected_model_version=settings.yolo_seg_model_version,
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
)
except AppError as exc:
status = "contract_incomplete"
limitation = (
"Configured YOLO segmentation weights are not runnable until their immutable "
f"runtime provenance sidecar validates: {exc.message}"
)
else:
configured = True
status = "configured"
limitation = "Configured for local YOLO segmentation inference over an existing raster tile manifest."
return DetectionModelCapability(
model_id=settings.yolo_seg_model_id,
@@ -186,9 +202,24 @@ class ModelRegistryService:
elif not model_path.exists() or not model_path.is_file():
limitation = "SAM_MODEL_PATH does not point to an existing local model file. GeoIntel will not download segmentation model weights automatically."
else:
configured = True
status = "configured"
limitation = "Configured for local class-agnostic SAM segmentation over an existing raster tile manifest."
try:
RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id=settings.sam_model_id,
task_type="segmentation",
expected_model_version=settings.sam_model_version,
allowed_frameworks=("ultralytics/sam", "sam", "ultralytics", "pytorch"),
)
except AppError as exc:
status = "contract_incomplete"
limitation = (
"Configured SAM weights are not runnable until their immutable runtime provenance "
f"sidecar validates: {exc.message}"
)
else:
configured = True
status = "configured"
limitation = "Configured for local class-agnostic SAM segmentation over an existing raster tile manifest."
return DetectionModelCapability(
model_id=settings.sam_model_id,
@@ -233,9 +264,24 @@ class ModelRegistryService:
status = "accelerator_unavailable"
limitation = exc.message
else:
configured = True
status = "configured"
limitation = "Configured for local YOLO inference over an existing raster tile manifest within its validated area scope."
try:
RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id=settings.yolo_model_id,
task_type="object_detection",
expected_model_version=settings.yolo_model_version,
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
)
except AppError as exc:
status = "contract_incomplete"
limitation = (
"Configured YOLO weights are not runnable until their immutable runtime provenance "
f"sidecar validates: {exc.message}"
)
else:
configured = True
status = "configured"
limitation = "Configured for local YOLO inference over an existing raster tile manifest within its validated area scope."
return DetectionModelCapability(
model_id=settings.yolo_model_id,
backend/app/services/qa_service.py
+7 -1
View File
@@ -11,10 +11,10 @@ from shapely.geometry.base import BaseGeometry
from shapely.strtree import STRtree
from shapely.ops import unary_union
from shapely.validation import make_valid
from shapely.geometry import shape
from app.core.errors import AppError
from app.models import Area, Dataset
from app.schemas.qa import QaProviderComparisonResult
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
from app.services.vector_operations_service import VectorOperationsService
@@ -264,6 +264,12 @@ class QaService:
reference_dataset_id,
expected_project_id=project_id,
)
DatasetConsumptionGate.assert_eligible(candidate_dataset, purpose="quality_assessment")
DatasetConsumptionGate.assert_eligible(
reference_dataset,
purpose="reference_validation",
reference_task="building_validation",
)
area_geometry = QaService._validate_area(
db,
backend/app/services/raster_operations_service.py
+28 -15
View File
@@ -13,6 +13,7 @@ from shapely.validation import make_valid
from app.core.errors import AppError
from app.models import Area, Dataset, DatasetVersion
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
from app.services.raster_service import extract_raster_metadata
from app.services.storage_service import StorageService
@@ -334,7 +335,10 @@ class RasterOperationsService:
dataset_type="raster",
source=f"operation:{operation_name}",
dataset_role="derived",
source_name=source_dataset.source_name,
# The source identity of this artifact is the server-owned
# derived-operation registry entry. The parent remains explicit
# in provenance and the lineage edge below.
source_name="derived",
source_metadata=source_dataset.source_metadata,
provenance_metadata=provenance,
imported_at=datetime.now(timezone.utc),
@@ -360,22 +364,31 @@ class RasterOperationsService:
resolution_json=metadata_payload.get("resolution"),
bands_json={"dtype": metadata_payload.get("dtype")} if metadata_payload.get("dtype") is not None else None,
metadata_json=metadata_payload,
status="ready",
status="validating",
)
db.add(derived_dataset)
db.add(
DatasetVersion(
dataset_id=derived_dataset.id,
version=1,
storage_path=derived_dataset.storage_path,
source_version=derived_dataset.source_version,
observed_at=derived_dataset.observed_at,
valid_from=derived_dataset.valid_from,
valid_to=derived_dataset.valid_to,
checksum_sha256=derived_dataset.checksum_sha256,
source_metadata=derived_dataset.source_metadata,
provenance_metadata=derived_dataset.provenance_metadata,
)
dataset_version = DatasetVersion(
dataset_id=derived_dataset.id,
version=1,
storage_path=derived_dataset.storage_path,
source_version=derived_dataset.source_version,
observed_at=derived_dataset.observed_at,
valid_from=derived_dataset.valid_from,
valid_to=derived_dataset.valid_to,
checksum_sha256=derived_dataset.checksum_sha256,
source_metadata=derived_dataset.source_metadata,
provenance_metadata=derived_dataset.provenance_metadata,
)
db.add(dataset_version)
DerivedDatasetGovernanceService.govern_raster(
db,
dataset=derived_dataset,
dataset_version=dataset_version,
raster_metadata=metadata_payload,
source_key="derived",
operation=operation_name,
parent_dataset=source_dataset,
operation_parameters=metadata_payload.get("operation_parameters"),
)
db.commit()
db.refresh(derived_dataset)
backend/app/services/runtime_model_provenance_service.py
+652
View File
@@ -0,0 +1,652 @@
"""Fail-closed provenance validation for configured local model files.
Configured YOLO and SAM weights are intentionally not trusted merely because a
file exists on the server. Before an adapter is allowed to load local weights,
this service verifies a neighbouring immutable sidecar manifest, validates the
model artifact against ``geointel.model.pytorch@1.0.0`` and binds that sidecar
to the server-owned source registry and source snapshot recorded in Postgres.
``validate_for_runtime`` remains a structural sidecar check for catalog and
preflight inspection. Production inference must call
``validate_for_production_runtime`` with a database session; it rejects an
unregistered, mismatched, stale, unsafe or quarantined source snapshot before
an adapter can load model bytes. This keeps focused unit tests able to inspect
sidecars without inventing database rows while keeping the production boundary
strict.
"""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timezone
from hashlib import sha256
import json
from pathlib import Path
import re
from typing import Any, Mapping
from uuid import UUID
from app.core.errors import AppError
from app.models import SourceRegistry, SourceSnapshot
from app.services.data_contract_validation import (
PYTORCH_MODEL_CONTRACT_KEY,
PYTORCH_MODEL_CONTRACT_VERSION,
LineageEvidence,
TransformationEvidence,
build_model_validation_input,
validate_registered_asset,
)
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
_CONSUMABLE_FRESHNESS = {"current", "not_applicable"}
_SAFE_SOURCE_REGISTRY_INGEST_STATUSES = {"configured", "ingested"}
_SAFE_SOURCE_SNAPSHOT_INGEST_STATUS = "ingested"
@dataclass(frozen=True)
class RuntimeModelProvenance:
"""Validated, immutable evidence attached to one local inference run."""
model_id: str
task_type: str
model_path: str
manifest_path: str
model_sha256: str
manifest_sha256: str
runtime_manifest_sha256: str
data_contract_key: str
data_contract_version: str
validation_report_sha256: str
source_registry_id: str
source_snapshot_id: str
source_snapshot_checksum_sha256: str
source_version: str
def as_dict(self) -> dict[str, str]:
return {
"model_id": self.model_id,
"task_type": self.task_type,
"model_path": self.model_path,
"manifest_path": self.manifest_path,
"model_sha256": self.model_sha256,
"manifest_sha256": self.manifest_sha256,
"runtime_manifest_sha256": self.runtime_manifest_sha256,
"data_contract_key": self.data_contract_key,
"data_contract_version": self.data_contract_version,
"validation_report_sha256": self.validation_report_sha256,
"source_registry_id": self.source_registry_id,
"source_snapshot_id": self.source_snapshot_id,
"source_snapshot_checksum_sha256": self.source_snapshot_checksum_sha256,
"source_version": self.source_version,
}
class RuntimeModelProvenanceService:
"""Validate model sidecars and their production database binding.
A sidecar lives next to its model as ``<model-file>.geointel-model.json``.
Its ``metadata.runtime_manifest_sha256`` is the SHA-256 of canonical JSON
after omitting that one self-referential field. Any other mutation of the
manifest therefore invalidates it. Structural validation is deliberately
separate from :meth:`validate_for_production_runtime`: discovery and
preflight have no database session, whereas every production inference
path must prove an active source registry/snapshot binding.
"""
MANIFEST_SUFFIX = ".geointel-model.json"
MANIFEST_SCHEMA_VERSION = "geointel.runtime-model-manifest/v1"
SOURCE_REGISTRY_KEY = "model"
@classmethod
def manifest_path_for_model(cls, model_path: str | Path) -> Path:
path = Path(model_path).expanduser()
return Path(f"{path}{cls.MANIFEST_SUFFIX}")
@staticmethod
def manifest_self_checksum(payload: Mapping[str, Any]) -> str:
"""Hash sidecar semantics without its self-referential checksum field."""
canonical_payload = json.loads(json.dumps(payload, sort_keys=True, ensure_ascii=True))
metadata = canonical_payload.get("metadata")
if isinstance(metadata, dict):
metadata.pop("runtime_manifest_sha256", None)
encoded = json.dumps(
canonical_payload,
sort_keys=True,
separators=(",", ":"),
ensure_ascii=True,
).encode("utf-8")
return sha256(encoded).hexdigest()
@classmethod
def validate_for_runtime(
cls,
*,
model_path: str | Path,
model_id: str,
task_type: str,
expected_model_version: str | None = None,
allowed_frameworks: tuple[str, ...] = (),
) -> RuntimeModelProvenance:
"""Return structural sidecar evidence without asserting database state.
This method is appropriate for read-only catalog/preflight checks and
focused sidecar unit tests. It is insufficient for production
inference; adapters must use :meth:`validate_for_production_runtime`.
"""
path = Path(model_path).expanduser()
if not path.exists() or not path.is_file():
cls._raise(
"MODEL_PROVENANCE_MODEL_FILE_MISSING",
"Configured model file is missing; runtime provenance cannot be verified.",
model_path=str(path),
)
manifest_path = cls.manifest_path_for_model(path)
if not manifest_path.exists() or not manifest_path.is_file():
cls._raise(
"MODEL_PROVENANCE_MANIFEST_MISSING",
"Configured model requires an immutable .geointel-model.json sidecar before inference.",
model_path=str(path),
manifest_path=str(manifest_path),
)
try:
raw_manifest = manifest_path.read_bytes()
payload = json.loads(raw_manifest.decode("utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
"Configured model sidecar must be a readable UTF-8 JSON object.",
manifest_path=str(manifest_path),
error_type=type(exc).__name__,
)
if not isinstance(payload, dict):
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
"Configured model sidecar must contain a JSON object.",
manifest_path=str(manifest_path),
)
cls._require_exact_text(
payload.get("schema_version"),
cls.MANIFEST_SCHEMA_VERSION,
field="schema_version",
manifest_path=manifest_path,
)
contract = cls._require_mapping(payload, "data_contract", manifest_path)
contract_key = cls._require_text(contract, "key", manifest_path)
contract_version = cls._require_text(contract, "version", manifest_path)
# A valid manifest for a different artifact family must never make a
# local PyTorch/SAM weight executable. The structural validator below
# has a registry lookup too, but pinning the identity here keeps this
# runtime gate fail-closed if more model contracts are introduced.
cls._require_exact_text(
contract_key,
PYTORCH_MODEL_CONTRACT_KEY,
field="data_contract.key",
manifest_path=manifest_path,
)
cls._require_exact_text(
contract_version,
PYTORCH_MODEL_CONTRACT_VERSION,
field="data_contract.version",
manifest_path=manifest_path,
)
model = cls._require_mapping(payload, "model", manifest_path)
declared_model_id = cls._require_text(model, "model_id", manifest_path)
declared_task_type = cls._require_text(model, "task_type", manifest_path)
cls._require_exact_text(declared_model_id, model_id, field="model.model_id", manifest_path=manifest_path)
cls._require_exact_text(declared_task_type, task_type, field="model.task_type", manifest_path=manifest_path)
declared_checksum = cls._require_checksum(model.get("sha256"), "model.sha256", manifest_path)
model_checksum = cls._file_sha256(path)
if declared_checksum != model_checksum:
cls._raise(
"MODEL_PROVENANCE_MODEL_CHECKSUM_MISMATCH",
"Model bytes do not match the checksum bound by the runtime sidecar.",
model_path=str(path),
expected=declared_checksum,
observed=model_checksum,
)
model_format = cls._require_text(model, "model_format", manifest_path)
framework = cls._require_text(model, "framework", manifest_path)
class_mapping = model.get("class_mapping")
if not isinstance(class_mapping, (dict, list, tuple)) or not class_mapping:
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
"model.class_mapping must be a non-empty mapping or sequence.",
manifest_path=str(manifest_path),
)
normalized_framework = framework.strip().lower()
if allowed_frameworks and normalized_framework not in {value.strip().lower() for value in allowed_frameworks}:
cls._raise(
"MODEL_PROVENANCE_FRAMEWORK_MISMATCH",
"Model framework does not match the configured runtime adapter.",
manifest_path=str(manifest_path),
expected=sorted({value.strip().lower() for value in allowed_frameworks}),
observed=framework,
)
source_version = cls._require_text(model, "source_version", manifest_path)
if expected_model_version and source_version != expected_model_version:
cls._raise(
"MODEL_PROVENANCE_VERSION_MISMATCH",
"Model sidecar version does not match the configured model version.",
manifest_path=str(manifest_path),
expected=expected_model_version,
observed=source_version,
)
source = cls._require_mapping(payload, "source", manifest_path)
source_registry_id = cls._require_uuid(source.get("source_registry_id"), "source.source_registry_id", manifest_path)
source_snapshot_id = cls._require_uuid(source.get("source_snapshot_id"), "source.source_snapshot_id", manifest_path)
cls._require_exact_text(
cls._require_text(source, "source_registry_key", manifest_path),
cls.SOURCE_REGISTRY_KEY,
field="source.source_registry_key",
manifest_path=manifest_path,
)
snapshot_checksum = cls._require_checksum(
source.get("source_snapshot_checksum_sha256"),
"source.source_snapshot_checksum_sha256",
manifest_path,
)
if snapshot_checksum != model_checksum:
cls._raise(
"MODEL_PROVENANCE_SNAPSHOT_CHECKSUM_MISMATCH",
"Model source snapshot checksum must bind the exact model bytes.",
manifest_path=str(manifest_path),
expected=model_checksum,
observed=snapshot_checksum,
)
metadata = cls._require_mapping(payload, "metadata", manifest_path)
training_manifest_sha256 = cls._require_checksum(
metadata.get("training_manifest_sha256"),
"metadata.training_manifest_sha256",
manifest_path,
)
declared_runtime_manifest_sha256 = cls._require_checksum(
metadata.get("runtime_manifest_sha256"),
"metadata.runtime_manifest_sha256",
manifest_path,
)
computed_runtime_manifest_sha256 = cls.manifest_self_checksum(payload)
if declared_runtime_manifest_sha256 != computed_runtime_manifest_sha256:
cls._raise(
"MODEL_PROVENANCE_MANIFEST_CHECKSUM_MISMATCH",
"Runtime model sidecar integrity checksum does not match its canonical contents.",
manifest_path=str(manifest_path),
expected=declared_runtime_manifest_sha256,
observed=computed_runtime_manifest_sha256,
)
lineage = cls._lineage_evidence(payload, manifest_path)
imported_at = cls._parse_imported_at(payload.get("imported_at"), manifest_path)
report = validate_registered_asset(
build_model_validation_input(
asset_id=f"{model_id}:{model_checksum}",
model_metadata={
"model_format": model_format,
"framework": framework,
"class_mapping": class_mapping,
},
checksum_sha256=declared_checksum,
computed_checksum_sha256=model_checksum,
source_registry_id=source_registry_id,
source_snapshot_id=source_snapshot_id,
imported_at=imported_at,
metadata={
"training_manifest_sha256": training_manifest_sha256,
"runtime_manifest_sha256": declared_runtime_manifest_sha256,
},
source_version=source_version,
lineage=lineage,
data_contract_key=contract_key,
data_contract_version=contract_version,
)
)
if report.failed:
cls._raise(
"MODEL_PROVENANCE_CONTRACT_FAILED",
"Configured model sidecar failed the exact versioned model data contract.",
manifest_path=str(manifest_path),
data_contract=f"{contract_key}@{contract_version}",
issue_codes=[issue.code for issue in report.issues],
validation_report_sha256=report.report_sha256,
)
return RuntimeModelProvenance(
model_id=model_id,
task_type=task_type,
model_path=str(path.resolve()),
manifest_path=str(manifest_path.resolve()),
model_sha256=model_checksum,
manifest_sha256=sha256(raw_manifest).hexdigest(),
runtime_manifest_sha256=declared_runtime_manifest_sha256,
data_contract_key=contract_key,
data_contract_version=contract_version,
validation_report_sha256=report.report_sha256,
source_registry_id=source_registry_id,
source_snapshot_id=source_snapshot_id,
source_snapshot_checksum_sha256=snapshot_checksum,
source_version=source_version,
)
@classmethod
def validate_for_production_runtime(
cls,
*,
db: Any,
model_path: str | Path,
model_id: str,
task_type: str,
expected_model_version: str | None = None,
allowed_frameworks: tuple[str, ...] = (),
) -> RuntimeModelProvenance:
"""Validate byte-bound model evidence against governed database state.
The sidecar is not itself a source of authority. The model bytes may
enter production inference only when the declared source registry and
immutable source snapshot both exist, belong together, are safe to
consume and bind the same SHA-256 and source version as the sidecar.
This check is intentionally invoked immediately before adapter loading.
"""
if db is None or not callable(getattr(db, "get", None)):
cls._raise(
"MODEL_PROVENANCE_DATABASE_REQUIRED",
"Production model inference requires a database session for source snapshot provenance.",
)
evidence = cls.validate_for_runtime(
model_path=model_path,
model_id=model_id,
task_type=task_type,
expected_model_version=expected_model_version,
allowed_frameworks=allowed_frameworks,
)
cls._assert_database_binding(db, evidence)
return evidence
@classmethod
def _assert_database_binding(cls, db: Any, evidence: RuntimeModelProvenance) -> None:
registry_id = UUID(evidence.source_registry_id)
snapshot_id = UUID(evidence.source_snapshot_id)
source_registry = db.get(SourceRegistry, registry_id)
if not isinstance(source_registry, SourceRegistry):
cls._raise(
"MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND",
"Configured model sidecar refers to a source registry record that does not exist.",
source_registry_id=evidence.source_registry_id,
model_id=evidence.model_id,
)
source_snapshot = db.get(SourceSnapshot, snapshot_id)
if not isinstance(source_snapshot, SourceSnapshot):
cls._raise(
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_NOT_FOUND",
"Configured model sidecar refers to a source snapshot record that does not exist.",
source_snapshot_id=evidence.source_snapshot_id,
model_id=evidence.model_id,
)
if str(source_registry.id) != evidence.source_registry_id or source_registry.source_key != cls.SOURCE_REGISTRY_KEY:
cls._raise(
"MODEL_PROVENANCE_SOURCE_REGISTRY_IDENTITY_MISMATCH",
"Database source registry does not match the immutable model sidecar identity.",
expected_source_registry_id=evidence.source_registry_id,
observed_source_registry_id=str(source_registry.id),
expected_source_key=cls.SOURCE_REGISTRY_KEY,
observed_source_key=source_registry.source_key,
)
if str(source_snapshot.id) != evidence.source_snapshot_id or str(source_snapshot.source_registry_id) != evidence.source_registry_id:
cls._raise(
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_REGISTRY_MISMATCH",
"Model source snapshot does not belong to the declared source registry.",
source_registry_id=evidence.source_registry_id,
source_snapshot_id=evidence.source_snapshot_id,
observed_snapshot_registry_id=str(source_snapshot.source_registry_id),
)
registry_ingest_status = cls._normalise_status(source_registry.ingest_status)
registry_freshness_status = cls._normalise_status(source_registry.freshness_status)
if (
registry_ingest_status not in _SAFE_SOURCE_REGISTRY_INGEST_STATUSES
or registry_freshness_status not in _CONSUMABLE_FRESHNESS
):
cls._raise(
"MODEL_PROVENANCE_SOURCE_REGISTRY_UNSAFE",
"Configured model source registry is not in a safe configured/current state.",
source_registry_id=evidence.source_registry_id,
ingest_status=registry_ingest_status,
freshness_status=registry_freshness_status,
)
snapshot_ingest_status = cls._normalise_status(source_snapshot.ingest_status)
snapshot_freshness_status = cls._normalise_status(source_snapshot.freshness_status)
if (
snapshot_ingest_status != _SAFE_SOURCE_SNAPSHOT_INGEST_STATUS
or snapshot_freshness_status not in _CONSUMABLE_FRESHNESS
):
cls._raise(
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_UNSAFE",
"Configured model source snapshot is not an ingested current immutable artifact.",
source_snapshot_id=evidence.source_snapshot_id,
ingest_status=snapshot_ingest_status,
freshness_status=snapshot_freshness_status,
)
snapshot_checksum = str(source_snapshot.checksum_sha256 or "").strip().lower()
if snapshot_checksum != evidence.source_snapshot_checksum_sha256 or snapshot_checksum != evidence.model_sha256:
cls._raise(
"MODEL_PROVENANCE_DATABASE_SNAPSHOT_CHECKSUM_MISMATCH",
"Database model source snapshot checksum does not bind the exact sidecar and model bytes.",
source_snapshot_id=evidence.source_snapshot_id,
expected_model_sha256=evidence.model_sha256,
expected_sidecar_snapshot_sha256=evidence.source_snapshot_checksum_sha256,
observed_snapshot_sha256=snapshot_checksum,
)
snapshot_version = str(source_snapshot.source_version or "").strip()
if snapshot_version != evidence.source_version:
cls._raise(
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_VERSION_MISMATCH",
"Database model source snapshot version does not match the immutable sidecar model version.",
source_snapshot_id=evidence.source_snapshot_id,
expected_source_version=evidence.source_version,
observed_source_version=snapshot_version or None,
)
# The persisted snapshot state is the primary quarantine signal. The
# relationship check is a defensive second line for an active
# quarantine record that predates or bypassed a status transition.
active_quarantines = getattr(source_snapshot, "quarantines", ())
if any(cls._normalise_status(getattr(item, "status", None)) == "quarantined" for item in active_quarantines):
cls._raise(
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_QUARANTINED",
"Configured model source snapshot has an active quarantine record.",
source_snapshot_id=evidence.source_snapshot_id,
)
@staticmethod
def _normalise_status(value: Any) -> str:
return str(value or "").strip().lower()
@classmethod
def _lineage_evidence(cls, payload: Mapping[str, Any], manifest_path: Path) -> LineageEvidence:
lineage = cls._require_mapping(payload, "lineage", manifest_path)
raw_asset_ids = lineage.get("upstream_asset_ids")
raw_checksums = lineage.get("upstream_checksums_sha256")
raw_transformations = lineage.get("transformations")
if not isinstance(raw_asset_ids, list) or not raw_asset_ids or not all(
isinstance(value, str) and value.strip() for value in raw_asset_ids
):
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
"lineage.upstream_asset_ids must be a non-empty string list.",
manifest_path=str(manifest_path),
)
if not isinstance(raw_checksums, list) or len(raw_checksums) != len(raw_asset_ids):
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
"lineage.upstream_checksums_sha256 must match upstream_asset_ids one-for-one.",
manifest_path=str(manifest_path),
)
upstream_checksums = tuple(
cls._require_checksum(value, f"lineage.upstream_checksums_sha256[{index}]", manifest_path)
for index, value in enumerate(raw_checksums)
)
if not isinstance(raw_transformations, list) or not raw_transformations:
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
"lineage.transformations must contain at least one immutable transformation record.",
manifest_path=str(manifest_path),
)
transformations: list[TransformationEvidence] = []
for index, raw in enumerate(raw_transformations):
if not isinstance(raw, Mapping):
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
"Each lineage transformation must be an object.",
manifest_path=str(manifest_path),
index=index,
)
transformations.append(
TransformationEvidence(
name=cls._require_text(raw, "name", manifest_path, prefix=f"lineage.transformations[{index}]."),
version=cls._require_text(raw, "version", manifest_path, prefix=f"lineage.transformations[{index}]."),
checksum_sha256=cls._require_checksum(
raw.get("checksum_sha256"),
f"lineage.transformations[{index}].checksum_sha256",
manifest_path,
),
)
)
return LineageEvidence(
upstream_asset_ids=tuple(raw_asset_ids),
upstream_checksums_sha256=upstream_checksums,
transformations=tuple(transformations),
)
@staticmethod
def _file_sha256(path: Path) -> str:
digest = sha256()
try:
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
except OSError as exc:
RuntimeModelProvenanceService._raise(
"MODEL_PROVENANCE_MODEL_FILE_UNREADABLE",
"Configured model file could not be read for checksum validation.",
model_path=str(path),
error_type=type(exc).__name__,
)
return digest.hexdigest()
@classmethod
def _parse_imported_at(cls, value: Any, manifest_path: Path) -> datetime:
if not isinstance(value, str) or not value.strip():
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
"imported_at must be a timezone-aware ISO-8601 timestamp.",
manifest_path=str(manifest_path),
)
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
"imported_at must be a timezone-aware ISO-8601 timestamp.",
manifest_path=str(manifest_path),
observed=value,
)
if parsed.tzinfo is None:
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
"imported_at must include a timezone offset.",
manifest_path=str(manifest_path),
observed=value,
)
return parsed.astimezone(timezone.utc)
@classmethod
def _require_mapping(cls, payload: Mapping[str, Any], key: str, manifest_path: Path) -> Mapping[str, Any]:
value = payload.get(key)
if not isinstance(value, Mapping):
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
f"{key} must be a JSON object.",
manifest_path=str(manifest_path),
)
return value
@classmethod
def _require_text(
cls,
payload: Mapping[str, Any],
key: str,
manifest_path: Path,
*,
prefix: str = "",
) -> str:
value = payload.get(key)
if not isinstance(value, str) or not value.strip():
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
f"{prefix}{key} must be a non-empty string.",
manifest_path=str(manifest_path),
)
return value.strip()
@classmethod
def _require_checksum(cls, value: Any, field: str, manifest_path: Path) -> str:
normalized = value.strip().lower() if isinstance(value, str) else ""
if not _SHA256.fullmatch(normalized) or value != normalized:
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
f"{field} must be a lowercase SHA-256 digest.",
manifest_path=str(manifest_path),
)
return normalized
@classmethod
def _require_uuid(cls, value: Any, field: str, manifest_path: Path) -> str:
if not isinstance(value, str) or not value.strip():
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
f"{field} must be a UUID string.",
manifest_path=str(manifest_path),
)
try:
return str(UUID(value))
except (AttributeError, ValueError):
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
f"{field} must be a UUID string.",
manifest_path=str(manifest_path),
observed=value,
)
@classmethod
def _require_exact_text(
cls,
observed: Any,
expected: str,
*,
field: str,
manifest_path: Path,
) -> None:
if not isinstance(observed, str) or observed.strip() != expected:
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
f"{field} does not match the configured runtime identity.",
manifest_path=str(manifest_path),
expected=expected,
observed=observed,
)
@staticmethod
def _raise(code: str, message: str, **details: Any) -> None:
raise AppError(code=code, message=message, details=details, status_code=422)
backend/app/services/segmentation_service.py
+70 -2
View File
@@ -21,9 +21,11 @@ from app.schemas.segmentation import (
)
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
from app.services.detection_service import DetectionService
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
from app.services.model_registry_service import ModelRegistryService
from app.services.qa_service import QaService
from app.services.quality_service import QualityService
from app.services.runtime_model_provenance_service import RuntimeModelProvenance, RuntimeModelProvenanceService
from app.services.segmentation_adapter import (
FixtureSegmentationAdapter,
SamSegmentationAdapter,
@@ -89,6 +91,18 @@ class SegmentationService:
status_code=400,
)
# Production segmentation must consume only a passed, complete and
# non-quarantined dataset. The fixture segmenter is QA/test-only and
# cannot be classified as production inference.
if model.model_id == "fixture-segmenter":
DatasetConsumptionGate.assert_eligible(
dataset,
purpose="quality_assessment",
fixture_mode=True,
)
elif model.model_id in configured_model_ids and model.configured:
DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
run_parameters = {
"model_id": model.model_id,
"confidence_threshold": confidence_threshold,
@@ -326,6 +340,27 @@ class SegmentationService:
if reference_dataset.dataset_type not in {"vector", "geojson"}:
raise AppError(code="INVALID_DATASET_TYPE", message="Reference dataset must be vector data", status_code=400)
candidate_dataset = db.get(Dataset, run.dataset_id)
if not candidate_dataset:
raise AppError(code="DATASET_NOT_FOUND", message="Segmentation source dataset not found", status_code=404)
run_parameters = run.parameters_json if isinstance(run.parameters_json, dict) else {}
fixture_parameters = run_parameters.get("parameters_json")
fixture_mode = bool(
run.model_name == "fixture-segmenter"
and isinstance(fixture_parameters, dict)
and fixture_parameters.get("fixture_mode") is True
)
DatasetConsumptionGate.assert_eligible(
candidate_dataset,
purpose="quality_assessment",
fixture_mode=fixture_mode,
)
DatasetConsumptionGate.assert_eligible(
reference_dataset,
purpose="reference_validation",
reference_task="building_validation",
)
segmentations = SegmentationService._query_segmentation_rows(
db,
analysis_run_id=analysis_run_id,
@@ -510,11 +545,26 @@ class SegmentationService:
) -> tuple[list[Segmentation], dict[str, Any]]:
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles)
if model_name == settings.sam_model_id:
adapter = sam_adapter_class(settings)
model_path = Path(settings.sam_model_path or "").expanduser()
allowed_frameworks = ("ultralytics/sam", "sam", "ultralytics", "pytorch")
adapter = sam_adapter_class(settings)
else:
adapter = yolo_seg_adapter_class(settings)
model_path = Path(settings.yolo_seg_model_path or "").expanduser()
allowed_frameworks = ("ultralytics/pytorch", "ultralytics", "pytorch")
adapter = yolo_seg_adapter_class(settings)
runtime_model_provenance = RuntimeModelProvenanceService.validate_for_production_runtime(
db=db,
model_path=model_path,
model_id=model_name,
task_type="segmentation",
expected_model_version=model_version,
allowed_frameworks=allowed_frameworks,
)
SegmentationService._attach_runtime_model_provenance(
analysis_run,
job,
runtime_model_provenance,
)
model = adapter.load_model(model_path)
allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)}
@@ -591,6 +641,7 @@ class SegmentationService:
"tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()),
"tile_index": candidate["tile_index"],
"device": settings.yolo_device,
"runtime_model_provenance": runtime_model_provenance.as_dict(),
},
)
db.add(segmentation)
@@ -603,8 +654,25 @@ class SegmentationService:
"suppressed_segmentation_count": len(candidates) - len(filtered_candidates),
"duplicate_iou_threshold": float(settings.segmentation_duplicate_iou_threshold),
"tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()),
"runtime_model_provenance": runtime_model_provenance.as_dict(),
}
@staticmethod
def _attach_runtime_model_provenance(
analysis_run: AnalysisRun,
job: Job,
provenance: RuntimeModelProvenance,
) -> None:
"""Record immutable model evidence with a configured segmentation run."""
evidence = provenance.as_dict()
analysis_parameters = dict(analysis_run.parameters_json or {})
analysis_parameters["runtime_model_provenance"] = evidence
analysis_run.parameters_json = analysis_parameters
job_parameters = dict(job.parameters_json or {})
job_parameters["runtime_model_provenance"] = evidence
job.parameters_json = job_parameters
@staticmethod
def _geodesic_area_m2(geometry: MultiPolygon | Polygon) -> float | None:
try:
backend/app/services/source_registry_service.py
+1509
View File
File diff suppressed because it is too large Load Diff
backend/app/services/vector_feature_service.py
+130 -5
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import json
import math
from pathlib import Path
from typing import Any, Iterable
from uuid import UUID
@@ -8,6 +9,7 @@ from uuid import UUID
from geoalchemy2.functions import ST_Intersects, ST_MakeEnvelope
from geoalchemy2.shape import from_shape
from geoalchemy2.shape import to_shape
from pyproj import CRS, Transformer
from shapely.geometry import box, mapping, shape
from shapely.ops import transform as transform_geometry
from shapely.validation import make_valid
@@ -268,7 +270,14 @@ class VectorFeatureService:
return ("municipality", municipality) if municipality else None
@staticmethod
def _feature_row(dataset_id: UUID, feature: dict[str, Any], index: int, feature_class: str | None) -> VectorFeature | None:
def _feature_row(
dataset_id: UUID,
feature: dict[str, Any],
index: int,
feature_class: str | None,
*,
source_crs: str = "EPSG:4326",
) -> VectorFeature | None:
geometry_payload = feature.get("geometry")
if geometry_payload is None:
return None
@@ -282,8 +291,7 @@ class VectorFeatureService:
geometry = make_valid(geometry)
if geometry.is_empty or not geometry.is_valid:
raise AppError(code="INVALID_GEOMETRY", message=f"Invalid feature geometry at index {index}", status_code=400)
if geometry.has_z:
geometry = transform_geometry(lambda x, y, z=None: (x, y), geometry)
geometry = VectorFeatureService._canonical_geometry(geometry, source_crs=source_crs, index=index)
properties = feature.get("properties") if isinstance(feature.get("properties"), dict) else {}
source_feature_id = feature.get("id")
@@ -298,6 +306,109 @@ class VectorFeatureService:
geometry=from_shape(geometry, srid=4326),
)
@staticmethod
def _canonical_geometry(geometry: Any, *, source_crs: str, index: int):
"""Transform one source geometry to canonical EPSG:4326 safely."""
if geometry.has_z:
geometry = transform_geometry(lambda x, y, z=None: (x, y), geometry)
# VectorFeature is deliberately canonical WGS84 storage. Treating
# Lambert or another source CRS as EPSG:4326 produces geometries that
# look syntactically valid but are spatially wrong. All governed
# import callers therefore pass the declared source CRS; the default
# only preserves compatibility for legacy, already-WGS84 call sites.
try:
parsed_source_crs = CRS.from_user_input(source_crs)
target_crs = CRS.from_epsg(4326)
except Exception as exc:
raise AppError(
code="INVALID_DATASET_CRS",
message=f"Invalid source CRS for vector feature at index {index}",
details={"source_crs": source_crs},
status_code=400,
) from exc
if not parsed_source_crs.equals(target_crs):
try:
transformer = Transformer.from_crs(parsed_source_crs, target_crs, always_xy=True)
geometry = transform_geometry(transformer.transform, geometry)
except Exception as exc:
raise AppError(
code="VECTOR_CRS_TRANSFORMATION_FAILED",
message=f"Could not transform vector feature at index {index} to EPSG:4326",
details={"source_crs": source_crs},
status_code=400,
) from exc
if geometry.is_empty or not geometry.is_valid:
geometry = make_valid(geometry)
if geometry.is_empty or not geometry.is_valid:
raise AppError(
code="INVALID_GEOMETRY",
message=f"Invalid transformed feature geometry at index {index}",
status_code=400,
)
min_x, min_y, max_x, max_y = geometry.bounds
if (
not all(math.isfinite(value) for value in (min_x, min_y, max_x, max_y))
or min_x < -180
or max_x > 180
or min_y < -90
or max_y > 90
):
raise AppError(
code="VECTOR_GEOMETRY_OUTSIDE_EPSG4326",
message=f"Transformed feature geometry at index {index} is outside EPSG:4326 bounds",
details={"source_crs": source_crs, "bounds": [min_x, min_y, max_x, max_y]},
status_code=400,
)
return geometry
@staticmethod
def canonicalize_geojson_payload(payload: dict[str, Any], *, source_crs: str) -> dict[str, Any]:
"""Return a canonical-WGS84 feature collection without losing source attributes.
Callers use this payload for validation, spatial indexing and
map-safe consumption storage. A non-canonical source file, when
retained, belongs to explicit provenance evidence rather than the
Dataset consumption path; no implicit CRS assumption is recorded.
"""
features = payload.get("features")
if payload.get("type") != "FeatureCollection" or not isinstance(features, list):
raise AppError(code="INVALID_GEOJSON", message="GeoJSON payload must be a FeatureCollection", status_code=400)
canonical_features: list[dict[str, Any]] = []
for index, feature in enumerate(features):
if not isinstance(feature, dict):
raise AppError(code="INVALID_GEOJSON", message=f"Feature {index} must be an object", status_code=400)
canonical_feature = dict(feature)
geometry_payload = feature.get("geometry")
if geometry_payload is not None:
try:
geometry = shape(geometry_payload)
except Exception as exc:
raise AppError(
code="INVALID_GEOJSON",
message=f"Invalid feature geometry at index {index}",
status_code=400,
) from exc
if not geometry.is_empty:
if not geometry.is_valid:
geometry = make_valid(geometry)
if geometry.is_empty or not geometry.is_valid:
raise AppError(
code="INVALID_GEOMETRY",
message=f"Invalid feature geometry at index {index}",
status_code=400,
)
canonical_feature["geometry"] = mapping(
VectorFeatureService._canonical_geometry(geometry, source_crs=source_crs, index=index)
)
canonical_features.append(canonical_feature)
return {
**{key: value for key, value in payload.items() if key not in {"crs", "features"}},
"type": "FeatureCollection",
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
"features": canonical_features,
}
@staticmethod
def _normalize_selection_bbox(bbox: dict[str, Any]) -> dict[str, float | str]:
try:
@@ -882,6 +993,7 @@ class VectorFeatureService:
feature_class: str | None = None,
*,
commit: bool = True,
source_crs: str = "EPSG:4326",
) -> list[VectorFeature]:
features = payload.get("features")
if payload.get("type") != "FeatureCollection" or not isinstance(features, list):
@@ -891,7 +1003,13 @@ class VectorFeatureService:
for index, feature in enumerate(features):
if not isinstance(feature, dict):
raise AppError(code="INVALID_GEOJSON", message=f"Feature {index} must be an object", status_code=400)
row = VectorFeatureService._feature_row(dataset_id, feature, index, feature_class)
row = VectorFeatureService._feature_row(
dataset_id,
feature,
index,
feature_class,
source_crs=source_crs,
)
if row is None:
continue
db.add(row)
@@ -910,6 +1028,7 @@ class VectorFeatureService:
feature_class: str | None = None,
*,
batch_size: int = 1000,
source_crs: str = "EPSG:4326",
) -> int:
if batch_size <= 0:
raise ValueError("batch_size must be positive")
@@ -942,7 +1061,13 @@ class VectorFeatureService:
message=f"Feature {index} in {path.name} must be an object",
status_code=400,
)
row = VectorFeatureService._feature_row(dataset_id, feature, index, feature_class)
row = VectorFeatureService._feature_row(
dataset_id,
feature,
index,
feature_class,
source_crs=source_crs,
)
if row is None:
continue
if row.source_feature_id:
backend/app/services/vector_operations_service.py
+128 -25
View File
@@ -3,13 +3,17 @@ from __future__ import annotations
import json
import uuid
from datetime import datetime, timezone
from hashlib import sha256
from pathlib import Path
import re
from typing import Any
from geoalchemy2.shape import to_shape
from pyproj import CRS, Transformer
from shapely.geometry import GeometryCollection, MultiPolygon, shape
from shapely.geometry.base import BaseGeometry
from shapely.geometry import mapping
from shapely.ops import transform as shapely_transform
from shapely.ops import unary_union
from shapely.validation import make_valid
from sqlalchemy.orm import Session
@@ -18,12 +22,16 @@ from app.core.errors import AppError
from app.models import Area, Dataset, DatasetVersion
from app.schemas.dataset import DatasetCreateResponse
from app.schemas.operations import VectorOperationResult
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
from app.services.geojson_service import parse_geojson_payload
from app.services.storage_service import StorageService
from app.services.vector_feature_service import VectorFeatureService
class VectorOperationsService:
CANONICAL_VECTOR_CRS = "EPSG:4326"
_CHECKSUM_SHA256 = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE)
@staticmethod
def _require_vector_dataset(dataset: Dataset) -> None:
if dataset.dataset_type not in {"vector", "geojson"}:
@@ -37,7 +45,8 @@ class VectorOperationsService:
if not path.exists():
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
stored_bytes = path.read_bytes()
payload = json.loads(stored_bytes.decode("utf-8"))
except Exception as exc:
raise AppError(code="INVALID_GEOJSON", message="Stored dataset is not valid JSON", status_code=400) from exc
@@ -47,6 +56,55 @@ class VectorOperationsService:
features = payload.get("features")
if not isinstance(features, list):
raise AppError(code="INVALID_GEOJSON", message="Dataset payload is missing features", status_code=400)
# The normal Dataset storage path is a consumption artifact, not a
# provenance source archive. Refuse projected/original source bytes
# here rather than letting a spatial operation interpret them as
# canonical map coordinates.
raw_crs = payload.get("crs")
if isinstance(raw_crs, dict):
crs_properties = raw_crs.get("properties")
raw_crs = crs_properties.get("name") if isinstance(crs_properties, dict) else None
stored_crs = str(raw_crs or VectorOperationsService.CANONICAL_VECTOR_CRS).strip().upper()
dataset_crs = str(dataset.crs or "").strip().upper()
if stored_crs != VectorOperationsService.CANONICAL_VECTOR_CRS or (
dataset_crs and dataset_crs != VectorOperationsService.CANONICAL_VECTOR_CRS
):
raise AppError(
code="DATASET_STORAGE_CRS_MISMATCH",
message="Vector operations require canonical EPSG:4326 dataset storage.",
details={
"stored_crs": raw_crs or VectorOperationsService.CANONICAL_VECTOR_CRS,
"dataset_crs": dataset.crs,
"expected_crs": VectorOperationsService.CANONICAL_VECTOR_CRS,
},
status_code=409,
)
expected_checksum = str(dataset.checksum_sha256 or "").strip().lower()
actual_checksum = sha256(stored_bytes).hexdigest()
governed_artifact = bool(
getattr(dataset, "data_contract_key", None)
or (
isinstance(getattr(dataset, "metadata_json", None), dict)
and dataset.metadata_json.get("canonical_storage_crs")
)
)
if expected_checksum and VectorOperationsService._CHECKSUM_SHA256.fullmatch(expected_checksum):
if expected_checksum != actual_checksum:
raise AppError(
code="DATASET_STORAGE_CHECKSUM_MISMATCH",
message="Vector dataset storage no longer matches its validated checksum.",
details={"expected_checksum_sha256": expected_checksum, "actual_checksum_sha256": actual_checksum},
status_code=409,
)
elif governed_artifact:
raise AppError(
code="DATASET_STORAGE_CHECKSUM_UNVERIFIABLE",
message="Governed vector storage requires a valid SHA-256 checksum before use.",
details={"checksum_sha256": dataset.checksum_sha256},
status_code=409,
)
return payload, [feature for feature in features if isinstance(feature, dict)]
@staticmethod
@@ -73,6 +131,25 @@ class VectorOperationsService:
raise AppError(code="EMPTY_VECTOR_DATASET", message="Dataset has no parseable geometries", status_code=422)
return geometries
@staticmethod
def _buffer_in_metres(geometry: BaseGeometry, distance_m: float, source_crs: str) -> BaseGeometry:
"""Buffer in a Belgian projected CRS, never in angular degrees."""
try:
input_crs = CRS.from_user_input(source_crs)
metric_crs = CRS.from_epsg(31370)
if input_crs == metric_crs:
return geometry.buffer(distance_m)
forward = Transformer.from_crs(input_crs, metric_crs, always_xy=True)
backward = Transformer.from_crs(metric_crs, input_crs, always_xy=True)
return shapely_transform(backward.transform, shapely_transform(forward.transform, geometry).buffer(distance_m))
except Exception as exc:
raise AppError(
code="INVALID_CRS",
message="A valid explicit CRS is required for metre-based vector buffering.",
status_code=400,
) from exc
@staticmethod
def inspect(db: Session, dataset_id: uuid.UUID) -> VectorOperationResult:
dataset = db.get(Dataset, dataset_id)
@@ -94,7 +171,7 @@ class VectorOperationsService:
feature_count=len(geometries),
geometry_type_summary=geometry_type_summary,
bounds_json={"min_x": float(bounds[0]), "min_y": float(bounds[1]), "max_x": float(bounds[2]), "max_y": float(bounds[3])},
crs=payload.get("crs") if isinstance(payload.get("crs"), str) else dataset.crs,
crs=VectorOperationsService.CANONICAL_VECTOR_CRS,
)
@staticmethod
@@ -184,9 +261,13 @@ class VectorOperationsService:
if distance_m <= 0:
raise AppError(code="INVALID_PARAMETERS", message="distance_m must be greater than 0", status_code=400)
_, features = VectorOperationsService._load_dataset_payload(source_dataset)
payload, features = VectorOperationsService._load_dataset_payload(source_dataset)
source_crs = VectorOperationsService.CANONICAL_VECTOR_CRS
geometries = VectorOperationsService._extract_geometries(features)
buffered_features = [(feature, geometry.buffer(distance_m)) for feature, geometry in geometries]
buffered_features = [
(feature, VectorOperationsService._buffer_in_metres(geometry, distance_m, source_crs))
for feature, geometry in geometries
]
output_features: list[dict[str, Any]] = []
for feature, geometry in buffered_features:
@@ -431,7 +512,14 @@ class VectorOperationsService:
if not output_name_value.strip():
output_name_value = f"{default_name}.geojson"
stored = json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
# All current governed vector storage is EPSG:4326. A legacy source
# with another CRS is not relabelled here: the derived contract will
# quarantine the result instead of placing non-WGS84 coordinates on
# the map as if they were WGS84.
output_crs = VectorOperationsService.CANONICAL_VECTOR_CRS
output_feature_collection = dict(feature_collection)
output_feature_collection["crs"] = output_crs
stored = json.dumps(output_feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
storage_info = StorageService.persist_dataset_file(
project_id=str(source_dataset.project_id),
dataset_id=str(derived_id),
@@ -441,7 +529,7 @@ class VectorOperationsService:
content_type="application/geo+json",
)
metadata = parse_geojson_payload(json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")))
metadata = parse_geojson_payload(json.dumps(output_feature_collection, ensure_ascii=False, separators=(",", ":")))
if metadata_extra:
metadata.update(metadata_extra)
derived_dataset = Dataset(
@@ -452,7 +540,7 @@ class VectorOperationsService:
dataset_type="vector",
source=f"operation:{operation}",
dataset_role=dataset_role,
source_name=source_name,
source_name=source_name or "derived",
source_metadata=source_metadata,
provenance_metadata=provenance_metadata,
imported_at=datetime.now(timezone.utc),
@@ -478,29 +566,44 @@ class VectorOperationsService:
resolution_json=metadata.get("resolution_json"),
bands_json=metadata.get("bands_json"),
metadata_json=metadata,
status="ready",
status="validating",
)
db.add(derived_dataset)
db.add(
DatasetVersion(
dataset_id=derived_dataset.id,
version=1,
storage_path=derived_dataset.storage_path,
source_version=derived_dataset.source_version,
observed_at=derived_dataset.observed_at,
valid_from=derived_dataset.valid_from,
valid_to=derived_dataset.valid_to,
checksum_sha256=derived_dataset.checksum_sha256,
source_metadata=derived_dataset.source_metadata,
provenance_metadata=derived_dataset.provenance_metadata,
)
dataset_version = DatasetVersion(
dataset_id=derived_dataset.id,
version=1,
storage_path=derived_dataset.storage_path,
source_version=derived_dataset.source_version,
observed_at=derived_dataset.observed_at,
valid_from=derived_dataset.valid_from,
valid_to=derived_dataset.valid_to,
checksum_sha256=derived_dataset.checksum_sha256,
source_metadata=derived_dataset.source_metadata,
provenance_metadata=derived_dataset.provenance_metadata,
)
db.commit()
db.refresh(derived_dataset)
if persist_vector_features:
db.add(dataset_version)
derived_source_key = "map_selection" if source_name == "map_selection" else "derived"
is_ready = DerivedDatasetGovernanceService.govern_vector(
db,
dataset=derived_dataset,
dataset_version=dataset_version,
feature_collection=output_feature_collection,
source_key=derived_source_key,
operation=f"vector.{operation}",
parent_dataset=source_dataset,
operation_parameters={
"operation": operation,
"output_name": output_name_value,
**(metadata_extra or {}),
},
)
if persist_vector_features and is_ready:
VectorFeatureService.persist_geojson_features(
db=db,
dataset_id=derived_dataset.id,
payload=feature_collection,
payload=output_feature_collection,
commit=False,
)
db.commit()
db.refresh(derived_dataset)
return derived_id
backend/app/services/yolo_preflight_service.py
+26
View File
@@ -9,6 +9,7 @@ from app.core.config import Settings, get_settings
from app.core.errors import AppError
from app.services.detection_service import DetectionService
from app.services.model_asset_catalog_service import ModelAssetCatalogService
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
from app.services.yolo_adapter import YoloDetectionAdapter
@@ -41,6 +42,8 @@ class YoloPreflightService:
"accelerator_ready": None,
"model_path_set": None,
"model_file_exists": None,
"model_provenance_manifest_path": None,
"model_provenance_valid": None,
"model_load_requested": check_model_load,
"model_load_ok": None,
"manifest_path_set": None,
@@ -98,6 +101,29 @@ class YoloPreflightService:
result["message"] = "YOLO_MODEL_PATH does not point to an existing local model file."
return result
result["checks"]["model_provenance_manifest_path"] = str(
RuntimeModelProvenanceService.manifest_path_for_model(model_path)
)
try:
RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id=resolved_settings.yolo_model_id,
task_type="object_detection",
expected_model_version=resolved_settings.yolo_model_version,
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
)
except AppError as exc:
result["checks"]["model_provenance_valid"] = False
result["status"] = "contract_incomplete"
result["message"] = (
"Configured YOLO weights are not runnable until their immutable runtime provenance sidecar validates: "
f"{exc.message}"
)
result["error_code"] = exc.code
result["details"] = exc.details
return result
result["checks"]["model_provenance_valid"] = True
if check_model_load:
try:
yolo_adapter_class(resolved_settings).load_model(model_path)