feat(provenance): govern source snapshots and data inputs
This commit is contained in:
File diff suppressed because it is too large
Load Diff
@@ -1 +1,15 @@
|
||||
__all__ = ["analysis", "areas", "assistant", "auth", "datasets", "health", "projects", "exports", "jobs", "external", "qa", "temporal"]
|
||||
__all__ = [
|
||||
"analysis",
|
||||
"areas",
|
||||
"assistant",
|
||||
"auth",
|
||||
"datasets",
|
||||
"exports",
|
||||
"external",
|
||||
"health",
|
||||
"jobs",
|
||||
"projects",
|
||||
"qa",
|
||||
"source_registry",
|
||||
"temporal",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from uuid import UUID
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy import func, or_
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.db.session import get_db
|
||||
from app.models import Dataset, DatasetLineageEdge, DatasetQuarantine, Project, SourceRegistry, SourceSnapshot
|
||||
from app.schemas import (
|
||||
DatasetLineageEdgeRead,
|
||||
DatasetProvenanceRead,
|
||||
DatasetQuarantineRead,
|
||||
Envelope,
|
||||
ItemList,
|
||||
SourceRegistryDetailRead,
|
||||
SourceRegistryRead,
|
||||
SourceSnapshotRead,
|
||||
)
|
||||
from app.utils.response import envelope
|
||||
|
||||
|
||||
router = APIRouter(tags=["source-registry"])
|
||||
|
||||
|
||||
def _source_read(source: SourceRegistry, *, snapshot_count: int = 0) -> SourceRegistryRead:
|
||||
return SourceRegistryRead.model_validate(source).model_copy(update={"snapshot_count": int(snapshot_count)})
|
||||
|
||||
|
||||
@router.get("/source-registry", response_model=Envelope[ItemList[SourceRegistryRead]])
|
||||
def list_source_registry(
|
||||
classification: str | None = None,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
query = (
|
||||
db.query(SourceRegistry, func.count(SourceSnapshot.id).label("snapshot_count"))
|
||||
.outerjoin(SourceSnapshot, SourceSnapshot.source_registry_id == SourceRegistry.id)
|
||||
)
|
||||
if classification:
|
||||
query = query.filter(SourceRegistry.classification == classification.strip().lower())
|
||||
rows = (
|
||||
query.group_by(SourceRegistry.id)
|
||||
.order_by(SourceRegistry.classification.asc(), SourceRegistry.display_name.asc())
|
||||
.all()
|
||||
)
|
||||
items = [_source_read(source, snapshot_count=count) for source, count in rows]
|
||||
return envelope({"items": items, "total": len(items)})
|
||||
|
||||
|
||||
@router.get("/source-registry/{source_key}", response_model=Envelope[SourceRegistryDetailRead])
|
||||
def get_source_registry_entry(source_key: str, db: Session = Depends(get_db)) -> dict:
|
||||
normalized_key = source_key.strip().lower()
|
||||
source = db.query(SourceRegistry).filter(SourceRegistry.source_key == normalized_key).one_or_none()
|
||||
if source is None:
|
||||
raise AppError(code="SOURCE_REGISTRY_ENTRY_NOT_FOUND", message="Source registry entry was not found", status_code=404)
|
||||
snapshots = (
|
||||
db.query(SourceSnapshot)
|
||||
.filter(SourceSnapshot.source_registry_id == source.id)
|
||||
.order_by(SourceSnapshot.fetched_at.desc(), SourceSnapshot.created_at.desc())
|
||||
.all()
|
||||
)
|
||||
detail = SourceRegistryDetailRead(
|
||||
source=_source_read(source, snapshot_count=len(snapshots)),
|
||||
snapshots=[SourceSnapshotRead.model_validate(snapshot) for snapshot in snapshots],
|
||||
)
|
||||
return envelope(detail)
|
||||
|
||||
|
||||
@router.get(
|
||||
"/projects/{project_id}/datasets/{dataset_id}/provenance",
|
||||
response_model=Envelope[DatasetProvenanceRead],
|
||||
)
|
||||
def get_dataset_provenance(
|
||||
project_id: UUID,
|
||||
dataset_id: UUID,
|
||||
db: Session = Depends(get_db),
|
||||
) -> dict:
|
||||
if db.get(Project, project_id) is None:
|
||||
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||
dataset = db.get(Dataset, dataset_id)
|
||||
if dataset is None or dataset.project_id != project_id:
|
||||
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
||||
|
||||
source = db.get(SourceRegistry, dataset.source_registry_id) if dataset.source_registry_id else None
|
||||
snapshot = db.get(SourceSnapshot, dataset.source_snapshot_id) if dataset.source_snapshot_id else None
|
||||
lineage = (
|
||||
db.query(DatasetLineageEdge)
|
||||
.filter(
|
||||
or_(
|
||||
DatasetLineageEdge.parent_dataset_id == dataset.id,
|
||||
DatasetLineageEdge.child_dataset_id == dataset.id,
|
||||
)
|
||||
)
|
||||
.order_by(DatasetLineageEdge.created_at.asc(), DatasetLineageEdge.id.asc())
|
||||
.all()
|
||||
)
|
||||
quarantines = (
|
||||
db.query(DatasetQuarantine)
|
||||
.filter(DatasetQuarantine.dataset_id == dataset.id)
|
||||
.order_by(DatasetQuarantine.created_at.desc(), DatasetQuarantine.id.desc())
|
||||
.all()
|
||||
)
|
||||
result = DatasetProvenanceRead(
|
||||
dataset_id=dataset.id,
|
||||
source=_source_read(source) if source else None,
|
||||
snapshot=SourceSnapshotRead.model_validate(snapshot) if snapshot else None,
|
||||
data_contract_key=dataset.data_contract_key,
|
||||
data_contract_version=dataset.data_contract_version,
|
||||
validation_status=dataset.validation_status,
|
||||
validation_report_json=dataset.validation_report_json,
|
||||
provenance_status=dataset.provenance_status,
|
||||
lineage_status=dataset.lineage_status,
|
||||
quarantine_status=dataset.quarantine_status,
|
||||
lineage=[DatasetLineageEdgeRead.model_validate(item) for item in lineage],
|
||||
quarantines=[DatasetQuarantineRead.model_validate(item) for item in quarantines],
|
||||
)
|
||||
return envelope(result)
|
||||
+2
-1
@@ -12,7 +12,7 @@ from fastapi.exceptions import RequestValidationError
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from app.api.routes import analysis, aoi_operations, areas, assistant, auth, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation, selection_partitions, temporal
|
||||
from app.api.routes import analysis, aoi_operations, areas, assistant, auth, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation, selection_partitions, source_registry, temporal
|
||||
from app.core.config import get_settings
|
||||
from app.core.errors import AppError
|
||||
from app.core.logging import configure_logging
|
||||
@@ -103,6 +103,7 @@ def create_app() -> FastAPI:
|
||||
app.include_router(quality_checks.router, prefix=settings.api_prefix)
|
||||
app.include_router(exports.router, prefix=settings.api_prefix)
|
||||
app.include_router(external.router, prefix=settings.api_prefix)
|
||||
app.include_router(source_registry.router, prefix=settings.api_prefix)
|
||||
app.include_router(demo.router, prefix=settings.api_prefix)
|
||||
app.include_router(qa.router, prefix=settings.api_prefix)
|
||||
app.include_router(detection.router, prefix=settings.api_prefix)
|
||||
|
||||
@@ -1,4 +1,24 @@
|
||||
from .entities import AoiOperation, AoiOperationPartition, AnalysisRun, Area, Dataset, DatasetVersion, Detection, DetectionReview, Export, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature
|
||||
from .entities import (
|
||||
AoiOperation,
|
||||
AoiOperationPartition,
|
||||
AnalysisRun,
|
||||
Area,
|
||||
Dataset,
|
||||
DatasetLineageEdge,
|
||||
DatasetQuarantine,
|
||||
DatasetVersion,
|
||||
Detection,
|
||||
DetectionReview,
|
||||
Export,
|
||||
Job,
|
||||
Metric,
|
||||
Project,
|
||||
QualityCheck,
|
||||
Segmentation,
|
||||
SourceRegistry,
|
||||
SourceSnapshot,
|
||||
VectorFeature,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AnalysisRun",
|
||||
@@ -6,6 +26,8 @@ __all__ = [
|
||||
"AoiOperationPartition",
|
||||
"Area",
|
||||
"Dataset",
|
||||
"DatasetLineageEdge",
|
||||
"DatasetQuarantine",
|
||||
"DatasetVersion",
|
||||
"Detection",
|
||||
"DetectionReview",
|
||||
@@ -15,5 +37,7 @@ __all__ = [
|
||||
"Project",
|
||||
"QualityCheck",
|
||||
"Segmentation",
|
||||
"SourceRegistry",
|
||||
"SourceSnapshot",
|
||||
"VectorFeature",
|
||||
]
|
||||
|
||||
@@ -12,6 +12,37 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship
|
||||
from app.db.base import Base
|
||||
|
||||
|
||||
SOURCE_CLASSIFICATIONS = (
|
||||
"authoritative",
|
||||
"corroborative",
|
||||
"contextual",
|
||||
"derived",
|
||||
"experimental",
|
||||
)
|
||||
SOURCE_FRESHNESS_STATUSES = (
|
||||
"unknown",
|
||||
"current",
|
||||
"due",
|
||||
"stale",
|
||||
"not_applicable",
|
||||
"review_required",
|
||||
)
|
||||
SOURCE_INGEST_STATUSES = (
|
||||
"registered",
|
||||
"configured",
|
||||
"not_configured",
|
||||
"available",
|
||||
"ingested",
|
||||
"failed",
|
||||
"quarantined",
|
||||
"legacy_unverified",
|
||||
)
|
||||
PROVENANCE_STATUSES = ("complete", "incomplete", "not_applicable")
|
||||
LINEAGE_STATUSES = ("complete", "incomplete", "not_applicable")
|
||||
VALIDATION_STATUSES = ("not_validated", "passed", "failed")
|
||||
QUARANTINE_STATUSES = ("not_quarantined", "quarantined")
|
||||
|
||||
|
||||
class Project(Base):
|
||||
__tablename__ = "projects"
|
||||
|
||||
@@ -42,6 +73,123 @@ class Area(Base):
|
||||
project: Mapped[Project] = relationship("Project", back_populates="areas")
|
||||
|
||||
|
||||
class SourceRegistry(Base):
|
||||
"""Server-owned source identity and authority contract.
|
||||
|
||||
Dataset metadata remains descriptive until a governed importer binds a
|
||||
dataset to both this registry entry and an immutable SourceSnapshot.
|
||||
"""
|
||||
|
||||
__tablename__ = "source_registry"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("source_key", name="uq_source_registry_source_key"),
|
||||
CheckConstraint(
|
||||
"classification IN ('authoritative', 'corroborative', 'contextual', 'derived', 'experimental')",
|
||||
name="ck_source_registry_classification",
|
||||
),
|
||||
CheckConstraint(
|
||||
"freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')",
|
||||
name="ck_source_registry_freshness_status",
|
||||
),
|
||||
CheckConstraint(
|
||||
"ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', "
|
||||
"'failed', 'quarantined', 'legacy_unverified')",
|
||||
name="ck_source_registry_ingest_status",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
source_key: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||
display_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
classification: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
authority_name: Mapped[str] = mapped_column(String(255), nullable=False, default="unknown", server_default="unknown")
|
||||
authority_scope_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
provider_adapter_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
license_name: Mapped[str] = mapped_column(String(255), nullable=False, default="unknown", server_default="unknown")
|
||||
license_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
usage_restrictions: Mapped[str] = mapped_column(Text, nullable=False, default="unknown", server_default="unknown")
|
||||
default_crs: Mapped[str] = mapped_column(String(64), nullable=False, default="unknown", server_default="unknown")
|
||||
default_units: Mapped[str] = mapped_column(String(120), nullable=False, default="unknown", server_default="unknown")
|
||||
spatial_resolution_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
temporal_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
geographic_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
expected_geometry_types_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
|
||||
expected_attributes_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
usage_policy_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
freshness_status: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default="unknown", server_default="unknown"
|
||||
)
|
||||
ingest_status: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default="registered", server_default="registered"
|
||||
)
|
||||
known_limitations_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
|
||||
registry_metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||
|
||||
snapshots: Mapped[list["SourceSnapshot"]] = relationship(
|
||||
"SourceSnapshot", back_populates="source_registry", cascade="all, delete-orphan"
|
||||
)
|
||||
datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="source_registry")
|
||||
dataset_versions: Mapped[list["DatasetVersion"]] = relationship("DatasetVersion", back_populates="source_registry")
|
||||
|
||||
|
||||
class SourceSnapshot(Base):
|
||||
"""Immutable source-version evidence recorded by governed ingestion."""
|
||||
|
||||
__tablename__ = "source_snapshots"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("source_registry_id", "snapshot_key", name="uq_source_snapshots_registry_key"),
|
||||
CheckConstraint(
|
||||
"freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')",
|
||||
name="ck_source_snapshots_freshness_status",
|
||||
),
|
||||
CheckConstraint(
|
||||
"ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', "
|
||||
"'failed', 'quarantined', 'legacy_unverified')",
|
||||
name="ck_source_snapshots_ingest_status",
|
||||
),
|
||||
CheckConstraint(
|
||||
"checksum_sha256 = lower(checksum_sha256) AND checksum_sha256 ~ '^[0-9a-f]{64}$'",
|
||||
name="ck_source_snapshots_checksum_sha256",
|
||||
),
|
||||
Index("ix_source_snapshots_registry_fetched", "source_registry_id", "fetched_at"),
|
||||
Index("ix_source_snapshots_checksum", "checksum_sha256"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
source_registry_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
snapshot_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
source_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
snapshot_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
|
||||
source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
crs: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
units: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
spatial_resolution_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
temporal_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
geographic_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
observed_schema_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
freshness_status: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default="unknown", server_default="unknown"
|
||||
)
|
||||
ingest_status: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default="registered", server_default="registered"
|
||||
)
|
||||
known_limitations_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
|
||||
snapshot_metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
source_registry: Mapped[SourceRegistry] = relationship("SourceRegistry", back_populates="snapshots")
|
||||
datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="source_snapshot")
|
||||
dataset_versions: Mapped[list["DatasetVersion"]] = relationship("DatasetVersion", back_populates="source_snapshot")
|
||||
quarantines: Mapped[list["DatasetQuarantine"]] = relationship("DatasetQuarantine", back_populates="source_snapshot")
|
||||
|
||||
|
||||
class Dataset(Base):
|
||||
__tablename__ = "datasets"
|
||||
__table_args__ = (
|
||||
@@ -49,6 +197,27 @@ class Dataset(Base):
|
||||
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
|
||||
name="ck_datasets_temporal_valid_range",
|
||||
),
|
||||
CheckConstraint(
|
||||
"validation_status IN ('not_validated', 'passed', 'failed')",
|
||||
name="ck_datasets_validation_status",
|
||||
),
|
||||
CheckConstraint(
|
||||
"provenance_status IN ('complete', 'incomplete', 'not_applicable')",
|
||||
name="ck_datasets_provenance_status",
|
||||
),
|
||||
CheckConstraint(
|
||||
"lineage_status IN ('complete', 'incomplete', 'not_applicable')",
|
||||
name="ck_datasets_lineage_status",
|
||||
),
|
||||
CheckConstraint(
|
||||
"quarantine_status IN ('not_quarantined', 'quarantined')",
|
||||
name="ck_datasets_quarantine_status",
|
||||
),
|
||||
CheckConstraint(
|
||||
"ingest_key IS NULL OR btrim(ingest_key) <> ''",
|
||||
name="ck_datasets_ingest_key_not_blank",
|
||||
),
|
||||
UniqueConstraint("project_id", "ingest_key", name="uq_datasets_project_ingest_key"),
|
||||
Index(
|
||||
"ix_datasets_project_temporal_series_observed",
|
||||
"project_id",
|
||||
@@ -69,6 +238,7 @@ class Dataset(Base):
|
||||
content_type: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
ingest_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
derived_from_dataset_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True),
|
||||
ForeignKey("datasets.id", ondelete="SET NULL"),
|
||||
@@ -84,6 +254,43 @@ class Dataset(Base):
|
||||
reference_layer_name: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
source_registry_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
data_contract_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
data_contract_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
validation_status: Mapped[str] = mapped_column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
default="not_validated",
|
||||
server_default="not_validated",
|
||||
comment="not_validated | passed | failed",
|
||||
)
|
||||
validation_report_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
provenance_status: Mapped[str] = mapped_column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
default="incomplete",
|
||||
server_default="incomplete",
|
||||
comment="complete | incomplete | not_applicable",
|
||||
)
|
||||
lineage_status: Mapped[str] = mapped_column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
default="incomplete",
|
||||
server_default="incomplete",
|
||||
comment="complete | incomplete | not_applicable",
|
||||
)
|
||||
quarantine_status: Mapped[str] = mapped_column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
default="not_quarantined",
|
||||
server_default="not_quarantined",
|
||||
comment="not_quarantined | quarantined",
|
||||
)
|
||||
imported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
temporal_series_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
@@ -106,6 +313,23 @@ class Dataset(Base):
|
||||
back_populates="dataset",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
source_registry: Mapped[SourceRegistry | None] = relationship("SourceRegistry", back_populates="datasets")
|
||||
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="datasets")
|
||||
parent_lineage_edges: Mapped[list["DatasetLineageEdge"]] = relationship(
|
||||
"DatasetLineageEdge",
|
||||
foreign_keys="DatasetLineageEdge.parent_dataset_id",
|
||||
back_populates="parent_dataset",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
child_lineage_edges: Mapped[list["DatasetLineageEdge"]] = relationship(
|
||||
"DatasetLineageEdge",
|
||||
foreign_keys="DatasetLineageEdge.child_dataset_id",
|
||||
back_populates="child_dataset",
|
||||
cascade="all, delete-orphan",
|
||||
)
|
||||
quarantines: Mapped[list["DatasetQuarantine"]] = relationship(
|
||||
"DatasetQuarantine", back_populates="dataset", cascade="all, delete-orphan"
|
||||
)
|
||||
|
||||
|
||||
class DatasetVersion(Base):
|
||||
@@ -115,7 +339,24 @@ class DatasetVersion(Base):
|
||||
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
|
||||
name="ck_dataset_versions_temporal_valid_range",
|
||||
),
|
||||
CheckConstraint(
|
||||
"validation_status IN ('not_validated', 'passed', 'failed')",
|
||||
name="ck_dataset_versions_validation_status",
|
||||
),
|
||||
CheckConstraint(
|
||||
"provenance_status IN ('complete', 'incomplete', 'not_applicable')",
|
||||
name="ck_dataset_versions_provenance_status",
|
||||
),
|
||||
CheckConstraint(
|
||||
"lineage_status IN ('complete', 'incomplete', 'not_applicable')",
|
||||
name="ck_dataset_versions_lineage_status",
|
||||
),
|
||||
CheckConstraint(
|
||||
"ingest_key IS NULL OR btrim(ingest_key) <> ''",
|
||||
name="ck_dataset_versions_ingest_key_not_blank",
|
||||
),
|
||||
Index("ix_dataset_versions_dataset_version", "dataset_id", "version", unique=True),
|
||||
UniqueConstraint("dataset_id", "ingest_key", name="uq_dataset_versions_dataset_ingest_key"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
@@ -127,11 +368,135 @@ class DatasetVersion(Base):
|
||||
valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
ingest_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
source_registry_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
data_contract_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
data_contract_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
validation_status: Mapped[str] = mapped_column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
default="not_validated",
|
||||
server_default="not_validated",
|
||||
comment="not_validated | passed | failed",
|
||||
)
|
||||
validation_report_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
provenance_status: Mapped[str] = mapped_column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
default="incomplete",
|
||||
server_default="incomplete",
|
||||
comment="complete | incomplete | not_applicable",
|
||||
)
|
||||
lineage_status: Mapped[str] = mapped_column(
|
||||
String(32),
|
||||
nullable=False,
|
||||
default="incomplete",
|
||||
server_default="incomplete",
|
||||
comment="complete | incomplete | not_applicable",
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
dataset: Mapped[Dataset] = relationship("Dataset", back_populates="versions")
|
||||
source_registry: Mapped[SourceRegistry | None] = relationship("SourceRegistry", back_populates="dataset_versions")
|
||||
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="dataset_versions")
|
||||
quarantines: Mapped[list["DatasetQuarantine"]] = relationship("DatasetQuarantine", back_populates="dataset_version")
|
||||
|
||||
|
||||
class DatasetLineageEdge(Base):
|
||||
"""Immutable relationship between input/output datasets and transforms."""
|
||||
|
||||
__tablename__ = "dataset_lineage_edges"
|
||||
__table_args__ = (
|
||||
CheckConstraint("parent_dataset_id <> child_dataset_id", name="ck_dataset_lineage_edges_distinct_datasets"),
|
||||
UniqueConstraint(
|
||||
"parent_dataset_id",
|
||||
"child_dataset_id",
|
||||
"relation_type",
|
||||
"transformation_name",
|
||||
name="uq_dataset_lineage_edges_relation",
|
||||
),
|
||||
Index("ix_dataset_lineage_edges_parent", "parent_dataset_id"),
|
||||
Index("ix_dataset_lineage_edges_child", "child_dataset_id"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
parent_dataset_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
child_dataset_id: Mapped[uuid.UUID] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False
|
||||
)
|
||||
parent_dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
child_dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
relation_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
transformation_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||
transformation_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
parameters_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
input_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
output_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
|
||||
parent_dataset: Mapped[Dataset] = relationship(
|
||||
"Dataset", foreign_keys=[parent_dataset_id], back_populates="parent_lineage_edges"
|
||||
)
|
||||
child_dataset: Mapped[Dataset] = relationship(
|
||||
"Dataset", foreign_keys=[child_dataset_id], back_populates="child_lineage_edges"
|
||||
)
|
||||
|
||||
|
||||
class DatasetQuarantine(Base):
|
||||
"""Durable fail-closed record for rejected or doubtful source artifacts."""
|
||||
|
||||
__tablename__ = "dataset_quarantines"
|
||||
__table_args__ = (
|
||||
CheckConstraint(
|
||||
"dataset_id IS NOT NULL OR dataset_version_id IS NOT NULL OR source_snapshot_id IS NOT NULL",
|
||||
name="ck_dataset_quarantines_target_present",
|
||||
),
|
||||
CheckConstraint(
|
||||
"status IN ('quarantined', 'released', 'rejected')",
|
||||
name="ck_dataset_quarantines_status",
|
||||
),
|
||||
Index("ix_dataset_quarantines_dataset_status", "dataset_id", "status"),
|
||||
Index("ix_dataset_quarantines_snapshot_status", "source_snapshot_id", "status"),
|
||||
)
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||
dataset_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
|
||||
)
|
||||
stage: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
reason_code: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||
details_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||
artifact_path: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
artifact_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||
status: Mapped[str] = mapped_column(
|
||||
String(32), nullable=False, default="quarantined", server_default="quarantined"
|
||||
)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
resolved_by: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||
|
||||
dataset: Mapped[Dataset | None] = relationship("Dataset", back_populates="quarantines")
|
||||
dataset_version: Mapped[DatasetVersion | None] = relationship("DatasetVersion", back_populates="quarantines")
|
||||
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="quarantines")
|
||||
|
||||
|
||||
class VectorFeature(Base):
|
||||
|
||||
@@ -32,6 +32,14 @@ from .source_catalog import (
|
||||
SourceCatalogProbeReport,
|
||||
SourceCatalogProbeSummary,
|
||||
)
|
||||
from .source_registry import (
|
||||
DatasetProvenanceRead,
|
||||
DatasetLineageEdgeRead,
|
||||
DatasetQuarantineRead,
|
||||
SourceRegistryDetailRead,
|
||||
SourceRegistryRead,
|
||||
SourceSnapshotRead,
|
||||
)
|
||||
from .grb_refresh import GrbRefreshLayerPlan, GrbRefreshPlan, GrbRefreshPlanSummary
|
||||
from .grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead
|
||||
from .official_vector import (
|
||||
@@ -207,6 +215,12 @@ __all__ = [
|
||||
"SourceCatalogProbeItem",
|
||||
"SourceCatalogProbeReport",
|
||||
"SourceCatalogProbeSummary",
|
||||
"SourceRegistryRead",
|
||||
"SourceRegistryDetailRead",
|
||||
"SourceSnapshotRead",
|
||||
"DatasetLineageEdgeRead",
|
||||
"DatasetQuarantineRead",
|
||||
"DatasetProvenanceRead",
|
||||
"GrbRefreshLayerPlan",
|
||||
"GrbRefreshPlan",
|
||||
"GrbRefreshPlanSummary",
|
||||
@@ -271,6 +285,10 @@ __all__ = [
|
||||
"FloodHazardSelectionSummary",
|
||||
"BathymetryProfileAcquireRequest",
|
||||
"BathymetryProfileAcquisitionResult",
|
||||
"BathymetryRasterMetric",
|
||||
"BathymetryRasterSelectionRequest",
|
||||
"BathymetryRasterSelectionResponse",
|
||||
"BathymetryRasterSelectionSummary",
|
||||
"BathymetryPartitionFinalizeRequest",
|
||||
"BathymetryPartitionFinalizationResult",
|
||||
"BathymetrySourceProbeRead",
|
||||
|
||||
@@ -35,6 +35,16 @@ class DatasetCreateResponse(BaseModel):
|
||||
reference_layer_name: str | None = None
|
||||
source_metadata: dict | None = None
|
||||
provenance_metadata: dict | None = None
|
||||
ingest_key: str | None = None
|
||||
source_registry_id: UUID | None = None
|
||||
source_snapshot_id: UUID | None = None
|
||||
data_contract_key: str | None = None
|
||||
data_contract_version: str | None = None
|
||||
validation_status: str | None = None
|
||||
validation_report_json: dict | None = None
|
||||
provenance_status: str | None = None
|
||||
lineage_status: str | None = None
|
||||
quarantine_status: str | None = None
|
||||
imported_at: datetime | None = None
|
||||
temporal_series_key: str | None = None
|
||||
observed_at: datetime | None = None
|
||||
@@ -97,6 +107,15 @@ class DatasetVersionRead(BaseModel):
|
||||
checksum_sha256: str | None = None
|
||||
source_metadata: dict | None = None
|
||||
provenance_metadata: dict | None = None
|
||||
ingest_key: str | None = None
|
||||
source_registry_id: UUID | None = None
|
||||
source_snapshot_id: UUID | None = None
|
||||
data_contract_key: str | None = None
|
||||
data_contract_version: str | None = None
|
||||
validation_status: str | None = None
|
||||
validation_report_json: dict | None = None
|
||||
provenance_status: str | None = None
|
||||
lineage_status: str | None = None
|
||||
created_at: datetime | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
@@ -144,6 +144,8 @@ class YoloPreflightChecks(BaseModel):
|
||||
accelerator_ready: bool | None = None
|
||||
model_path_set: bool | None = None
|
||||
model_file_exists: bool | None = None
|
||||
model_provenance_manifest_path: str | None = None
|
||||
model_provenance_valid: bool | None = None
|
||||
model_load_requested: bool
|
||||
model_load_ok: bool | None = None
|
||||
manifest_path_set: bool | None = None
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime
|
||||
from uuid import UUID
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class SourceRegistryRead(BaseModel):
|
||||
"""Read-only, server-owned source-authority definition."""
|
||||
|
||||
id: UUID
|
||||
source_key: str
|
||||
display_name: str
|
||||
classification: str
|
||||
authority_name: str
|
||||
authority_scope_json: dict
|
||||
provider_adapter_key: str | None = None
|
||||
source_url: str | None = None
|
||||
license_name: str
|
||||
license_url: str | None = None
|
||||
usage_restrictions: str
|
||||
default_crs: str
|
||||
default_units: str
|
||||
spatial_resolution_json: dict
|
||||
temporal_coverage_json: dict
|
||||
geographic_coverage_json: dict
|
||||
expected_geometry_types_json: list
|
||||
expected_attributes_json: dict
|
||||
usage_policy_json: dict
|
||||
freshness_status: str
|
||||
ingest_status: str
|
||||
known_limitations_json: list
|
||||
registry_metadata_json: dict
|
||||
created_at: datetime | None = None
|
||||
updated_at: datetime | None = None
|
||||
snapshot_count: int = 0
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class SourceSnapshotRead(BaseModel):
|
||||
"""Immutable version/snapshot evidence attached to an imported dataset."""
|
||||
|
||||
id: UUID
|
||||
source_registry_id: UUID
|
||||
snapshot_key: str
|
||||
source_version: str | None = None
|
||||
snapshot_at: datetime | None = None
|
||||
fetched_at: datetime | None = None
|
||||
source_url: str | None = None
|
||||
checksum_sha256: str | None = None
|
||||
crs: str | None = None
|
||||
units: str | None = None
|
||||
spatial_resolution_json: dict
|
||||
temporal_coverage_json: dict
|
||||
geographic_coverage_json: dict
|
||||
observed_schema_json: dict
|
||||
freshness_status: str
|
||||
ingest_status: str
|
||||
known_limitations_json: list
|
||||
snapshot_metadata_json: dict
|
||||
created_at: datetime | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class SourceRegistryDetailRead(BaseModel):
|
||||
source: SourceRegistryRead
|
||||
snapshots: list[SourceSnapshotRead]
|
||||
|
||||
|
||||
class DatasetLineageEdgeRead(BaseModel):
|
||||
id: UUID
|
||||
parent_dataset_id: UUID
|
||||
child_dataset_id: UUID
|
||||
parent_dataset_version_id: UUID | None = None
|
||||
child_dataset_version_id: UUID | None = None
|
||||
relation_type: str
|
||||
transformation_name: str
|
||||
transformation_version: str | None = None
|
||||
parameters_json: dict | None = None
|
||||
input_checksum_sha256: str | None = None
|
||||
output_checksum_sha256: str | None = None
|
||||
created_at: datetime | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class DatasetQuarantineRead(BaseModel):
|
||||
id: UUID
|
||||
dataset_id: UUID | None = None
|
||||
dataset_version_id: UUID | None = None
|
||||
source_snapshot_id: UUID | None = None
|
||||
stage: str
|
||||
reason_code: str
|
||||
details_json: dict | None = None
|
||||
artifact_path: str | None = None
|
||||
artifact_checksum_sha256: str | None = None
|
||||
status: str
|
||||
created_at: datetime | None = None
|
||||
resolved_at: datetime | None = None
|
||||
resolved_by: str | None = None
|
||||
|
||||
model_config = {"from_attributes": True}
|
||||
|
||||
|
||||
class DatasetProvenanceRead(BaseModel):
|
||||
dataset_id: UUID
|
||||
source: SourceRegistryRead | None = None
|
||||
snapshot: SourceSnapshotRead | None = None
|
||||
data_contract_key: str | None = None
|
||||
data_contract_version: str | None = None
|
||||
validation_status: str | None = None
|
||||
validation_report_json: dict | None = None
|
||||
provenance_status: str | None = None
|
||||
lineage_status: str | None = None
|
||||
quarantine_status: str | None = None
|
||||
lineage: list[DatasetLineageEdgeRead] = Field(default_factory=list)
|
||||
quarantines: list[DatasetQuarantineRead] = Field(default_factory=list)
|
||||
@@ -12,6 +12,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset, Project
|
||||
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||
from app.schemas.coverage import (
|
||||
CoverageBBox,
|
||||
CoverageCatalogResponse,
|
||||
@@ -463,6 +464,10 @@ class CoverageRegistryService:
|
||||
for dataset in datasets:
|
||||
if dataset.status != "ready" or dataset.source_name not in definition.materialized_source_names:
|
||||
continue
|
||||
# A source-name claim alone must not cause an unsafe artifact to
|
||||
# appear as operational authoritative coverage.
|
||||
if not DatasetConsumptionGate.eligible_for_authoritative_coverage(dataset):
|
||||
continue
|
||||
layer_names = definition.materialized_layer_names
|
||||
if definition.contract.source_name == "digitaal_vlaanderen":
|
||||
theme_sources = FLANDERS_THEME_DATASETS.get(theme, {})
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,160 @@
|
||||
"""Fail-closed quarantine decisions for validated data assets.
|
||||
|
||||
Persistence is intentionally delegated to the caller's transaction. This
|
||||
module derives stable decisions from immutable validation reports and blocks a
|
||||
quarantined or failed asset from training, production inference and derived
|
||||
processing until an explicit, separately persisted release action exists.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
from hashlib import sha256
|
||||
from typing import Any
|
||||
import json
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.services.data_contract_validation import QuarantineStatus, ValidationReport, ValidationStatus
|
||||
|
||||
|
||||
class AssetUse(StrEnum):
|
||||
TRAINING = "training"
|
||||
PRODUCTION_INFERENCE = "production_inference"
|
||||
DERIVED_PROCESSING = "derived_processing"
|
||||
EXPORT = "export"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class QuarantineDecision:
|
||||
"""A deterministic, auditable quarantine decision for one report."""
|
||||
|
||||
asset_id: str
|
||||
data_contract_key: str
|
||||
data_contract_version: str
|
||||
validation_report_sha256: str
|
||||
validation_status: ValidationStatus
|
||||
quarantine_status: QuarantineStatus
|
||||
reason_codes: tuple[str, ...]
|
||||
idempotency_key: str
|
||||
requires_explicit_release: bool = False
|
||||
|
||||
@property
|
||||
def eligible_for_use(self) -> bool:
|
||||
return self.validation_status == ValidationStatus.PASSED and self.quarantine_status == QuarantineStatus.NOT_QUARANTINED
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return {
|
||||
"asset_id": self.asset_id,
|
||||
"data_contract_key": self.data_contract_key,
|
||||
"data_contract_version": self.data_contract_version,
|
||||
"validation_report_sha256": self.validation_report_sha256,
|
||||
"validation_status": self.validation_status.value,
|
||||
"quarantine_status": self.quarantine_status.value,
|
||||
"reason_codes": list(self.reason_codes),
|
||||
"idempotency_key": self.idempotency_key,
|
||||
"requires_explicit_release": self.requires_explicit_release,
|
||||
"eligible_for_use": self.eligible_for_use,
|
||||
}
|
||||
|
||||
|
||||
class DataQuarantineService:
|
||||
"""Derive and enforce safe use decisions from validation results."""
|
||||
|
||||
@staticmethod
|
||||
def decide(
|
||||
report: ValidationReport,
|
||||
*,
|
||||
previous: QuarantineDecision | None = None,
|
||||
) -> QuarantineDecision:
|
||||
"""Create a stable decision without silently releasing old quarantines.
|
||||
|
||||
A fresh passing validation report can be persisted as a new validated
|
||||
version by the import transaction. It cannot automatically release an
|
||||
existing quarantined record: the caller must explicitly record that
|
||||
reviewed state transition against the new report/version.
|
||||
"""
|
||||
|
||||
is_quarantined = report.validation_status == ValidationStatus.FAILED or report.quarantine_status == QuarantineStatus.QUARANTINED
|
||||
failure_codes = tuple(
|
||||
sorted(
|
||||
{
|
||||
issue.code
|
||||
for issue in report.issues
|
||||
if issue.severity.value == "error" or is_quarantined
|
||||
}
|
||||
)
|
||||
)
|
||||
requires_explicit_release = False
|
||||
reason_codes = failure_codes
|
||||
status = QuarantineStatus.QUARANTINED if is_quarantined else QuarantineStatus.NOT_QUARANTINED
|
||||
|
||||
if previous is not None and previous.quarantine_status == QuarantineStatus.QUARANTINED and not is_quarantined:
|
||||
status = QuarantineStatus.QUARANTINED
|
||||
requires_explicit_release = True
|
||||
reason_codes = ("QUARANTINE_RELEASE_REQUIRES_EXPLICIT_PERSISTENCE",)
|
||||
|
||||
idempotency_key = _decision_key(
|
||||
asset_id=report.asset_id,
|
||||
contract_key=report.data_contract_key,
|
||||
contract_version=report.data_contract_version,
|
||||
report_sha256=report.report_sha256,
|
||||
quarantine_status=status,
|
||||
reason_codes=reason_codes,
|
||||
requires_explicit_release=requires_explicit_release,
|
||||
)
|
||||
return QuarantineDecision(
|
||||
asset_id=report.asset_id,
|
||||
data_contract_key=report.data_contract_key,
|
||||
data_contract_version=report.data_contract_version,
|
||||
validation_report_sha256=report.report_sha256,
|
||||
validation_status=report.validation_status,
|
||||
quarantine_status=status,
|
||||
reason_codes=reason_codes,
|
||||
idempotency_key=idempotency_key,
|
||||
requires_explicit_release=requires_explicit_release,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def require_eligible(decision: QuarantineDecision, *, use: AssetUse) -> None:
|
||||
"""Raise a typed error before a non-eligible artifact reaches a pipeline."""
|
||||
|
||||
if decision.eligible_for_use:
|
||||
return
|
||||
raise AppError(
|
||||
code="DATASET_QUARANTINED",
|
||||
message="Dataset is quarantined or failed validation and cannot enter this pipeline.",
|
||||
status_code=409,
|
||||
details={
|
||||
"asset_id": decision.asset_id,
|
||||
"use": use.value,
|
||||
"quarantine_status": decision.quarantine_status.value,
|
||||
"validation_status": decision.validation_status.value,
|
||||
"reason_codes": list(decision.reason_codes),
|
||||
"validation_report_sha256": decision.validation_report_sha256,
|
||||
"idempotency_key": decision.idempotency_key,
|
||||
"requires_explicit_release": decision.requires_explicit_release,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _decision_key(
|
||||
*,
|
||||
asset_id: str,
|
||||
contract_key: str,
|
||||
contract_version: str,
|
||||
report_sha256: str,
|
||||
quarantine_status: QuarantineStatus,
|
||||
reason_codes: tuple[str, ...],
|
||||
requires_explicit_release: bool,
|
||||
) -> str:
|
||||
payload = {
|
||||
"asset_id": asset_id,
|
||||
"contract_key": contract_key,
|
||||
"contract_version": contract_version,
|
||||
"report_sha256": report_sha256,
|
||||
"quarantine_status": quarantine_status.value,
|
||||
"reason_codes": list(reason_codes),
|
||||
"requires_explicit_release": requires_explicit_release,
|
||||
}
|
||||
return sha256(json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")).hexdigest()
|
||||
@@ -0,0 +1,440 @@
|
||||
"""Fail-closed provenance gates at data-consumption boundaries.
|
||||
|
||||
Import validation protects newly staged assets, but a persisted record can
|
||||
subsequently become quarantined or have its provenance marked incomplete. The
|
||||
callers of this service therefore re-check the durable Dataset state directly
|
||||
before production inference, QA, derived processing, export, or authoritative
|
||||
coverage reporting.
|
||||
|
||||
The only legacy relaxation is deliberately narrow: an *explicitly tagged*
|
||||
fixture with no Phase-2 state can be used for fixture QA. A caller-provided
|
||||
``fixture_mode`` flag alone never creates that trust claim. Fixture data can
|
||||
never become a production inference, derived-processing, export or
|
||||
authoritative-coverage input, and it never relaxes a recorded failed,
|
||||
incomplete, or quarantined state.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
import re
|
||||
from typing import Any, Literal
|
||||
|
||||
from sqlalchemy import inspect as sa_inspect
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Dataset
|
||||
|
||||
|
||||
DatasetConsumptionPurpose = Literal[
|
||||
"production_inference",
|
||||
"quality_assessment",
|
||||
"reference_validation",
|
||||
"derived_processing",
|
||||
"authoritative_coverage",
|
||||
"export",
|
||||
]
|
||||
|
||||
_FIXTURE_SOURCE_KEYS = {"fixture", "test", "test_fixture", "test-fixture", "unit-test-fixture"}
|
||||
_UNTRUSTED_SOURCE_KEYS = {"manual", "fixture", "experimental", "legacy_unknown"}
|
||||
_CHECKSUM_SHA256 = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE)
|
||||
_CONSUMABLE_SNAPSHOT_FRESHNESS = {"current", "not_applicable"}
|
||||
_VALID_PURPOSES = {
|
||||
"production_inference",
|
||||
"quality_assessment",
|
||||
"reference_validation",
|
||||
"derived_processing",
|
||||
"authoritative_coverage",
|
||||
"export",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DatasetConsumptionDecision:
|
||||
"""Auditable decision returned by the consumption gate."""
|
||||
|
||||
eligible: bool
|
||||
purpose: DatasetConsumptionPurpose
|
||||
fixture_legacy_exception: bool
|
||||
reasons: tuple[str, ...]
|
||||
evidence: dict[str, Any]
|
||||
|
||||
|
||||
class DatasetConsumptionGate:
|
||||
"""Evaluate durable provenance before a Dataset is consumed downstream."""
|
||||
|
||||
@staticmethod
|
||||
def _value(dataset: Any, field: str, default: Any = None) -> Any:
|
||||
if isinstance(dataset, Mapping):
|
||||
return dataset.get(field, default)
|
||||
return getattr(dataset, field, default)
|
||||
|
||||
@staticmethod
|
||||
def _mapping(value: Any) -> dict[str, Any]:
|
||||
return dict(value) if isinstance(value, Mapping) else {}
|
||||
|
||||
@classmethod
|
||||
def _normalise(cls, value: Any) -> str:
|
||||
return str(value or "").strip().lower()
|
||||
|
||||
@classmethod
|
||||
def is_explicit_fixture(cls, dataset: Any) -> bool:
|
||||
source = cls._normalise(cls._value(dataset, "source"))
|
||||
source_name = cls._normalise(cls._value(dataset, "source_name"))
|
||||
metadata = cls._mapping(cls._value(dataset, "metadata_json"))
|
||||
source_metadata = cls._mapping(cls._value(dataset, "source_metadata"))
|
||||
provenance = cls._mapping(cls._value(dataset, "provenance_metadata"))
|
||||
return bool(
|
||||
source in _FIXTURE_SOURCE_KEYS
|
||||
or source_name in _FIXTURE_SOURCE_KEYS
|
||||
or metadata.get("fixture") is True
|
||||
or metadata.get("fixture_mode") is True
|
||||
or source_metadata.get("fixture") is True
|
||||
or source_metadata.get("fixture_mode") is True
|
||||
or provenance.get("fixture") is True
|
||||
or provenance.get("fixture_mode") is True
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _phase2_state_is_absent(cls, dataset: Any) -> bool:
|
||||
fields = (
|
||||
"data_contract_key",
|
||||
"data_contract_version",
|
||||
"validation_status",
|
||||
"provenance_status",
|
||||
"lineage_status",
|
||||
"quarantine_status",
|
||||
"source_registry_id",
|
||||
"source_snapshot_id",
|
||||
)
|
||||
return all(cls._value(dataset, field) in {None, ""} for field in fields)
|
||||
|
||||
@staticmethod
|
||||
def _is_transient_orm_dataset(dataset: Any) -> bool:
|
||||
"""Recognize only unpersisted ORM fixtures, never database rows."""
|
||||
|
||||
if not isinstance(dataset, Dataset):
|
||||
return False
|
||||
try:
|
||||
return bool(sa_inspect(dataset).transient)
|
||||
except Exception: # pragma: no cover - defensive for unusual test doubles
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def _evidence(
|
||||
cls,
|
||||
dataset: Any,
|
||||
purpose: DatasetConsumptionPurpose,
|
||||
reference_task: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
source_registry = cls._value(dataset, "source_registry")
|
||||
source_snapshot = cls._value(dataset, "source_snapshot")
|
||||
source_policy = cls._mapping(cls._value(source_registry, "usage_policy_json"))
|
||||
validation_authority = cls._mapping(source_policy.get("validation_authority"))
|
||||
reference_approvals = cls._mapping(source_policy.get("reference_validation_approvals"))
|
||||
return {
|
||||
"dataset_id": str(cls._value(dataset, "id") or ""),
|
||||
"purpose": purpose,
|
||||
"dataset_status": cls._normalise(cls._value(dataset, "status")),
|
||||
"source": cls._normalise(cls._value(dataset, "source")),
|
||||
"source_name": cls._normalise(cls._value(dataset, "source_name")),
|
||||
"data_contract_key": cls._value(dataset, "data_contract_key"),
|
||||
"data_contract_version": cls._value(dataset, "data_contract_version"),
|
||||
"validation_status": cls._normalise(cls._value(dataset, "validation_status")),
|
||||
"provenance_status": cls._normalise(cls._value(dataset, "provenance_status")),
|
||||
"lineage_status": cls._normalise(cls._value(dataset, "lineage_status")),
|
||||
"quarantine_status": cls._normalise(cls._value(dataset, "quarantine_status")),
|
||||
"checksum_sha256": cls._value(dataset, "checksum_sha256"),
|
||||
"source_registry_id": str(cls._value(dataset, "source_registry_id") or ""),
|
||||
"source_snapshot_id": str(cls._value(dataset, "source_snapshot_id") or ""),
|
||||
"source_classification": cls._normalise(cls._value(source_registry, "classification")),
|
||||
"source_key": cls._normalise(cls._value(source_registry, "source_key")),
|
||||
"source_ground_truth_allowed": source_policy.get("ground_truth_allowed") is True,
|
||||
"source_validation_authority": validation_authority,
|
||||
"source_reference_validation_approvals": reference_approvals,
|
||||
"source_authority_scope": cls._mapping(cls._value(source_registry, "authority_scope_json")),
|
||||
"reference_task": cls._normalise(reference_task),
|
||||
"snapshot_source_registry_id": str(cls._value(source_snapshot, "source_registry_id") or ""),
|
||||
"snapshot_ingest_status": cls._normalise(cls._value(source_snapshot, "ingest_status")),
|
||||
"snapshot_freshness_status": cls._normalise(cls._value(source_snapshot, "freshness_status")),
|
||||
"snapshot_checksum_sha256": cls._value(source_snapshot, "checksum_sha256"),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def evaluate(
|
||||
cls,
|
||||
dataset: Any,
|
||||
*,
|
||||
purpose: DatasetConsumptionPurpose,
|
||||
fixture_mode: bool = False,
|
||||
reference_task: str | None = None,
|
||||
) -> DatasetConsumptionDecision:
|
||||
"""Return a stable decision without mutating the Dataset.
|
||||
|
||||
``fixture_mode`` can be supplied only by an explicitly fixture-only
|
||||
caller. It is evidence for a QA fixture path, never a relaxation for
|
||||
a production boundary or an explicit unsafe state.
|
||||
"""
|
||||
|
||||
if purpose not in _VALID_PURPOSES:
|
||||
raise ValueError(f"Unsupported dataset-consumption purpose: {purpose}")
|
||||
|
||||
evidence = cls._evidence(dataset, purpose, reference_task)
|
||||
reasons: list[str] = []
|
||||
explicit_fixture = cls.is_explicit_fixture(dataset)
|
||||
phase2_absent = cls._phase2_state_is_absent(dataset)
|
||||
|
||||
# These are irrevocable safety states. They are checked before a
|
||||
# fixture exception, so fixture rows cannot hide a bad recorded state.
|
||||
if evidence["dataset_status"] in {"failed", "quarantined"}:
|
||||
reasons.append("dataset_status_unsafe")
|
||||
if evidence["quarantine_status"] == "quarantined":
|
||||
reasons.append("dataset_quarantined")
|
||||
if evidence["validation_status"] == "failed":
|
||||
reasons.append("validation_failed")
|
||||
if evidence["provenance_status"] == "incomplete":
|
||||
reasons.append("provenance_incomplete")
|
||||
if evidence["lineage_status"] == "incomplete":
|
||||
reasons.append("lineage_incomplete")
|
||||
if evidence["snapshot_ingest_status"] in {"failed", "quarantined"}:
|
||||
reasons.append("source_snapshot_unsafe")
|
||||
# A source family can be authoritative while an individual snapshot
|
||||
# remains too old or insufficiently described to trust. Historical
|
||||
# data that is intentionally valid needs an explicit
|
||||
# ``not_applicable`` contract policy; an omitted, due or stale status
|
||||
# cannot silently enter a production boundary.
|
||||
if evidence["source_snapshot_id"] and evidence["snapshot_freshness_status"] not in _CONSUMABLE_SNAPSHOT_FRESHNESS:
|
||||
reasons.append("source_snapshot_freshness_not_eligible")
|
||||
|
||||
if reasons:
|
||||
return DatasetConsumptionDecision(
|
||||
eligible=False,
|
||||
purpose=purpose,
|
||||
fixture_legacy_exception=False,
|
||||
reasons=tuple(sorted(set(reasons))),
|
||||
evidence=evidence,
|
||||
)
|
||||
|
||||
# Fixtures are evidence for tests and QA only. They cannot become
|
||||
# production inference, derived processing or export inputs merely by
|
||||
# presenting a fixture flag at a public service boundary.
|
||||
if phase2_absent and explicit_fixture:
|
||||
if purpose == "quality_assessment":
|
||||
return DatasetConsumptionDecision(
|
||||
eligible=True,
|
||||
purpose=purpose,
|
||||
fixture_legacy_exception=True,
|
||||
reasons=(),
|
||||
evidence=evidence,
|
||||
)
|
||||
if purpose == "authoritative_coverage":
|
||||
reasons.append("fixture_not_authoritative_coverage")
|
||||
else:
|
||||
reasons.append("fixture_qa_only")
|
||||
elif phase2_absent and fixture_mode:
|
||||
# `fixture_mode` is a caller flag, not a source trust claim. A
|
||||
# manual/unknown production dataset must never self-designate as a
|
||||
# fixture merely by supplying this parameter.
|
||||
reasons.append("fixture_source_required")
|
||||
|
||||
# Existing service tests construct transient SQLAlchemy Dataset objects
|
||||
# directly rather than retrieving a persisted row. A production
|
||||
# `db.get()` result is persistent and never enters this branch. This
|
||||
# compatibility path is intentionally unavailable to coverage, where
|
||||
# a fixture must never appear authoritative.
|
||||
if (
|
||||
phase2_absent
|
||||
and not reasons
|
||||
and cls._is_transient_orm_dataset(dataset)
|
||||
and purpose == "quality_assessment"
|
||||
):
|
||||
return DatasetConsumptionDecision(
|
||||
eligible=True,
|
||||
purpose=purpose,
|
||||
fixture_legacy_exception=True,
|
||||
reasons=(),
|
||||
evidence=evidence,
|
||||
)
|
||||
|
||||
# QA unit tests intentionally use projection objects
|
||||
# instead of persisted ORM Datasets. Those projections cannot enter an
|
||||
# application API boundary; keep the exception isolated to read-only
|
||||
# candidate verification. Inference, reference validation, derived
|
||||
# processing, export and coverage never accept a projection.
|
||||
if phase2_absent and not reasons and not isinstance(dataset, Dataset) and purpose == "quality_assessment":
|
||||
return DatasetConsumptionDecision(
|
||||
eligible=True,
|
||||
purpose=purpose,
|
||||
fixture_legacy_exception=True,
|
||||
reasons=(),
|
||||
evidence=evidence,
|
||||
)
|
||||
|
||||
if phase2_absent:
|
||||
reasons.append("phase2_provenance_missing")
|
||||
if evidence["dataset_status"] != "ready":
|
||||
reasons.append("dataset_not_ready")
|
||||
if evidence["validation_status"] != "passed":
|
||||
reasons.append("validation_not_passed")
|
||||
if evidence["provenance_status"] != "complete":
|
||||
reasons.append("provenance_not_complete")
|
||||
if evidence["lineage_status"] not in {"complete", "not_applicable"}:
|
||||
reasons.append("lineage_not_complete")
|
||||
if evidence["quarantine_status"] != "not_quarantined":
|
||||
reasons.append("quarantine_status_not_clear")
|
||||
if not evidence["data_contract_key"] or not evidence["data_contract_version"]:
|
||||
reasons.append("data_contract_not_versioned")
|
||||
if not _CHECKSUM_SHA256.fullmatch(str(evidence["checksum_sha256"] or "")):
|
||||
reasons.append("dataset_checksum_invalid")
|
||||
if not evidence["source_registry_id"]:
|
||||
reasons.append("source_registry_missing")
|
||||
if not evidence["source_snapshot_id"]:
|
||||
reasons.append("source_snapshot_missing")
|
||||
if evidence["source_registry_id"] and not evidence["source_classification"]:
|
||||
reasons.append("source_registry_unresolved")
|
||||
if evidence["source_snapshot_id"] and evidence["snapshot_ingest_status"] != "ingested":
|
||||
reasons.append("source_snapshot_not_ingested")
|
||||
if evidence["source_snapshot_id"] and not _CHECKSUM_SHA256.fullmatch(
|
||||
str(evidence["snapshot_checksum_sha256"] or "")
|
||||
):
|
||||
reasons.append("source_snapshot_checksum_invalid")
|
||||
if (
|
||||
evidence["source_registry_id"]
|
||||
and evidence["source_snapshot_id"]
|
||||
and evidence["snapshot_source_registry_id"]
|
||||
and evidence["source_registry_id"] != evidence["snapshot_source_registry_id"]
|
||||
):
|
||||
reasons.append("source_snapshot_registry_mismatch")
|
||||
if (
|
||||
_CHECKSUM_SHA256.fullmatch(str(evidence["checksum_sha256"] or ""))
|
||||
and _CHECKSUM_SHA256.fullmatch(str(evidence["snapshot_checksum_sha256"] or ""))
|
||||
and str(evidence["checksum_sha256"]).lower() != str(evidence["snapshot_checksum_sha256"]).lower()
|
||||
):
|
||||
reasons.append("source_snapshot_checksum_mismatch")
|
||||
|
||||
if evidence["source_key"] and evidence["source_name"] and evidence["source_key"] != evidence["source_name"]:
|
||||
reasons.append("source_registry_identity_mismatch")
|
||||
|
||||
# Contextual, corroborative, authoritative and properly derived
|
||||
# sources can serve their declared non-ground-truth roles once the
|
||||
# full governed contract passes. Experimental/manual sources cannot
|
||||
# cross a production boundary; only an explicitly marked fixture may
|
||||
# participate in candidate QA.
|
||||
experimental_source = (
|
||||
evidence["source_classification"] == "experimental"
|
||||
or evidence["source_key"] in _UNTRUSTED_SOURCE_KEYS
|
||||
)
|
||||
if experimental_source:
|
||||
if purpose == "quality_assessment" and explicit_fixture:
|
||||
pass
|
||||
elif purpose == "quality_assessment":
|
||||
reasons.append("experimental_source_requires_fixture_qa")
|
||||
else:
|
||||
reasons.append("experimental_source_not_allowed_for_purpose")
|
||||
|
||||
if purpose == "authoritative_coverage":
|
||||
if evidence["source_classification"] != "authoritative":
|
||||
reasons.append("coverage_source_not_authoritative")
|
||||
if evidence["source_key"] and evidence["source_name"] and evidence["source_key"] != evidence["source_name"]:
|
||||
reasons.append("coverage_source_identity_mismatch")
|
||||
|
||||
if purpose == "reference_validation":
|
||||
if cls._normalise(cls._value(dataset, "dataset_role")) != "reference":
|
||||
reasons.append("reference_dataset_role_required")
|
||||
if evidence["source_classification"] != "authoritative":
|
||||
reasons.append("reference_source_not_authoritative")
|
||||
if evidence["source_ground_truth_allowed"] is not True:
|
||||
reasons.append("reference_source_not_ground_truth_allowed")
|
||||
if not evidence["reference_task"]:
|
||||
reasons.append("reference_validation_task_required")
|
||||
elif not cls._reference_task_is_approved(evidence):
|
||||
reasons.append("reference_task_authority_not_approved")
|
||||
|
||||
return DatasetConsumptionDecision(
|
||||
eligible=not reasons,
|
||||
purpose=purpose,
|
||||
fixture_legacy_exception=False,
|
||||
reasons=tuple(sorted(set(reasons))),
|
||||
evidence=evidence,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def assert_eligible(
|
||||
cls,
|
||||
dataset: Any,
|
||||
*,
|
||||
purpose: DatasetConsumptionPurpose,
|
||||
fixture_mode: bool = False,
|
||||
reference_task: str | None = None,
|
||||
) -> DatasetConsumptionDecision:
|
||||
decision = cls.evaluate(
|
||||
dataset,
|
||||
purpose=purpose,
|
||||
fixture_mode=fixture_mode,
|
||||
reference_task=reference_task,
|
||||
)
|
||||
if decision.eligible:
|
||||
return decision
|
||||
code = "DATASET_QUARANTINED" if any(
|
||||
reason in {"dataset_status_unsafe", "dataset_quarantined", "validation_failed", "source_snapshot_unsafe"}
|
||||
for reason in decision.reasons
|
||||
) else "DATASET_PROVENANCE_INCOMPLETE"
|
||||
raise AppError(
|
||||
code=code,
|
||||
message="Dataset cannot be consumed until its provenance and validation gates are satisfied.",
|
||||
status_code=409,
|
||||
details={
|
||||
"dataset_id": decision.evidence["dataset_id"],
|
||||
"purpose": purpose,
|
||||
"reasons": list(decision.reasons),
|
||||
"fixture_legacy_exception": decision.fixture_legacy_exception,
|
||||
},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def eligible_for_authoritative_coverage(cls, dataset: Any) -> bool:
|
||||
"""Return false instead of raising so coverage can report a gap safely."""
|
||||
|
||||
return cls.evaluate(dataset, purpose="authoritative_coverage").eligible
|
||||
|
||||
@staticmethod
|
||||
def _reference_task_is_approved(evidence: Mapping[str, Any]) -> bool:
|
||||
"""Require task-specific primary authority or an explicit zone approval.
|
||||
|
||||
``classification=authoritative`` is deliberately not a blanket
|
||||
permission to serve as building truth. A source may be authoritative
|
||||
for an address lifecycle or elevation product while remaining only
|
||||
corroborative for footprint QA. A regional product that is marked
|
||||
``*_pending_contract`` similarly remains blocked until an operator
|
||||
records a narrow product-and-zone approval in its server-owned policy.
|
||||
"""
|
||||
|
||||
task = str(evidence.get("reference_task") or "").strip().lower()
|
||||
authority = DatasetConsumptionGate._normalise(
|
||||
DatasetConsumptionGate._mapping(evidence.get("source_validation_authority")).get(task)
|
||||
)
|
||||
if authority == "primary":
|
||||
return True
|
||||
if authority not in {"approved", "approved_product_zone"}:
|
||||
return False
|
||||
|
||||
approvals = DatasetConsumptionGate._mapping(evidence.get("source_reference_validation_approvals"))
|
||||
approval = DatasetConsumptionGate._mapping(approvals.get(task))
|
||||
if approval.get("approved") is not True:
|
||||
return False
|
||||
|
||||
source_key = str(evidence.get("source_key") or "").strip().lower()
|
||||
source_scope = DatasetConsumptionGate._mapping(evidence.get("source_authority_scope"))
|
||||
source_zone = str(source_scope.get("zone") or source_scope.get("scope") or "").strip()
|
||||
approved_keys = approval.get("source_keys")
|
||||
approved_zones = approval.get("zones")
|
||||
if not isinstance(approved_keys, list) or source_key not in {
|
||||
str(value).strip().lower() for value in approved_keys
|
||||
}:
|
||||
return False
|
||||
if not isinstance(approved_zones, list) or source_zone not in {
|
||||
str(value).strip() for value in approved_zones
|
||||
}:
|
||||
return False
|
||||
return True
|
||||
File diff suppressed because it is too large
Load Diff
@@ -12,6 +12,7 @@ from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Area, Dataset, DatasetVersion, Metric, Project, QualityCheck
|
||||
from app.schemas.demo import DemoWorkflowResponse
|
||||
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
|
||||
from app.services.geojson_service import parse_geojson_payload
|
||||
from app.services.qa_service import QaService
|
||||
from app.services.quality_service import QualityService
|
||||
@@ -30,17 +31,17 @@ class DemoWorkflowService:
|
||||
EXPECTED_METRICS_FILENAME = "expected_qa_metrics.json"
|
||||
|
||||
@staticmethod
|
||||
def _add_initial_version(db: Session, dataset: Dataset) -> None:
|
||||
db.add(
|
||||
DatasetVersion(
|
||||
dataset_id=dataset.id,
|
||||
version=1,
|
||||
storage_path=dataset.storage_path,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
)
|
||||
def _add_initial_version(db: Session, dataset: Dataset) -> DatasetVersion:
|
||||
version = DatasetVersion(
|
||||
dataset_id=dataset.id,
|
||||
version=1,
|
||||
storage_path=dataset.storage_path,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
source_metadata=dataset.source_metadata,
|
||||
provenance_metadata=dataset.provenance_metadata,
|
||||
)
|
||||
db.add(version)
|
||||
return version
|
||||
|
||||
@staticmethod
|
||||
def _repo_root() -> Path:
|
||||
@@ -233,24 +234,34 @@ class DemoWorkflowService:
|
||||
crs=metadata.get("crs"),
|
||||
bounds_json=metadata.get("bounds_json"),
|
||||
metadata_json=metadata,
|
||||
status="ready",
|
||||
status="validating",
|
||||
)
|
||||
db.add(dataset)
|
||||
DemoWorkflowService._add_initial_version(db, dataset)
|
||||
version = DemoWorkflowService._add_initial_version(db, dataset)
|
||||
is_ready = DerivedDatasetGovernanceService.govern_vector(
|
||||
db,
|
||||
dataset=dataset,
|
||||
dataset_version=version,
|
||||
feature_collection=payload,
|
||||
source_key="fixture",
|
||||
operation="demo.fixture_vector",
|
||||
operation_parameters={"fixture_name": filename, "role": role},
|
||||
)
|
||||
if is_ready:
|
||||
VectorFeatureService.persist_geojson_features(
|
||||
db=db,
|
||||
dataset_id=dataset.id,
|
||||
payload=payload,
|
||||
feature_class=reference_layer_name or "building",
|
||||
commit=False,
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
VectorFeatureService.persist_geojson_features(
|
||||
db=db,
|
||||
dataset_id=dataset.id,
|
||||
payload=payload,
|
||||
feature_class=reference_layer_name or "building",
|
||||
)
|
||||
return dataset
|
||||
|
||||
@staticmethod
|
||||
def _create_demo_raster_bytes() -> bytes:
|
||||
numpy = importlib.import_module("numpy")
|
||||
rasterio = importlib.import_module("rasterio")
|
||||
rasterio_io = importlib.import_module("rasterio.io")
|
||||
rasterio_transform = importlib.import_module("rasterio.transform")
|
||||
|
||||
@@ -318,10 +329,19 @@ class DemoWorkflowService:
|
||||
crs=metadata.get("crs"),
|
||||
bounds_json=bounds_json,
|
||||
metadata_json=metadata,
|
||||
status="ready",
|
||||
status="validating",
|
||||
)
|
||||
db.add(dataset)
|
||||
DemoWorkflowService._add_initial_version(db, dataset)
|
||||
version = DemoWorkflowService._add_initial_version(db, dataset)
|
||||
DerivedDatasetGovernanceService.govern_raster(
|
||||
db,
|
||||
dataset=dataset,
|
||||
dataset_version=version,
|
||||
raster_metadata=metadata,
|
||||
source_key="fixture",
|
||||
operation="demo.fixture_raster",
|
||||
operation_parameters={"fixture_name": DemoWorkflowService.RASTER_FILENAME},
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(dataset)
|
||||
return dataset
|
||||
|
||||
@@ -0,0 +1,516 @@
|
||||
"""Governance for persisted derived and fixture datasets.
|
||||
|
||||
Dataset importers own raw-source ingestion. This small service owns the
|
||||
other persistence boundary: artifacts produced inside the workbench (vector
|
||||
and raster operations) and the explicitly local demo fixtures. It is kept
|
||||
separate from :mod:`dataset_service` so an operation can never create a ready
|
||||
dataset without a source registry binding, immutable snapshot, validation
|
||||
report and, for derived results, a durable lineage edge.
|
||||
|
||||
The ``Session.query`` capability check deliberately preserves lightweight
|
||||
unit-test doubles used by pre-Phase-2 tests. Real SQLAlchemy sessions always
|
||||
take the governed branch; the compatibility branch is not reachable in the
|
||||
application runtime.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from hashlib import sha256
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Mapping
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.models import Dataset, DatasetVersion
|
||||
from app.services.data_contract_validation import (
|
||||
IssueSeverity,
|
||||
LineageStatus,
|
||||
LineageEvidence,
|
||||
ProvenanceStatus,
|
||||
QuarantineStatus,
|
||||
TransformationEvidence,
|
||||
ValidationIssue,
|
||||
ValidationReport,
|
||||
ValidationStatus,
|
||||
build_raster_ingest_input,
|
||||
build_vector_ingest_input,
|
||||
validate_registered_asset,
|
||||
)
|
||||
from app.services.data_quarantine_service import DataQuarantineService
|
||||
from app.services.dataset_consumption_gate_service import DatasetConsumptionDecision, DatasetConsumptionGate
|
||||
from app.services.source_registry_service import SourceRegistryService
|
||||
|
||||
|
||||
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||
_CANONICAL_VECTOR_CRS = "EPSG:4326"
|
||||
|
||||
|
||||
class DerivedDatasetGovernanceService:
|
||||
"""Apply Phase-2 provenance rules to non-importer Dataset creation.
|
||||
|
||||
Callers add the dataset and first immutable version, then call one of the
|
||||
``govern_*`` methods before committing. A failed contract deliberately
|
||||
leaves the artifact and its durable quarantine record in the transaction;
|
||||
it is never silently promoted to ``ready``.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def persistence_available(db: Session) -> bool:
|
||||
"""Return whether this is a real ORM persistence session.
|
||||
|
||||
Historical unit tests use minimal fakes with ``add``/``commit`` only.
|
||||
Keeping that explicitly isolated avoids pretending a fake test store
|
||||
has source-registry guarantees while production remains fail-closed.
|
||||
"""
|
||||
|
||||
return callable(getattr(db, "query", None)) and callable(getattr(db, "flush", None))
|
||||
|
||||
@classmethod
|
||||
def govern_vector(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
dataset: Dataset,
|
||||
dataset_version: DatasetVersion,
|
||||
feature_collection: Mapping[str, Any],
|
||||
source_key: str,
|
||||
operation: str,
|
||||
parent_dataset: Dataset | None = None,
|
||||
operation_parameters: Mapping[str, Any] | None = None,
|
||||
) -> bool:
|
||||
"""Validate and bind a vector result, returning ``True`` when ready."""
|
||||
|
||||
if not cls.persistence_available(db):
|
||||
# Explicit compatibility for historical minimal test fixtures.
|
||||
# Production sessions always have query/flush and never take this
|
||||
# branch.
|
||||
dataset.status = "ready"
|
||||
return True
|
||||
|
||||
parent_gate = cls._parent_derived_processing_gate(parent_dataset)
|
||||
metadata = dict(dataset.metadata_json or {})
|
||||
output_crs = str(metadata.get("crs") or dataset.crs or _CANONICAL_VECTOR_CRS)
|
||||
source = SourceRegistryService.ensure_server_owned_source(db, source_key)
|
||||
snapshot = cls._record_snapshot(
|
||||
db,
|
||||
source_key=source_key,
|
||||
dataset=dataset,
|
||||
operation=operation,
|
||||
source_crs=output_crs,
|
||||
spatial_resolution=metadata.get("resolution_json") or {"status": "not_applicable"},
|
||||
geographic_coverage={"bounds": metadata.get("bounds_json") or dataset.bounds_json},
|
||||
observed_schema={
|
||||
"dataset_type": "vector",
|
||||
"geometry_types": metadata.get("geometry_types") or [],
|
||||
"feature_count": metadata.get("feature_count"),
|
||||
},
|
||||
)
|
||||
lineage = cls._lineage_evidence(parent_dataset, operation, operation_parameters)
|
||||
report = validate_registered_asset(
|
||||
build_vector_ingest_input(
|
||||
asset_id=str(dataset.id),
|
||||
source_crs=output_crs,
|
||||
storage_crs=output_crs,
|
||||
feature_collection=feature_collection,
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
computed_checksum_sha256=dataset.checksum_sha256,
|
||||
source_registry_id=str(source.id),
|
||||
source_snapshot_id=str(snapshot.id),
|
||||
imported_at=dataset.imported_at or datetime.now(timezone.utc),
|
||||
metadata=cls._contract_metadata(metadata, source.license_name),
|
||||
observed_at=dataset.observed_at,
|
||||
valid_from=dataset.valid_from,
|
||||
valid_to=dataset.valid_to,
|
||||
temporal_unknown_reason=cls._temporal_unknown_reason(dataset),
|
||||
source_version=dataset.source_version,
|
||||
source_version_unknown_reason=cls._source_version_unknown_reason(dataset),
|
||||
lineage=lineage,
|
||||
)
|
||||
)
|
||||
if parent_gate is not None and not parent_gate.eligible:
|
||||
report = cls._with_parent_gate_failure(report, parent_gate)
|
||||
return cls._apply(
|
||||
db,
|
||||
dataset=dataset,
|
||||
dataset_version=dataset_version,
|
||||
source=source,
|
||||
snapshot=snapshot,
|
||||
report=report,
|
||||
stage="derived_vector_validation",
|
||||
parent_dataset=parent_dataset,
|
||||
operation=operation,
|
||||
operation_parameters=operation_parameters,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def govern_raster(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
dataset: Dataset,
|
||||
dataset_version: DatasetVersion,
|
||||
raster_metadata: Mapping[str, Any],
|
||||
source_key: str,
|
||||
operation: str,
|
||||
parent_dataset: Dataset | None = None,
|
||||
operation_parameters: Mapping[str, Any] | None = None,
|
||||
) -> bool:
|
||||
"""Validate and bind a raster result, returning ``True`` when ready."""
|
||||
|
||||
if not cls.persistence_available(db):
|
||||
dataset.status = "ready"
|
||||
return True
|
||||
|
||||
parent_gate = cls._parent_derived_processing_gate(parent_dataset)
|
||||
metadata = dict(raster_metadata or {})
|
||||
output_crs = str(metadata.get("crs") or dataset.crs or "") or None
|
||||
source = SourceRegistryService.ensure_server_owned_source(db, source_key)
|
||||
snapshot = cls._record_snapshot(
|
||||
db,
|
||||
source_key=source_key,
|
||||
dataset=dataset,
|
||||
operation=operation,
|
||||
source_crs=output_crs,
|
||||
spatial_resolution=cls._raster_resolution(metadata, output_crs),
|
||||
geographic_coverage={"bounds": metadata.get("bounds") or dataset.bounds_json},
|
||||
observed_schema={
|
||||
"dataset_type": "raster",
|
||||
"width": metadata.get("width"),
|
||||
"height": metadata.get("height"),
|
||||
"band_count": metadata.get("band_count"),
|
||||
"dtype": metadata.get("dtype"),
|
||||
},
|
||||
)
|
||||
lineage = cls._lineage_evidence(parent_dataset, operation, operation_parameters)
|
||||
report = validate_registered_asset(
|
||||
build_raster_ingest_input(
|
||||
asset_id=str(dataset.id),
|
||||
source_crs=output_crs,
|
||||
storage_crs=output_crs,
|
||||
raster_profile=metadata,
|
||||
bounds=metadata.get("bounds") or dataset.bounds_json,
|
||||
resolution=cls._raster_resolution(metadata, output_crs),
|
||||
checksum_sha256=dataset.checksum_sha256,
|
||||
computed_checksum_sha256=dataset.checksum_sha256,
|
||||
source_registry_id=str(source.id),
|
||||
source_snapshot_id=str(snapshot.id),
|
||||
imported_at=dataset.imported_at or datetime.now(timezone.utc),
|
||||
metadata=cls._contract_metadata(metadata, source.license_name),
|
||||
observed_at=dataset.observed_at,
|
||||
valid_from=dataset.valid_from,
|
||||
valid_to=dataset.valid_to,
|
||||
temporal_unknown_reason=cls._temporal_unknown_reason(dataset),
|
||||
source_version=dataset.source_version,
|
||||
source_version_unknown_reason=cls._source_version_unknown_reason(dataset),
|
||||
lineage=lineage,
|
||||
)
|
||||
)
|
||||
if parent_gate is not None and not parent_gate.eligible:
|
||||
report = cls._with_parent_gate_failure(report, parent_gate)
|
||||
return cls._apply(
|
||||
db,
|
||||
dataset=dataset,
|
||||
dataset_version=dataset_version,
|
||||
source=source,
|
||||
snapshot=snapshot,
|
||||
report=report,
|
||||
stage="derived_raster_validation",
|
||||
parent_dataset=parent_dataset,
|
||||
operation=operation,
|
||||
operation_parameters=operation_parameters,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _record_snapshot(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
source_key: str,
|
||||
dataset: Dataset,
|
||||
operation: str,
|
||||
source_crs: str | None,
|
||||
spatial_resolution: Mapping[str, Any] | None,
|
||||
geographic_coverage: Mapping[str, Any] | None,
|
||||
observed_schema: Mapping[str, Any] | None,
|
||||
):
|
||||
checksum = cls._checksum_or_placeholder(dataset.checksum_sha256)
|
||||
# A derived artifact has a distinct creation event even when its bytes
|
||||
# equal a prior output. Include the immutable dataset id so source
|
||||
# snapshots never collide on a different fetched_at timestamp.
|
||||
snapshot_key = f"{source_key}:{operation}:{dataset.id}:{checksum}"
|
||||
return SourceRegistryService.record_snapshot(
|
||||
db,
|
||||
source_key=source_key,
|
||||
snapshot_key=snapshot_key,
|
||||
checksum_sha256=checksum,
|
||||
source_version=dataset.source_version or f"{operation}:1.0.0",
|
||||
snapshot_at=dataset.observed_at,
|
||||
fetched_at=dataset.imported_at or datetime.now(timezone.utc),
|
||||
crs=source_crs,
|
||||
units=cls._units_for_crs(source_crs),
|
||||
spatial_resolution=dict(spatial_resolution or {"status": "unknown"}),
|
||||
temporal_coverage={
|
||||
"observed_at": cls._datetime_value(dataset.observed_at),
|
||||
"valid_from": cls._datetime_value(dataset.valid_from),
|
||||
"valid_to": cls._datetime_value(dataset.valid_to),
|
||||
},
|
||||
geographic_coverage=dict(geographic_coverage or {"status": "unknown"}),
|
||||
observed_schema=dict(observed_schema or {"status": "unknown"}),
|
||||
# A transform cannot establish source freshness. With no source
|
||||
# observation it is not applicable; with one it still needs an
|
||||
# explicit policy review rather than a fabricated "current" flag.
|
||||
freshness_status="not_applicable" if dataset.observed_at is None else "review_required",
|
||||
ingest_status="ingested",
|
||||
known_limitations=[
|
||||
"Derived and fixture artifacts inherit no automatic source authority beyond their explicit registry entry and lineage.",
|
||||
],
|
||||
snapshot_metadata={
|
||||
"operation": operation,
|
||||
"dataset_id": str(dataset.id),
|
||||
"storage_path": dataset.storage_path,
|
||||
"artifact_checksum_sha256": dataset.checksum_sha256,
|
||||
},
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _apply(
|
||||
cls,
|
||||
db: Session,
|
||||
*,
|
||||
dataset: Dataset,
|
||||
dataset_version: DatasetVersion,
|
||||
source: Any,
|
||||
snapshot: Any,
|
||||
report: ValidationReport,
|
||||
stage: str,
|
||||
parent_dataset: Dataset | None,
|
||||
operation: str,
|
||||
operation_parameters: Mapping[str, Any] | None,
|
||||
) -> bool:
|
||||
fields = report.persistence_fields()
|
||||
dataset.validation_report_json = fields["validation_report_json"]
|
||||
dataset_version.validation_report_json = fields["validation_report_json"]
|
||||
SourceRegistryService.bind_dataset_provenance(
|
||||
dataset,
|
||||
source=source,
|
||||
snapshot=snapshot,
|
||||
data_contract_key=fields["data_contract_key"],
|
||||
data_contract_version=fields["data_contract_version"],
|
||||
validation_status=fields["validation_status"],
|
||||
provenance_status=fields["provenance_status"],
|
||||
lineage_status=fields["lineage_status"],
|
||||
)
|
||||
SourceRegistryService.bind_dataset_version_provenance(
|
||||
dataset_version,
|
||||
source=source,
|
||||
snapshot=snapshot,
|
||||
data_contract_key=fields["data_contract_key"],
|
||||
data_contract_version=fields["data_contract_version"],
|
||||
validation_status=fields["validation_status"],
|
||||
provenance_status=fields["provenance_status"],
|
||||
lineage_status=fields["lineage_status"],
|
||||
)
|
||||
|
||||
# IDs for DatasetVersion defaults exist only after the caller adds and
|
||||
# flushes both rows. The operation services call this before commit.
|
||||
db.flush()
|
||||
if parent_dataset is not None:
|
||||
SourceRegistryService.record_lineage_edge(
|
||||
db,
|
||||
parent_dataset_id=parent_dataset.id,
|
||||
child_dataset_id=dataset.id,
|
||||
parent_dataset_version_id=cls._latest_parent_version_id(db, parent_dataset),
|
||||
child_dataset_version_id=dataset_version.id,
|
||||
relation_type="derived_from",
|
||||
transformation_name=operation,
|
||||
transformation_version="1.0.0",
|
||||
parameters=dict(operation_parameters or {}),
|
||||
input_checksum_sha256=cls._valid_checksum(parent_dataset.checksum_sha256),
|
||||
output_checksum_sha256=cls._valid_checksum(dataset.checksum_sha256),
|
||||
)
|
||||
|
||||
decision = DataQuarantineService.decide(report)
|
||||
if decision.eligible_for_use:
|
||||
dataset.status = "ready"
|
||||
dataset.quarantine_status = "not_quarantined"
|
||||
return True
|
||||
|
||||
SourceRegistryService.quarantine_dataset(
|
||||
db,
|
||||
dataset=dataset,
|
||||
dataset_version=dataset_version,
|
||||
source_snapshot=snapshot,
|
||||
stage=stage,
|
||||
reason_code=(decision.reason_codes[0] if decision.reason_codes else "DATA_CONTRACT_FAILED"),
|
||||
details={"validation_report": report.to_dict(), "quarantine_decision": decision.to_dict()},
|
||||
artifact_path=dataset.storage_path,
|
||||
artifact_checksum_sha256=cls._valid_checksum(dataset.checksum_sha256),
|
||||
)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _contract_metadata(metadata: Mapping[str, Any], license_name: str) -> dict[str, Any]:
|
||||
values = dict(metadata)
|
||||
values.setdefault("license", license_name)
|
||||
return values
|
||||
|
||||
@classmethod
|
||||
def _lineage_evidence(
|
||||
cls,
|
||||
parent_dataset: Dataset | None,
|
||||
operation: str,
|
||||
operation_parameters: Mapping[str, Any] | None,
|
||||
) -> LineageEvidence:
|
||||
upstream_ids: tuple[str, ...] = ()
|
||||
upstream_checksums: tuple[str, ...] = ()
|
||||
if parent_dataset is not None:
|
||||
upstream_ids = (str(parent_dataset.id),)
|
||||
# An invalid/missing parent checksum intentionally fails the
|
||||
# derived contract rather than inventing traceability. The same
|
||||
# applies to a legacy/unvalidated parent: it may remain visible
|
||||
# as evidence, but cannot create a new ready derived asset.
|
||||
parent_is_governed = (
|
||||
parent_dataset.status == "ready"
|
||||
and parent_dataset.quarantine_status == "not_quarantined"
|
||||
and parent_dataset.validation_status == "passed"
|
||||
and parent_dataset.provenance_status == "complete"
|
||||
and parent_dataset.source_registry_id is not None
|
||||
and parent_dataset.source_snapshot_id is not None
|
||||
)
|
||||
upstream_checksums = (
|
||||
parent_dataset.checksum_sha256 if parent_is_governed else "parent_dataset_not_governed",
|
||||
)
|
||||
transform_checksum = cls._stable_hash(
|
||||
{"operation": operation, "version": "1.0.0", "parameters": dict(operation_parameters or {})}
|
||||
)
|
||||
return LineageEvidence(
|
||||
upstream_asset_ids=upstream_ids,
|
||||
upstream_checksums_sha256=upstream_checksums,
|
||||
transformations=(
|
||||
TransformationEvidence(
|
||||
name=operation,
|
||||
version="1.0.0",
|
||||
checksum_sha256=transform_checksum,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _parent_derived_processing_gate(parent_dataset: Dataset | None) -> DatasetConsumptionDecision | None:
|
||||
"""Evaluate the durable parent boundary before creating a ready child.
|
||||
|
||||
We deliberately turn a rejected parent into a child validation failure
|
||||
(instead of simply raising): the output is then persisted with its
|
||||
source snapshot, validation evidence and durable quarantine record.
|
||||
This makes an attempted derivation from a manual or experimental
|
||||
dataset observable and prevents a caller from bypassing the boundary
|
||||
by invoking the governance service directly.
|
||||
"""
|
||||
|
||||
if parent_dataset is None:
|
||||
return None
|
||||
return DatasetConsumptionGate.evaluate(parent_dataset, purpose="derived_processing")
|
||||
|
||||
@staticmethod
|
||||
def _with_parent_gate_failure(
|
||||
report: ValidationReport,
|
||||
decision: DatasetConsumptionDecision,
|
||||
) -> ValidationReport:
|
||||
"""Attach an auditable, fail-closed lineage failure to a report."""
|
||||
|
||||
issue = ValidationIssue(
|
||||
code="PARENT_DATASET_NOT_ELIGIBLE_FOR_DERIVED_PROCESSING",
|
||||
category="lineage",
|
||||
field="lineage.parent_dataset",
|
||||
message="Parent dataset failed the governed derived-processing consumption gate.",
|
||||
severity=IssueSeverity.ERROR,
|
||||
expected="eligible governed parent dataset",
|
||||
observed={
|
||||
"dataset_id": decision.evidence.get("dataset_id"),
|
||||
"reasons": list(decision.reasons),
|
||||
"source_key": decision.evidence.get("source_key"),
|
||||
"source_classification": decision.evidence.get("source_classification"),
|
||||
},
|
||||
)
|
||||
return ValidationReport(
|
||||
asset_id=report.asset_id,
|
||||
data_contract_key=report.data_contract_key,
|
||||
data_contract_version=report.data_contract_version,
|
||||
contract_fingerprint_sha256=report.contract_fingerprint_sha256,
|
||||
validation_status=ValidationStatus.FAILED,
|
||||
provenance_status=(
|
||||
ProvenanceStatus.INCOMPLETE
|
||||
if report.provenance_status == ProvenanceStatus.COMPLETE
|
||||
else report.provenance_status
|
||||
),
|
||||
lineage_status=LineageStatus.INCOMPLETE,
|
||||
quarantine_status=QuarantineStatus.QUARANTINED,
|
||||
validation_scope=report.validation_scope,
|
||||
checked_at=report.checked_at,
|
||||
issues=(*report.issues, issue),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _latest_parent_version_id(db: Session, dataset: Dataset):
|
||||
version = (
|
||||
db.query(DatasetVersion)
|
||||
.filter(DatasetVersion.dataset_id == dataset.id)
|
||||
.order_by(DatasetVersion.version.desc())
|
||||
.first()
|
||||
)
|
||||
return version.id if version is not None else None
|
||||
|
||||
@staticmethod
|
||||
def _raster_resolution(metadata: Mapping[str, Any], crs: str | None) -> dict[str, Any] | None:
|
||||
values = metadata.get("resolution")
|
||||
if not isinstance(values, (list, tuple)) or len(values) < 2:
|
||||
return None
|
||||
try:
|
||||
return {"x": abs(float(values[0])), "y": abs(float(values[1])), "unit": DerivedDatasetGovernanceService._resolution_unit(crs)}
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _resolution_unit(crs: str | None) -> str:
|
||||
return "degree" if str(crs or "").upper() == _CANONICAL_VECTOR_CRS else "m"
|
||||
|
||||
@staticmethod
|
||||
def _units_for_crs(crs: str | None) -> str:
|
||||
return "degrees" if str(crs or "").upper() == _CANONICAL_VECTOR_CRS else "metres"
|
||||
|
||||
@staticmethod
|
||||
def _temporal_unknown_reason(dataset: Dataset) -> str | None:
|
||||
if dataset.observed_at is not None:
|
||||
return None
|
||||
return "Derived or fixture artifact inherits no precise observation timestamp from its input."
|
||||
|
||||
@staticmethod
|
||||
def _source_version_unknown_reason(dataset: Dataset) -> str | None:
|
||||
if dataset.source_version:
|
||||
return None
|
||||
return "Derived or fixture artifact has no source edition; the transform version is recorded separately."
|
||||
|
||||
@staticmethod
|
||||
def _datetime_value(value: datetime | None) -> str | None:
|
||||
return value.astimezone(timezone.utc).isoformat() if value is not None else None
|
||||
|
||||
@staticmethod
|
||||
def _stable_hash(value: Mapping[str, Any]) -> str:
|
||||
return sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def _valid_checksum(value: str | None) -> str | None:
|
||||
normalized = str(value or "").strip().lower()
|
||||
return normalized if _SHA256.fullmatch(normalized) else None
|
||||
|
||||
@classmethod
|
||||
def _checksum_or_placeholder(cls, value: str | None) -> str:
|
||||
checksum = cls._valid_checksum(value)
|
||||
if checksum is not None:
|
||||
return checksum
|
||||
# The contract receives the original invalid/missing checksum and
|
||||
# quarantines it. A deterministic placeholder only permits storing
|
||||
# the rejected snapshot without fabricating a valid artifact hash.
|
||||
return cls._stable_hash({"invalid_dataset_checksum": value or "missing"})
|
||||
@@ -19,10 +19,12 @@ from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project, Vect
|
||||
from app.schemas.detection import DetectionListResponse, DetectionRead, DetectionRunListResponse, DetectionRunRead, DetectionRunResponse
|
||||
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
|
||||
from app.services.detection_qa_service import DetectionQaService
|
||||
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
||||
from app.services.model_registry_service import ModelRegistryService
|
||||
from app.services.qa_service import QaService
|
||||
from app.services.quality_service import QualityService
|
||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenance, RuntimeModelProvenanceService
|
||||
from app.services.temporal_compatibility_service import TemporalCompatibilityService
|
||||
from app.services.yolo_adapter import YoloDetectionAdapter
|
||||
|
||||
@@ -97,6 +99,18 @@ class DetectionService:
|
||||
if model.model_id == resolved_settings.yolo_model_id and resolved_settings.yolo_enforce_validation_scope:
|
||||
DetectionService._validate_model_area_scope(db, dataset, resolved_settings)
|
||||
|
||||
# Never enter a production inference path with a persisted dataset
|
||||
# that has failed validation, incomplete provenance, or an active
|
||||
# quarantine. Fixture detection is a separate QA/test-only path.
|
||||
if model.model_id == "manual-fixture-detector":
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
dataset,
|
||||
purpose="quality_assessment",
|
||||
fixture_mode=True,
|
||||
)
|
||||
elif model.model_id == resolved_settings.yolo_model_id and model.configured:
|
||||
DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
|
||||
|
||||
run_parameters = {
|
||||
"model_id": model.model_id,
|
||||
"model_asset_id": selected_model_asset.model_asset_id if selected_model_asset else None,
|
||||
@@ -352,15 +366,6 @@ class DetectionService:
|
||||
reference_dataset,
|
||||
)
|
||||
|
||||
detections = DetectionService._query_detection_rows(
|
||||
db,
|
||||
analysis_run_id=analysis_run_id,
|
||||
dataset_id=run.dataset_id,
|
||||
class_name=class_name,
|
||||
min_confidence=min_confidence,
|
||||
)
|
||||
raw_candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in detections]
|
||||
candidate_geometries = raw_candidate_geometries
|
||||
run_parameters = run.parameters_json if isinstance(run.parameters_json, dict) else {}
|
||||
manifest_path = DetectionQaService.tile_manifest_path(run_parameters)
|
||||
resolved_settings = get_settings()
|
||||
@@ -375,6 +380,33 @@ class DetectionService:
|
||||
status_code=422,
|
||||
)
|
||||
|
||||
fixture_parameters = run_parameters.get("parameters_json")
|
||||
fixture_mode = bool(
|
||||
run.model_name == "manual-fixture-detector"
|
||||
and isinstance(fixture_parameters, dict)
|
||||
and fixture_parameters.get("fixture_mode") is True
|
||||
)
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
candidate_dataset,
|
||||
purpose="quality_assessment",
|
||||
fixture_mode=fixture_mode,
|
||||
)
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
reference_dataset,
|
||||
purpose="reference_validation",
|
||||
reference_task="building_validation",
|
||||
)
|
||||
|
||||
detections = DetectionService._query_detection_rows(
|
||||
db,
|
||||
analysis_run_id=analysis_run_id,
|
||||
dataset_id=run.dataset_id,
|
||||
class_name=class_name,
|
||||
min_confidence=min_confidence,
|
||||
)
|
||||
raw_candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in detections]
|
||||
candidate_geometries = raw_candidate_geometries
|
||||
|
||||
coverage = None
|
||||
if manifest_path:
|
||||
manifest = DetectionService._load_tile_manifest(manifest_path, resolved_settings.yolo_max_tiles)
|
||||
@@ -728,6 +760,19 @@ class DetectionService:
|
||||
) -> tuple[list[Detection], dict[str, Any]]:
|
||||
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles)
|
||||
model_path = Path(settings.yolo_model_path or "").expanduser()
|
||||
runtime_model_provenance = RuntimeModelProvenanceService.validate_for_production_runtime(
|
||||
db=db,
|
||||
model_path=model_path,
|
||||
model_id=model_name,
|
||||
task_type="object_detection",
|
||||
expected_model_version=model_version,
|
||||
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
|
||||
)
|
||||
DetectionService._attach_runtime_model_provenance(
|
||||
analysis_run,
|
||||
job,
|
||||
runtime_model_provenance,
|
||||
)
|
||||
adapter = yolo_adapter_class(settings)
|
||||
model = adapter.load_model(model_path)
|
||||
allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)}
|
||||
@@ -785,7 +830,10 @@ class DetectionService:
|
||||
"y_max": float(bbox[3]),
|
||||
},
|
||||
source_tile_path=candidate["source_tile_path"],
|
||||
properties_json=candidate["properties"],
|
||||
properties_json={
|
||||
**candidate["properties"],
|
||||
"runtime_model_provenance": runtime_model_provenance.as_dict(),
|
||||
},
|
||||
)
|
||||
db.add(detection)
|
||||
persisted.append(detection)
|
||||
@@ -796,8 +844,30 @@ class DetectionService:
|
||||
"raw_detection_count": len(candidates),
|
||||
"suppressed_detection_count": len(candidates) - len(filtered_candidates),
|
||||
"duplicate_iou_threshold": float(settings.yolo_duplicate_iou_threshold),
|
||||
"runtime_model_provenance": runtime_model_provenance.as_dict(),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _attach_runtime_model_provenance(
|
||||
analysis_run: AnalysisRun,
|
||||
job: Job,
|
||||
provenance: RuntimeModelProvenance,
|
||||
) -> None:
|
||||
"""Persist byte-bound model evidence with the run before adapter loading.
|
||||
|
||||
Individual detections retain the same evidence in ``properties_json``;
|
||||
this run-level copy is the compact audit root for a complete inference.
|
||||
Assigning fresh dictionaries matters for SQLAlchemy JSON change tracking.
|
||||
"""
|
||||
|
||||
evidence = provenance.as_dict()
|
||||
analysis_parameters = dict(analysis_run.parameters_json or {})
|
||||
analysis_parameters["runtime_model_provenance"] = evidence
|
||||
analysis_run.parameters_json = analysis_parameters
|
||||
job_parameters = dict(job.parameters_json or {})
|
||||
job_parameters["runtime_model_provenance"] = evidence
|
||||
job.parameters_json = job_parameters
|
||||
|
||||
@staticmethod
|
||||
def _canonical_class_name(value: Any) -> str:
|
||||
return str(value or "").strip().casefold()
|
||||
|
||||
@@ -23,6 +23,7 @@ from app.schemas.flood_hazard import FloodHazardPartitionSelectionRequest, Flood
|
||||
from app.schemas.temporal import TemporalComparisonRequest
|
||||
from app.schemas.thematic_raster import ThematicRasterSelectionRequest
|
||||
from app.services.dataset_service import DatasetService
|
||||
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
|
||||
from app.services.segmentation_service import SegmentationService
|
||||
@@ -34,12 +35,39 @@ from app.services.vector_feature_service import VectorFeatureService
|
||||
|
||||
|
||||
class ExportService:
|
||||
@staticmethod
|
||||
def _assert_run_source_dataset_exportable(db: Session, run: AnalysisRun) -> Dataset:
|
||||
"""Block an output export when its persisted source dataset is unsafe."""
|
||||
|
||||
if not run.dataset_id:
|
||||
raise AppError(
|
||||
code="DATASET_PROVENANCE_INCOMPLETE",
|
||||
message="Analysis output cannot be exported without a persisted source dataset.",
|
||||
status_code=409,
|
||||
)
|
||||
dataset = db.get(Dataset, run.dataset_id)
|
||||
if not dataset or dataset.project_id != run.project_id:
|
||||
raise AppError(code="DATASET_NOT_FOUND", message="Analysis source dataset not found", status_code=404)
|
||||
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
|
||||
return dataset
|
||||
|
||||
@staticmethod
|
||||
def export_map_result(
|
||||
db: Session,
|
||||
payload: MapResultExportRequest,
|
||||
) -> ExportCreateResponse:
|
||||
if payload.mode == "evolution":
|
||||
earlier_dataset = db.get(Dataset, payload.earlier_dataset_id)
|
||||
later_dataset = db.get(Dataset, payload.later_dataset_id)
|
||||
if (
|
||||
not earlier_dataset
|
||||
or not later_dataset
|
||||
or earlier_dataset.project_id != payload.project_id
|
||||
or later_dataset.project_id != payload.project_id
|
||||
):
|
||||
raise AppError(code="DATASET_NOT_FOUND", message="Temporal export dataset not found", status_code=404)
|
||||
DatasetConsumptionGate.assert_eligible(earlier_dataset, purpose="export")
|
||||
DatasetConsumptionGate.assert_eligible(later_dataset, purpose="export")
|
||||
comparison = TemporalAnalysisService.compare(
|
||||
db,
|
||||
project_id=payload.project_id,
|
||||
@@ -86,6 +114,7 @@ class ExportService:
|
||||
dataset = db.get(Dataset, payload.dataset_id)
|
||||
if not dataset or dataset.project_id != payload.project_id:
|
||||
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
||||
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
|
||||
if dataset.dataset_type in DatasetService.VECTOR_TYPES:
|
||||
if payload.partitioned:
|
||||
return ExportService.export_partitioned_vector_selection_geojson(
|
||||
@@ -219,6 +248,7 @@ class ExportService:
|
||||
limit: int = 1000,
|
||||
name: str | None = None,
|
||||
) -> ExportCreateResponse:
|
||||
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
|
||||
if dataset.source_name != "vmm_vha_bathymetry_profiles" or partition_scope_key != "flanders":
|
||||
raise AppError(
|
||||
code="PARTITIONED_VECTOR_EXPORT_UNSUPPORTED",
|
||||
@@ -308,6 +338,7 @@ class ExportService:
|
||||
details={"dataset_type": dataset.dataset_type},
|
||||
status_code=400,
|
||||
)
|
||||
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
|
||||
|
||||
selection_kwargs: dict[str, Any] = {
|
||||
"dataset_id": dataset_id,
|
||||
@@ -374,6 +405,7 @@ class ExportService:
|
||||
details={"dataset_type": dataset.dataset_type},
|
||||
status_code=400,
|
||||
)
|
||||
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
|
||||
|
||||
feature_collection = DatasetService.get_dataset_geojson(db, dataset_id)
|
||||
filename = ExportService._filename(name, f"{dataset.id}.geojson", ".geojson")
|
||||
@@ -401,6 +433,7 @@ class ExportService:
|
||||
run = db.get(AnalysisRun, analysis_run_id)
|
||||
if not run or run.analysis_type != "detection":
|
||||
raise AppError(code="DETECTION_RUN_NOT_FOUND", message="Detection run not found", status_code=404)
|
||||
ExportService._assert_run_source_dataset_exportable(db, run)
|
||||
|
||||
feature_collection = DetectionService.detections_to_geojson(db, analysis_run_id=analysis_run_id)
|
||||
filename = ExportService._filename(name, f"{run.id}-detections.geojson", ".geojson")
|
||||
@@ -428,6 +461,7 @@ class ExportService:
|
||||
run = db.get(AnalysisRun, analysis_run_id)
|
||||
if not run or run.analysis_type != "segmentation":
|
||||
raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404)
|
||||
ExportService._assert_run_source_dataset_exportable(db, run)
|
||||
|
||||
feature_collection = SegmentationService.segmentations_to_geojson(db, analysis_run_id=analysis_run_id)
|
||||
filename = ExportService._filename(name, f"{run.id}-segmentations.geojson", ".geojson")
|
||||
|
||||
@@ -9,6 +9,7 @@ from app.services.segmentation_adapter import (
|
||||
SamSegmentationAdapter,
|
||||
YoloSegmentationAdapter,
|
||||
)
|
||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||
from app.services.yolo_adapter import YoloDetectionAdapter
|
||||
from app.core.errors import AppError
|
||||
|
||||
@@ -144,9 +145,24 @@ class ModelRegistryService:
|
||||
elif not model_path.exists() or not model_path.is_file():
|
||||
limitation = "YOLO_SEG_MODEL_PATH does not point to an existing local model file. GeoIntel will not download segmentation model weights automatically."
|
||||
else:
|
||||
configured = True
|
||||
status = "configured"
|
||||
limitation = "Configured for local YOLO segmentation inference over an existing raster tile manifest."
|
||||
try:
|
||||
RuntimeModelProvenanceService.validate_for_runtime(
|
||||
model_path=model_path,
|
||||
model_id=settings.yolo_seg_model_id,
|
||||
task_type="segmentation",
|
||||
expected_model_version=settings.yolo_seg_model_version,
|
||||
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
|
||||
)
|
||||
except AppError as exc:
|
||||
status = "contract_incomplete"
|
||||
limitation = (
|
||||
"Configured YOLO segmentation weights are not runnable until their immutable "
|
||||
f"runtime provenance sidecar validates: {exc.message}"
|
||||
)
|
||||
else:
|
||||
configured = True
|
||||
status = "configured"
|
||||
limitation = "Configured for local YOLO segmentation inference over an existing raster tile manifest."
|
||||
|
||||
return DetectionModelCapability(
|
||||
model_id=settings.yolo_seg_model_id,
|
||||
@@ -186,9 +202,24 @@ class ModelRegistryService:
|
||||
elif not model_path.exists() or not model_path.is_file():
|
||||
limitation = "SAM_MODEL_PATH does not point to an existing local model file. GeoIntel will not download segmentation model weights automatically."
|
||||
else:
|
||||
configured = True
|
||||
status = "configured"
|
||||
limitation = "Configured for local class-agnostic SAM segmentation over an existing raster tile manifest."
|
||||
try:
|
||||
RuntimeModelProvenanceService.validate_for_runtime(
|
||||
model_path=model_path,
|
||||
model_id=settings.sam_model_id,
|
||||
task_type="segmentation",
|
||||
expected_model_version=settings.sam_model_version,
|
||||
allowed_frameworks=("ultralytics/sam", "sam", "ultralytics", "pytorch"),
|
||||
)
|
||||
except AppError as exc:
|
||||
status = "contract_incomplete"
|
||||
limitation = (
|
||||
"Configured SAM weights are not runnable until their immutable runtime provenance "
|
||||
f"sidecar validates: {exc.message}"
|
||||
)
|
||||
else:
|
||||
configured = True
|
||||
status = "configured"
|
||||
limitation = "Configured for local class-agnostic SAM segmentation over an existing raster tile manifest."
|
||||
|
||||
return DetectionModelCapability(
|
||||
model_id=settings.sam_model_id,
|
||||
@@ -233,9 +264,24 @@ class ModelRegistryService:
|
||||
status = "accelerator_unavailable"
|
||||
limitation = exc.message
|
||||
else:
|
||||
configured = True
|
||||
status = "configured"
|
||||
limitation = "Configured for local YOLO inference over an existing raster tile manifest within its validated area scope."
|
||||
try:
|
||||
RuntimeModelProvenanceService.validate_for_runtime(
|
||||
model_path=model_path,
|
||||
model_id=settings.yolo_model_id,
|
||||
task_type="object_detection",
|
||||
expected_model_version=settings.yolo_model_version,
|
||||
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
|
||||
)
|
||||
except AppError as exc:
|
||||
status = "contract_incomplete"
|
||||
limitation = (
|
||||
"Configured YOLO weights are not runnable until their immutable runtime provenance "
|
||||
f"sidecar validates: {exc.message}"
|
||||
)
|
||||
else:
|
||||
configured = True
|
||||
status = "configured"
|
||||
limitation = "Configured for local YOLO inference over an existing raster tile manifest within its validated area scope."
|
||||
|
||||
return DetectionModelCapability(
|
||||
model_id=settings.yolo_model_id,
|
||||
|
||||
@@ -11,10 +11,10 @@ from shapely.geometry.base import BaseGeometry
|
||||
from shapely.strtree import STRtree
|
||||
from shapely.ops import unary_union
|
||||
from shapely.validation import make_valid
|
||||
from shapely.geometry import shape
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset
|
||||
from app.schemas.qa import QaProviderComparisonResult
|
||||
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||
from app.services.vector_operations_service import VectorOperationsService
|
||||
|
||||
|
||||
@@ -264,6 +264,12 @@ class QaService:
|
||||
reference_dataset_id,
|
||||
expected_project_id=project_id,
|
||||
)
|
||||
DatasetConsumptionGate.assert_eligible(candidate_dataset, purpose="quality_assessment")
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
reference_dataset,
|
||||
purpose="reference_validation",
|
||||
reference_task="building_validation",
|
||||
)
|
||||
|
||||
area_geometry = QaService._validate_area(
|
||||
db,
|
||||
|
||||
@@ -13,6 +13,7 @@ from shapely.validation import make_valid
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Area, Dataset, DatasetVersion
|
||||
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
|
||||
from app.services.raster_service import extract_raster_metadata
|
||||
from app.services.storage_service import StorageService
|
||||
|
||||
@@ -334,7 +335,10 @@ class RasterOperationsService:
|
||||
dataset_type="raster",
|
||||
source=f"operation:{operation_name}",
|
||||
dataset_role="derived",
|
||||
source_name=source_dataset.source_name,
|
||||
# The source identity of this artifact is the server-owned
|
||||
# derived-operation registry entry. The parent remains explicit
|
||||
# in provenance and the lineage edge below.
|
||||
source_name="derived",
|
||||
source_metadata=source_dataset.source_metadata,
|
||||
provenance_metadata=provenance,
|
||||
imported_at=datetime.now(timezone.utc),
|
||||
@@ -360,22 +364,31 @@ class RasterOperationsService:
|
||||
resolution_json=metadata_payload.get("resolution"),
|
||||
bands_json={"dtype": metadata_payload.get("dtype")} if metadata_payload.get("dtype") is not None else None,
|
||||
metadata_json=metadata_payload,
|
||||
status="ready",
|
||||
status="validating",
|
||||
)
|
||||
db.add(derived_dataset)
|
||||
db.add(
|
||||
DatasetVersion(
|
||||
dataset_id=derived_dataset.id,
|
||||
version=1,
|
||||
storage_path=derived_dataset.storage_path,
|
||||
source_version=derived_dataset.source_version,
|
||||
observed_at=derived_dataset.observed_at,
|
||||
valid_from=derived_dataset.valid_from,
|
||||
valid_to=derived_dataset.valid_to,
|
||||
checksum_sha256=derived_dataset.checksum_sha256,
|
||||
source_metadata=derived_dataset.source_metadata,
|
||||
provenance_metadata=derived_dataset.provenance_metadata,
|
||||
)
|
||||
dataset_version = DatasetVersion(
|
||||
dataset_id=derived_dataset.id,
|
||||
version=1,
|
||||
storage_path=derived_dataset.storage_path,
|
||||
source_version=derived_dataset.source_version,
|
||||
observed_at=derived_dataset.observed_at,
|
||||
valid_from=derived_dataset.valid_from,
|
||||
valid_to=derived_dataset.valid_to,
|
||||
checksum_sha256=derived_dataset.checksum_sha256,
|
||||
source_metadata=derived_dataset.source_metadata,
|
||||
provenance_metadata=derived_dataset.provenance_metadata,
|
||||
)
|
||||
db.add(dataset_version)
|
||||
DerivedDatasetGovernanceService.govern_raster(
|
||||
db,
|
||||
dataset=derived_dataset,
|
||||
dataset_version=dataset_version,
|
||||
raster_metadata=metadata_payload,
|
||||
source_key="derived",
|
||||
operation=operation_name,
|
||||
parent_dataset=source_dataset,
|
||||
operation_parameters=metadata_payload.get("operation_parameters"),
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(derived_dataset)
|
||||
|
||||
@@ -0,0 +1,652 @@
|
||||
"""Fail-closed provenance validation for configured local model files.
|
||||
|
||||
Configured YOLO and SAM weights are intentionally not trusted merely because a
|
||||
file exists on the server. Before an adapter is allowed to load local weights,
|
||||
this service verifies a neighbouring immutable sidecar manifest, validates the
|
||||
model artifact against ``geointel.model.pytorch@1.0.0`` and binds that sidecar
|
||||
to the server-owned source registry and source snapshot recorded in Postgres.
|
||||
|
||||
``validate_for_runtime`` remains a structural sidecar check for catalog and
|
||||
preflight inspection. Production inference must call
|
||||
``validate_for_production_runtime`` with a database session; it rejects an
|
||||
unregistered, mismatched, stale, unsafe or quarantined source snapshot before
|
||||
an adapter can load model bytes. This keeps focused unit tests able to inspect
|
||||
sidecars without inventing database rows while keeping the production boundary
|
||||
strict.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timezone
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
from typing import Any, Mapping
|
||||
from uuid import UUID
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import SourceRegistry, SourceSnapshot
|
||||
from app.services.data_contract_validation import (
|
||||
PYTORCH_MODEL_CONTRACT_KEY,
|
||||
PYTORCH_MODEL_CONTRACT_VERSION,
|
||||
LineageEvidence,
|
||||
TransformationEvidence,
|
||||
build_model_validation_input,
|
||||
validate_registered_asset,
|
||||
)
|
||||
|
||||
|
||||
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||
_CONSUMABLE_FRESHNESS = {"current", "not_applicable"}
|
||||
_SAFE_SOURCE_REGISTRY_INGEST_STATUSES = {"configured", "ingested"}
|
||||
_SAFE_SOURCE_SNAPSHOT_INGEST_STATUS = "ingested"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RuntimeModelProvenance:
|
||||
"""Validated, immutable evidence attached to one local inference run."""
|
||||
|
||||
model_id: str
|
||||
task_type: str
|
||||
model_path: str
|
||||
manifest_path: str
|
||||
model_sha256: str
|
||||
manifest_sha256: str
|
||||
runtime_manifest_sha256: str
|
||||
data_contract_key: str
|
||||
data_contract_version: str
|
||||
validation_report_sha256: str
|
||||
source_registry_id: str
|
||||
source_snapshot_id: str
|
||||
source_snapshot_checksum_sha256: str
|
||||
source_version: str
|
||||
|
||||
def as_dict(self) -> dict[str, str]:
|
||||
return {
|
||||
"model_id": self.model_id,
|
||||
"task_type": self.task_type,
|
||||
"model_path": self.model_path,
|
||||
"manifest_path": self.manifest_path,
|
||||
"model_sha256": self.model_sha256,
|
||||
"manifest_sha256": self.manifest_sha256,
|
||||
"runtime_manifest_sha256": self.runtime_manifest_sha256,
|
||||
"data_contract_key": self.data_contract_key,
|
||||
"data_contract_version": self.data_contract_version,
|
||||
"validation_report_sha256": self.validation_report_sha256,
|
||||
"source_registry_id": self.source_registry_id,
|
||||
"source_snapshot_id": self.source_snapshot_id,
|
||||
"source_snapshot_checksum_sha256": self.source_snapshot_checksum_sha256,
|
||||
"source_version": self.source_version,
|
||||
}
|
||||
|
||||
|
||||
class RuntimeModelProvenanceService:
|
||||
"""Validate model sidecars and their production database binding.
|
||||
|
||||
A sidecar lives next to its model as ``<model-file>.geointel-model.json``.
|
||||
Its ``metadata.runtime_manifest_sha256`` is the SHA-256 of canonical JSON
|
||||
after omitting that one self-referential field. Any other mutation of the
|
||||
manifest therefore invalidates it. Structural validation is deliberately
|
||||
separate from :meth:`validate_for_production_runtime`: discovery and
|
||||
preflight have no database session, whereas every production inference
|
||||
path must prove an active source registry/snapshot binding.
|
||||
"""
|
||||
|
||||
MANIFEST_SUFFIX = ".geointel-model.json"
|
||||
MANIFEST_SCHEMA_VERSION = "geointel.runtime-model-manifest/v1"
|
||||
SOURCE_REGISTRY_KEY = "model"
|
||||
|
||||
@classmethod
|
||||
def manifest_path_for_model(cls, model_path: str | Path) -> Path:
|
||||
path = Path(model_path).expanduser()
|
||||
return Path(f"{path}{cls.MANIFEST_SUFFIX}")
|
||||
|
||||
@staticmethod
|
||||
def manifest_self_checksum(payload: Mapping[str, Any]) -> str:
|
||||
"""Hash sidecar semantics without its self-referential checksum field."""
|
||||
|
||||
canonical_payload = json.loads(json.dumps(payload, sort_keys=True, ensure_ascii=True))
|
||||
metadata = canonical_payload.get("metadata")
|
||||
if isinstance(metadata, dict):
|
||||
metadata.pop("runtime_manifest_sha256", None)
|
||||
encoded = json.dumps(
|
||||
canonical_payload,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
ensure_ascii=True,
|
||||
).encode("utf-8")
|
||||
return sha256(encoded).hexdigest()
|
||||
|
||||
@classmethod
|
||||
def validate_for_runtime(
|
||||
cls,
|
||||
*,
|
||||
model_path: str | Path,
|
||||
model_id: str,
|
||||
task_type: str,
|
||||
expected_model_version: str | None = None,
|
||||
allowed_frameworks: tuple[str, ...] = (),
|
||||
) -> RuntimeModelProvenance:
|
||||
"""Return structural sidecar evidence without asserting database state.
|
||||
|
||||
This method is appropriate for read-only catalog/preflight checks and
|
||||
focused sidecar unit tests. It is insufficient for production
|
||||
inference; adapters must use :meth:`validate_for_production_runtime`.
|
||||
"""
|
||||
|
||||
path = Path(model_path).expanduser()
|
||||
if not path.exists() or not path.is_file():
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MODEL_FILE_MISSING",
|
||||
"Configured model file is missing; runtime provenance cannot be verified.",
|
||||
model_path=str(path),
|
||||
)
|
||||
manifest_path = cls.manifest_path_for_model(path)
|
||||
if not manifest_path.exists() or not manifest_path.is_file():
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_MISSING",
|
||||
"Configured model requires an immutable .geointel-model.json sidecar before inference.",
|
||||
model_path=str(path),
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
|
||||
try:
|
||||
raw_manifest = manifest_path.read_bytes()
|
||||
payload = json.loads(raw_manifest.decode("utf-8"))
|
||||
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
"Configured model sidecar must be a readable UTF-8 JSON object.",
|
||||
manifest_path=str(manifest_path),
|
||||
error_type=type(exc).__name__,
|
||||
)
|
||||
if not isinstance(payload, dict):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
"Configured model sidecar must contain a JSON object.",
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
|
||||
cls._require_exact_text(
|
||||
payload.get("schema_version"),
|
||||
cls.MANIFEST_SCHEMA_VERSION,
|
||||
field="schema_version",
|
||||
manifest_path=manifest_path,
|
||||
)
|
||||
contract = cls._require_mapping(payload, "data_contract", manifest_path)
|
||||
contract_key = cls._require_text(contract, "key", manifest_path)
|
||||
contract_version = cls._require_text(contract, "version", manifest_path)
|
||||
# A valid manifest for a different artifact family must never make a
|
||||
# local PyTorch/SAM weight executable. The structural validator below
|
||||
# has a registry lookup too, but pinning the identity here keeps this
|
||||
# runtime gate fail-closed if more model contracts are introduced.
|
||||
cls._require_exact_text(
|
||||
contract_key,
|
||||
PYTORCH_MODEL_CONTRACT_KEY,
|
||||
field="data_contract.key",
|
||||
manifest_path=manifest_path,
|
||||
)
|
||||
cls._require_exact_text(
|
||||
contract_version,
|
||||
PYTORCH_MODEL_CONTRACT_VERSION,
|
||||
field="data_contract.version",
|
||||
manifest_path=manifest_path,
|
||||
)
|
||||
model = cls._require_mapping(payload, "model", manifest_path)
|
||||
declared_model_id = cls._require_text(model, "model_id", manifest_path)
|
||||
declared_task_type = cls._require_text(model, "task_type", manifest_path)
|
||||
cls._require_exact_text(declared_model_id, model_id, field="model.model_id", manifest_path=manifest_path)
|
||||
cls._require_exact_text(declared_task_type, task_type, field="model.task_type", manifest_path=manifest_path)
|
||||
|
||||
declared_checksum = cls._require_checksum(model.get("sha256"), "model.sha256", manifest_path)
|
||||
model_checksum = cls._file_sha256(path)
|
||||
if declared_checksum != model_checksum:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MODEL_CHECKSUM_MISMATCH",
|
||||
"Model bytes do not match the checksum bound by the runtime sidecar.",
|
||||
model_path=str(path),
|
||||
expected=declared_checksum,
|
||||
observed=model_checksum,
|
||||
)
|
||||
|
||||
model_format = cls._require_text(model, "model_format", manifest_path)
|
||||
framework = cls._require_text(model, "framework", manifest_path)
|
||||
class_mapping = model.get("class_mapping")
|
||||
if not isinstance(class_mapping, (dict, list, tuple)) or not class_mapping:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
"model.class_mapping must be a non-empty mapping or sequence.",
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
normalized_framework = framework.strip().lower()
|
||||
if allowed_frameworks and normalized_framework not in {value.strip().lower() for value in allowed_frameworks}:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_FRAMEWORK_MISMATCH",
|
||||
"Model framework does not match the configured runtime adapter.",
|
||||
manifest_path=str(manifest_path),
|
||||
expected=sorted({value.strip().lower() for value in allowed_frameworks}),
|
||||
observed=framework,
|
||||
)
|
||||
source_version = cls._require_text(model, "source_version", manifest_path)
|
||||
if expected_model_version and source_version != expected_model_version:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_VERSION_MISMATCH",
|
||||
"Model sidecar version does not match the configured model version.",
|
||||
manifest_path=str(manifest_path),
|
||||
expected=expected_model_version,
|
||||
observed=source_version,
|
||||
)
|
||||
|
||||
source = cls._require_mapping(payload, "source", manifest_path)
|
||||
source_registry_id = cls._require_uuid(source.get("source_registry_id"), "source.source_registry_id", manifest_path)
|
||||
source_snapshot_id = cls._require_uuid(source.get("source_snapshot_id"), "source.source_snapshot_id", manifest_path)
|
||||
cls._require_exact_text(
|
||||
cls._require_text(source, "source_registry_key", manifest_path),
|
||||
cls.SOURCE_REGISTRY_KEY,
|
||||
field="source.source_registry_key",
|
||||
manifest_path=manifest_path,
|
||||
)
|
||||
snapshot_checksum = cls._require_checksum(
|
||||
source.get("source_snapshot_checksum_sha256"),
|
||||
"source.source_snapshot_checksum_sha256",
|
||||
manifest_path,
|
||||
)
|
||||
if snapshot_checksum != model_checksum:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_SNAPSHOT_CHECKSUM_MISMATCH",
|
||||
"Model source snapshot checksum must bind the exact model bytes.",
|
||||
manifest_path=str(manifest_path),
|
||||
expected=model_checksum,
|
||||
observed=snapshot_checksum,
|
||||
)
|
||||
|
||||
metadata = cls._require_mapping(payload, "metadata", manifest_path)
|
||||
training_manifest_sha256 = cls._require_checksum(
|
||||
metadata.get("training_manifest_sha256"),
|
||||
"metadata.training_manifest_sha256",
|
||||
manifest_path,
|
||||
)
|
||||
declared_runtime_manifest_sha256 = cls._require_checksum(
|
||||
metadata.get("runtime_manifest_sha256"),
|
||||
"metadata.runtime_manifest_sha256",
|
||||
manifest_path,
|
||||
)
|
||||
computed_runtime_manifest_sha256 = cls.manifest_self_checksum(payload)
|
||||
if declared_runtime_manifest_sha256 != computed_runtime_manifest_sha256:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_CHECKSUM_MISMATCH",
|
||||
"Runtime model sidecar integrity checksum does not match its canonical contents.",
|
||||
manifest_path=str(manifest_path),
|
||||
expected=declared_runtime_manifest_sha256,
|
||||
observed=computed_runtime_manifest_sha256,
|
||||
)
|
||||
|
||||
lineage = cls._lineage_evidence(payload, manifest_path)
|
||||
imported_at = cls._parse_imported_at(payload.get("imported_at"), manifest_path)
|
||||
report = validate_registered_asset(
|
||||
build_model_validation_input(
|
||||
asset_id=f"{model_id}:{model_checksum}",
|
||||
model_metadata={
|
||||
"model_format": model_format,
|
||||
"framework": framework,
|
||||
"class_mapping": class_mapping,
|
||||
},
|
||||
checksum_sha256=declared_checksum,
|
||||
computed_checksum_sha256=model_checksum,
|
||||
source_registry_id=source_registry_id,
|
||||
source_snapshot_id=source_snapshot_id,
|
||||
imported_at=imported_at,
|
||||
metadata={
|
||||
"training_manifest_sha256": training_manifest_sha256,
|
||||
"runtime_manifest_sha256": declared_runtime_manifest_sha256,
|
||||
},
|
||||
source_version=source_version,
|
||||
lineage=lineage,
|
||||
data_contract_key=contract_key,
|
||||
data_contract_version=contract_version,
|
||||
)
|
||||
)
|
||||
if report.failed:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_CONTRACT_FAILED",
|
||||
"Configured model sidecar failed the exact versioned model data contract.",
|
||||
manifest_path=str(manifest_path),
|
||||
data_contract=f"{contract_key}@{contract_version}",
|
||||
issue_codes=[issue.code for issue in report.issues],
|
||||
validation_report_sha256=report.report_sha256,
|
||||
)
|
||||
return RuntimeModelProvenance(
|
||||
model_id=model_id,
|
||||
task_type=task_type,
|
||||
model_path=str(path.resolve()),
|
||||
manifest_path=str(manifest_path.resolve()),
|
||||
model_sha256=model_checksum,
|
||||
manifest_sha256=sha256(raw_manifest).hexdigest(),
|
||||
runtime_manifest_sha256=declared_runtime_manifest_sha256,
|
||||
data_contract_key=contract_key,
|
||||
data_contract_version=contract_version,
|
||||
validation_report_sha256=report.report_sha256,
|
||||
source_registry_id=source_registry_id,
|
||||
source_snapshot_id=source_snapshot_id,
|
||||
source_snapshot_checksum_sha256=snapshot_checksum,
|
||||
source_version=source_version,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def validate_for_production_runtime(
|
||||
cls,
|
||||
*,
|
||||
db: Any,
|
||||
model_path: str | Path,
|
||||
model_id: str,
|
||||
task_type: str,
|
||||
expected_model_version: str | None = None,
|
||||
allowed_frameworks: tuple[str, ...] = (),
|
||||
) -> RuntimeModelProvenance:
|
||||
"""Validate byte-bound model evidence against governed database state.
|
||||
|
||||
The sidecar is not itself a source of authority. The model bytes may
|
||||
enter production inference only when the declared source registry and
|
||||
immutable source snapshot both exist, belong together, are safe to
|
||||
consume and bind the same SHA-256 and source version as the sidecar.
|
||||
This check is intentionally invoked immediately before adapter loading.
|
||||
"""
|
||||
|
||||
if db is None or not callable(getattr(db, "get", None)):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_DATABASE_REQUIRED",
|
||||
"Production model inference requires a database session for source snapshot provenance.",
|
||||
)
|
||||
|
||||
evidence = cls.validate_for_runtime(
|
||||
model_path=model_path,
|
||||
model_id=model_id,
|
||||
task_type=task_type,
|
||||
expected_model_version=expected_model_version,
|
||||
allowed_frameworks=allowed_frameworks,
|
||||
)
|
||||
cls._assert_database_binding(db, evidence)
|
||||
return evidence
|
||||
|
||||
@classmethod
|
||||
def _assert_database_binding(cls, db: Any, evidence: RuntimeModelProvenance) -> None:
|
||||
registry_id = UUID(evidence.source_registry_id)
|
||||
snapshot_id = UUID(evidence.source_snapshot_id)
|
||||
source_registry = db.get(SourceRegistry, registry_id)
|
||||
if not isinstance(source_registry, SourceRegistry):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND",
|
||||
"Configured model sidecar refers to a source registry record that does not exist.",
|
||||
source_registry_id=evidence.source_registry_id,
|
||||
model_id=evidence.model_id,
|
||||
)
|
||||
source_snapshot = db.get(SourceSnapshot, snapshot_id)
|
||||
if not isinstance(source_snapshot, SourceSnapshot):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_NOT_FOUND",
|
||||
"Configured model sidecar refers to a source snapshot record that does not exist.",
|
||||
source_snapshot_id=evidence.source_snapshot_id,
|
||||
model_id=evidence.model_id,
|
||||
)
|
||||
|
||||
if str(source_registry.id) != evidence.source_registry_id or source_registry.source_key != cls.SOURCE_REGISTRY_KEY:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_SOURCE_REGISTRY_IDENTITY_MISMATCH",
|
||||
"Database source registry does not match the immutable model sidecar identity.",
|
||||
expected_source_registry_id=evidence.source_registry_id,
|
||||
observed_source_registry_id=str(source_registry.id),
|
||||
expected_source_key=cls.SOURCE_REGISTRY_KEY,
|
||||
observed_source_key=source_registry.source_key,
|
||||
)
|
||||
if str(source_snapshot.id) != evidence.source_snapshot_id or str(source_snapshot.source_registry_id) != evidence.source_registry_id:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_REGISTRY_MISMATCH",
|
||||
"Model source snapshot does not belong to the declared source registry.",
|
||||
source_registry_id=evidence.source_registry_id,
|
||||
source_snapshot_id=evidence.source_snapshot_id,
|
||||
observed_snapshot_registry_id=str(source_snapshot.source_registry_id),
|
||||
)
|
||||
|
||||
registry_ingest_status = cls._normalise_status(source_registry.ingest_status)
|
||||
registry_freshness_status = cls._normalise_status(source_registry.freshness_status)
|
||||
if (
|
||||
registry_ingest_status not in _SAFE_SOURCE_REGISTRY_INGEST_STATUSES
|
||||
or registry_freshness_status not in _CONSUMABLE_FRESHNESS
|
||||
):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_SOURCE_REGISTRY_UNSAFE",
|
||||
"Configured model source registry is not in a safe configured/current state.",
|
||||
source_registry_id=evidence.source_registry_id,
|
||||
ingest_status=registry_ingest_status,
|
||||
freshness_status=registry_freshness_status,
|
||||
)
|
||||
|
||||
snapshot_ingest_status = cls._normalise_status(source_snapshot.ingest_status)
|
||||
snapshot_freshness_status = cls._normalise_status(source_snapshot.freshness_status)
|
||||
if (
|
||||
snapshot_ingest_status != _SAFE_SOURCE_SNAPSHOT_INGEST_STATUS
|
||||
or snapshot_freshness_status not in _CONSUMABLE_FRESHNESS
|
||||
):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_UNSAFE",
|
||||
"Configured model source snapshot is not an ingested current immutable artifact.",
|
||||
source_snapshot_id=evidence.source_snapshot_id,
|
||||
ingest_status=snapshot_ingest_status,
|
||||
freshness_status=snapshot_freshness_status,
|
||||
)
|
||||
|
||||
snapshot_checksum = str(source_snapshot.checksum_sha256 or "").strip().lower()
|
||||
if snapshot_checksum != evidence.source_snapshot_checksum_sha256 or snapshot_checksum != evidence.model_sha256:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_DATABASE_SNAPSHOT_CHECKSUM_MISMATCH",
|
||||
"Database model source snapshot checksum does not bind the exact sidecar and model bytes.",
|
||||
source_snapshot_id=evidence.source_snapshot_id,
|
||||
expected_model_sha256=evidence.model_sha256,
|
||||
expected_sidecar_snapshot_sha256=evidence.source_snapshot_checksum_sha256,
|
||||
observed_snapshot_sha256=snapshot_checksum,
|
||||
)
|
||||
|
||||
snapshot_version = str(source_snapshot.source_version or "").strip()
|
||||
if snapshot_version != evidence.source_version:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_VERSION_MISMATCH",
|
||||
"Database model source snapshot version does not match the immutable sidecar model version.",
|
||||
source_snapshot_id=evidence.source_snapshot_id,
|
||||
expected_source_version=evidence.source_version,
|
||||
observed_source_version=snapshot_version or None,
|
||||
)
|
||||
|
||||
# The persisted snapshot state is the primary quarantine signal. The
|
||||
# relationship check is a defensive second line for an active
|
||||
# quarantine record that predates or bypassed a status transition.
|
||||
active_quarantines = getattr(source_snapshot, "quarantines", ())
|
||||
if any(cls._normalise_status(getattr(item, "status", None)) == "quarantined" for item in active_quarantines):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_QUARANTINED",
|
||||
"Configured model source snapshot has an active quarantine record.",
|
||||
source_snapshot_id=evidence.source_snapshot_id,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _normalise_status(value: Any) -> str:
|
||||
return str(value or "").strip().lower()
|
||||
|
||||
@classmethod
|
||||
def _lineage_evidence(cls, payload: Mapping[str, Any], manifest_path: Path) -> LineageEvidence:
|
||||
lineage = cls._require_mapping(payload, "lineage", manifest_path)
|
||||
raw_asset_ids = lineage.get("upstream_asset_ids")
|
||||
raw_checksums = lineage.get("upstream_checksums_sha256")
|
||||
raw_transformations = lineage.get("transformations")
|
||||
if not isinstance(raw_asset_ids, list) or not raw_asset_ids or not all(
|
||||
isinstance(value, str) and value.strip() for value in raw_asset_ids
|
||||
):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
"lineage.upstream_asset_ids must be a non-empty string list.",
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
if not isinstance(raw_checksums, list) or len(raw_checksums) != len(raw_asset_ids):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
"lineage.upstream_checksums_sha256 must match upstream_asset_ids one-for-one.",
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
upstream_checksums = tuple(
|
||||
cls._require_checksum(value, f"lineage.upstream_checksums_sha256[{index}]", manifest_path)
|
||||
for index, value in enumerate(raw_checksums)
|
||||
)
|
||||
if not isinstance(raw_transformations, list) or not raw_transformations:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
"lineage.transformations must contain at least one immutable transformation record.",
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
transformations: list[TransformationEvidence] = []
|
||||
for index, raw in enumerate(raw_transformations):
|
||||
if not isinstance(raw, Mapping):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
"Each lineage transformation must be an object.",
|
||||
manifest_path=str(manifest_path),
|
||||
index=index,
|
||||
)
|
||||
transformations.append(
|
||||
TransformationEvidence(
|
||||
name=cls._require_text(raw, "name", manifest_path, prefix=f"lineage.transformations[{index}]."),
|
||||
version=cls._require_text(raw, "version", manifest_path, prefix=f"lineage.transformations[{index}]."),
|
||||
checksum_sha256=cls._require_checksum(
|
||||
raw.get("checksum_sha256"),
|
||||
f"lineage.transformations[{index}].checksum_sha256",
|
||||
manifest_path,
|
||||
),
|
||||
)
|
||||
)
|
||||
return LineageEvidence(
|
||||
upstream_asset_ids=tuple(raw_asset_ids),
|
||||
upstream_checksums_sha256=upstream_checksums,
|
||||
transformations=tuple(transformations),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _file_sha256(path: Path) -> str:
|
||||
digest = sha256()
|
||||
try:
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
except OSError as exc:
|
||||
RuntimeModelProvenanceService._raise(
|
||||
"MODEL_PROVENANCE_MODEL_FILE_UNREADABLE",
|
||||
"Configured model file could not be read for checksum validation.",
|
||||
model_path=str(path),
|
||||
error_type=type(exc).__name__,
|
||||
)
|
||||
return digest.hexdigest()
|
||||
|
||||
@classmethod
|
||||
def _parse_imported_at(cls, value: Any, manifest_path: Path) -> datetime:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
"imported_at must be a timezone-aware ISO-8601 timestamp.",
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
"imported_at must be a timezone-aware ISO-8601 timestamp.",
|
||||
manifest_path=str(manifest_path),
|
||||
observed=value,
|
||||
)
|
||||
if parsed.tzinfo is None:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
"imported_at must include a timezone offset.",
|
||||
manifest_path=str(manifest_path),
|
||||
observed=value,
|
||||
)
|
||||
return parsed.astimezone(timezone.utc)
|
||||
|
||||
@classmethod
|
||||
def _require_mapping(cls, payload: Mapping[str, Any], key: str, manifest_path: Path) -> Mapping[str, Any]:
|
||||
value = payload.get(key)
|
||||
if not isinstance(value, Mapping):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
f"{key} must be a JSON object.",
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
return value
|
||||
|
||||
@classmethod
|
||||
def _require_text(
|
||||
cls,
|
||||
payload: Mapping[str, Any],
|
||||
key: str,
|
||||
manifest_path: Path,
|
||||
*,
|
||||
prefix: str = "",
|
||||
) -> str:
|
||||
value = payload.get(key)
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
f"{prefix}{key} must be a non-empty string.",
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
return value.strip()
|
||||
|
||||
@classmethod
|
||||
def _require_checksum(cls, value: Any, field: str, manifest_path: Path) -> str:
|
||||
normalized = value.strip().lower() if isinstance(value, str) else ""
|
||||
if not _SHA256.fullmatch(normalized) or value != normalized:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
f"{field} must be a lowercase SHA-256 digest.",
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
return normalized
|
||||
|
||||
@classmethod
|
||||
def _require_uuid(cls, value: Any, field: str, manifest_path: Path) -> str:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
f"{field} must be a UUID string.",
|
||||
manifest_path=str(manifest_path),
|
||||
)
|
||||
try:
|
||||
return str(UUID(value))
|
||||
except (AttributeError, ValueError):
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
f"{field} must be a UUID string.",
|
||||
manifest_path=str(manifest_path),
|
||||
observed=value,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _require_exact_text(
|
||||
cls,
|
||||
observed: Any,
|
||||
expected: str,
|
||||
*,
|
||||
field: str,
|
||||
manifest_path: Path,
|
||||
) -> None:
|
||||
if not isinstance(observed, str) or observed.strip() != expected:
|
||||
cls._raise(
|
||||
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||
f"{field} does not match the configured runtime identity.",
|
||||
manifest_path=str(manifest_path),
|
||||
expected=expected,
|
||||
observed=observed,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _raise(code: str, message: str, **details: Any) -> None:
|
||||
raise AppError(code=code, message=message, details=details, status_code=422)
|
||||
@@ -21,9 +21,11 @@ from app.schemas.segmentation import (
|
||||
)
|
||||
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||
from app.services.model_registry_service import ModelRegistryService
|
||||
from app.services.qa_service import QaService
|
||||
from app.services.quality_service import QualityService
|
||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenance, RuntimeModelProvenanceService
|
||||
from app.services.segmentation_adapter import (
|
||||
FixtureSegmentationAdapter,
|
||||
SamSegmentationAdapter,
|
||||
@@ -89,6 +91,18 @@ class SegmentationService:
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
# Production segmentation must consume only a passed, complete and
|
||||
# non-quarantined dataset. The fixture segmenter is QA/test-only and
|
||||
# cannot be classified as production inference.
|
||||
if model.model_id == "fixture-segmenter":
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
dataset,
|
||||
purpose="quality_assessment",
|
||||
fixture_mode=True,
|
||||
)
|
||||
elif model.model_id in configured_model_ids and model.configured:
|
||||
DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
|
||||
|
||||
run_parameters = {
|
||||
"model_id": model.model_id,
|
||||
"confidence_threshold": confidence_threshold,
|
||||
@@ -326,6 +340,27 @@ class SegmentationService:
|
||||
if reference_dataset.dataset_type not in {"vector", "geojson"}:
|
||||
raise AppError(code="INVALID_DATASET_TYPE", message="Reference dataset must be vector data", status_code=400)
|
||||
|
||||
candidate_dataset = db.get(Dataset, run.dataset_id)
|
||||
if not candidate_dataset:
|
||||
raise AppError(code="DATASET_NOT_FOUND", message="Segmentation source dataset not found", status_code=404)
|
||||
run_parameters = run.parameters_json if isinstance(run.parameters_json, dict) else {}
|
||||
fixture_parameters = run_parameters.get("parameters_json")
|
||||
fixture_mode = bool(
|
||||
run.model_name == "fixture-segmenter"
|
||||
and isinstance(fixture_parameters, dict)
|
||||
and fixture_parameters.get("fixture_mode") is True
|
||||
)
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
candidate_dataset,
|
||||
purpose="quality_assessment",
|
||||
fixture_mode=fixture_mode,
|
||||
)
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
reference_dataset,
|
||||
purpose="reference_validation",
|
||||
reference_task="building_validation",
|
||||
)
|
||||
|
||||
segmentations = SegmentationService._query_segmentation_rows(
|
||||
db,
|
||||
analysis_run_id=analysis_run_id,
|
||||
@@ -510,11 +545,26 @@ class SegmentationService:
|
||||
) -> tuple[list[Segmentation], dict[str, Any]]:
|
||||
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles)
|
||||
if model_name == settings.sam_model_id:
|
||||
adapter = sam_adapter_class(settings)
|
||||
model_path = Path(settings.sam_model_path or "").expanduser()
|
||||
allowed_frameworks = ("ultralytics/sam", "sam", "ultralytics", "pytorch")
|
||||
adapter = sam_adapter_class(settings)
|
||||
else:
|
||||
adapter = yolo_seg_adapter_class(settings)
|
||||
model_path = Path(settings.yolo_seg_model_path or "").expanduser()
|
||||
allowed_frameworks = ("ultralytics/pytorch", "ultralytics", "pytorch")
|
||||
adapter = yolo_seg_adapter_class(settings)
|
||||
runtime_model_provenance = RuntimeModelProvenanceService.validate_for_production_runtime(
|
||||
db=db,
|
||||
model_path=model_path,
|
||||
model_id=model_name,
|
||||
task_type="segmentation",
|
||||
expected_model_version=model_version,
|
||||
allowed_frameworks=allowed_frameworks,
|
||||
)
|
||||
SegmentationService._attach_runtime_model_provenance(
|
||||
analysis_run,
|
||||
job,
|
||||
runtime_model_provenance,
|
||||
)
|
||||
model = adapter.load_model(model_path)
|
||||
|
||||
allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)}
|
||||
@@ -591,6 +641,7 @@ class SegmentationService:
|
||||
"tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()),
|
||||
"tile_index": candidate["tile_index"],
|
||||
"device": settings.yolo_device,
|
||||
"runtime_model_provenance": runtime_model_provenance.as_dict(),
|
||||
},
|
||||
)
|
||||
db.add(segmentation)
|
||||
@@ -603,8 +654,25 @@ class SegmentationService:
|
||||
"suppressed_segmentation_count": len(candidates) - len(filtered_candidates),
|
||||
"duplicate_iou_threshold": float(settings.segmentation_duplicate_iou_threshold),
|
||||
"tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()),
|
||||
"runtime_model_provenance": runtime_model_provenance.as_dict(),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _attach_runtime_model_provenance(
|
||||
analysis_run: AnalysisRun,
|
||||
job: Job,
|
||||
provenance: RuntimeModelProvenance,
|
||||
) -> None:
|
||||
"""Record immutable model evidence with a configured segmentation run."""
|
||||
|
||||
evidence = provenance.as_dict()
|
||||
analysis_parameters = dict(analysis_run.parameters_json or {})
|
||||
analysis_parameters["runtime_model_provenance"] = evidence
|
||||
analysis_run.parameters_json = analysis_parameters
|
||||
job_parameters = dict(job.parameters_json or {})
|
||||
job_parameters["runtime_model_provenance"] = evidence
|
||||
job.parameters_json = job_parameters
|
||||
|
||||
@staticmethod
|
||||
def _geodesic_area_m2(geometry: MultiPolygon | Polygon) -> float | None:
|
||||
try:
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterable
|
||||
from uuid import UUID
|
||||
@@ -8,6 +9,7 @@ from uuid import UUID
|
||||
from geoalchemy2.functions import ST_Intersects, ST_MakeEnvelope
|
||||
from geoalchemy2.shape import from_shape
|
||||
from geoalchemy2.shape import to_shape
|
||||
from pyproj import CRS, Transformer
|
||||
from shapely.geometry import box, mapping, shape
|
||||
from shapely.ops import transform as transform_geometry
|
||||
from shapely.validation import make_valid
|
||||
@@ -268,7 +270,14 @@ class VectorFeatureService:
|
||||
return ("municipality", municipality) if municipality else None
|
||||
|
||||
@staticmethod
|
||||
def _feature_row(dataset_id: UUID, feature: dict[str, Any], index: int, feature_class: str | None) -> VectorFeature | None:
|
||||
def _feature_row(
|
||||
dataset_id: UUID,
|
||||
feature: dict[str, Any],
|
||||
index: int,
|
||||
feature_class: str | None,
|
||||
*,
|
||||
source_crs: str = "EPSG:4326",
|
||||
) -> VectorFeature | None:
|
||||
geometry_payload = feature.get("geometry")
|
||||
if geometry_payload is None:
|
||||
return None
|
||||
@@ -282,8 +291,7 @@ class VectorFeatureService:
|
||||
geometry = make_valid(geometry)
|
||||
if geometry.is_empty or not geometry.is_valid:
|
||||
raise AppError(code="INVALID_GEOMETRY", message=f"Invalid feature geometry at index {index}", status_code=400)
|
||||
if geometry.has_z:
|
||||
geometry = transform_geometry(lambda x, y, z=None: (x, y), geometry)
|
||||
geometry = VectorFeatureService._canonical_geometry(geometry, source_crs=source_crs, index=index)
|
||||
|
||||
properties = feature.get("properties") if isinstance(feature.get("properties"), dict) else {}
|
||||
source_feature_id = feature.get("id")
|
||||
@@ -298,6 +306,109 @@ class VectorFeatureService:
|
||||
geometry=from_shape(geometry, srid=4326),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _canonical_geometry(geometry: Any, *, source_crs: str, index: int):
|
||||
"""Transform one source geometry to canonical EPSG:4326 safely."""
|
||||
if geometry.has_z:
|
||||
geometry = transform_geometry(lambda x, y, z=None: (x, y), geometry)
|
||||
|
||||
# VectorFeature is deliberately canonical WGS84 storage. Treating
|
||||
# Lambert or another source CRS as EPSG:4326 produces geometries that
|
||||
# look syntactically valid but are spatially wrong. All governed
|
||||
# import callers therefore pass the declared source CRS; the default
|
||||
# only preserves compatibility for legacy, already-WGS84 call sites.
|
||||
try:
|
||||
parsed_source_crs = CRS.from_user_input(source_crs)
|
||||
target_crs = CRS.from_epsg(4326)
|
||||
except Exception as exc:
|
||||
raise AppError(
|
||||
code="INVALID_DATASET_CRS",
|
||||
message=f"Invalid source CRS for vector feature at index {index}",
|
||||
details={"source_crs": source_crs},
|
||||
status_code=400,
|
||||
) from exc
|
||||
if not parsed_source_crs.equals(target_crs):
|
||||
try:
|
||||
transformer = Transformer.from_crs(parsed_source_crs, target_crs, always_xy=True)
|
||||
geometry = transform_geometry(transformer.transform, geometry)
|
||||
except Exception as exc:
|
||||
raise AppError(
|
||||
code="VECTOR_CRS_TRANSFORMATION_FAILED",
|
||||
message=f"Could not transform vector feature at index {index} to EPSG:4326",
|
||||
details={"source_crs": source_crs},
|
||||
status_code=400,
|
||||
) from exc
|
||||
if geometry.is_empty or not geometry.is_valid:
|
||||
geometry = make_valid(geometry)
|
||||
if geometry.is_empty or not geometry.is_valid:
|
||||
raise AppError(
|
||||
code="INVALID_GEOMETRY",
|
||||
message=f"Invalid transformed feature geometry at index {index}",
|
||||
status_code=400,
|
||||
)
|
||||
min_x, min_y, max_x, max_y = geometry.bounds
|
||||
if (
|
||||
not all(math.isfinite(value) for value in (min_x, min_y, max_x, max_y))
|
||||
or min_x < -180
|
||||
or max_x > 180
|
||||
or min_y < -90
|
||||
or max_y > 90
|
||||
):
|
||||
raise AppError(
|
||||
code="VECTOR_GEOMETRY_OUTSIDE_EPSG4326",
|
||||
message=f"Transformed feature geometry at index {index} is outside EPSG:4326 bounds",
|
||||
details={"source_crs": source_crs, "bounds": [min_x, min_y, max_x, max_y]},
|
||||
status_code=400,
|
||||
)
|
||||
return geometry
|
||||
|
||||
@staticmethod
|
||||
def canonicalize_geojson_payload(payload: dict[str, Any], *, source_crs: str) -> dict[str, Any]:
|
||||
"""Return a canonical-WGS84 feature collection without losing source attributes.
|
||||
|
||||
Callers use this payload for validation, spatial indexing and
|
||||
map-safe consumption storage. A non-canonical source file, when
|
||||
retained, belongs to explicit provenance evidence rather than the
|
||||
Dataset consumption path; no implicit CRS assumption is recorded.
|
||||
"""
|
||||
features = payload.get("features")
|
||||
if payload.get("type") != "FeatureCollection" or not isinstance(features, list):
|
||||
raise AppError(code="INVALID_GEOJSON", message="GeoJSON payload must be a FeatureCollection", status_code=400)
|
||||
canonical_features: list[dict[str, Any]] = []
|
||||
for index, feature in enumerate(features):
|
||||
if not isinstance(feature, dict):
|
||||
raise AppError(code="INVALID_GEOJSON", message=f"Feature {index} must be an object", status_code=400)
|
||||
canonical_feature = dict(feature)
|
||||
geometry_payload = feature.get("geometry")
|
||||
if geometry_payload is not None:
|
||||
try:
|
||||
geometry = shape(geometry_payload)
|
||||
except Exception as exc:
|
||||
raise AppError(
|
||||
code="INVALID_GEOJSON",
|
||||
message=f"Invalid feature geometry at index {index}",
|
||||
status_code=400,
|
||||
) from exc
|
||||
if not geometry.is_empty:
|
||||
if not geometry.is_valid:
|
||||
geometry = make_valid(geometry)
|
||||
if geometry.is_empty or not geometry.is_valid:
|
||||
raise AppError(
|
||||
code="INVALID_GEOMETRY",
|
||||
message=f"Invalid feature geometry at index {index}",
|
||||
status_code=400,
|
||||
)
|
||||
canonical_feature["geometry"] = mapping(
|
||||
VectorFeatureService._canonical_geometry(geometry, source_crs=source_crs, index=index)
|
||||
)
|
||||
canonical_features.append(canonical_feature)
|
||||
return {
|
||||
**{key: value for key, value in payload.items() if key not in {"crs", "features"}},
|
||||
"type": "FeatureCollection",
|
||||
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
|
||||
"features": canonical_features,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _normalize_selection_bbox(bbox: dict[str, Any]) -> dict[str, float | str]:
|
||||
try:
|
||||
@@ -882,6 +993,7 @@ class VectorFeatureService:
|
||||
feature_class: str | None = None,
|
||||
*,
|
||||
commit: bool = True,
|
||||
source_crs: str = "EPSG:4326",
|
||||
) -> list[VectorFeature]:
|
||||
features = payload.get("features")
|
||||
if payload.get("type") != "FeatureCollection" or not isinstance(features, list):
|
||||
@@ -891,7 +1003,13 @@ class VectorFeatureService:
|
||||
for index, feature in enumerate(features):
|
||||
if not isinstance(feature, dict):
|
||||
raise AppError(code="INVALID_GEOJSON", message=f"Feature {index} must be an object", status_code=400)
|
||||
row = VectorFeatureService._feature_row(dataset_id, feature, index, feature_class)
|
||||
row = VectorFeatureService._feature_row(
|
||||
dataset_id,
|
||||
feature,
|
||||
index,
|
||||
feature_class,
|
||||
source_crs=source_crs,
|
||||
)
|
||||
if row is None:
|
||||
continue
|
||||
db.add(row)
|
||||
@@ -910,6 +1028,7 @@ class VectorFeatureService:
|
||||
feature_class: str | None = None,
|
||||
*,
|
||||
batch_size: int = 1000,
|
||||
source_crs: str = "EPSG:4326",
|
||||
) -> int:
|
||||
if batch_size <= 0:
|
||||
raise ValueError("batch_size must be positive")
|
||||
@@ -942,7 +1061,13 @@ class VectorFeatureService:
|
||||
message=f"Feature {index} in {path.name} must be an object",
|
||||
status_code=400,
|
||||
)
|
||||
row = VectorFeatureService._feature_row(dataset_id, feature, index, feature_class)
|
||||
row = VectorFeatureService._feature_row(
|
||||
dataset_id,
|
||||
feature,
|
||||
index,
|
||||
feature_class,
|
||||
source_crs=source_crs,
|
||||
)
|
||||
if row is None:
|
||||
continue
|
||||
if row.source_feature_id:
|
||||
|
||||
@@ -3,13 +3,17 @@ from __future__ import annotations
|
||||
import json
|
||||
import uuid
|
||||
from datetime import datetime, timezone
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from geoalchemy2.shape import to_shape
|
||||
from pyproj import CRS, Transformer
|
||||
from shapely.geometry import GeometryCollection, MultiPolygon, shape
|
||||
from shapely.geometry.base import BaseGeometry
|
||||
from shapely.geometry import mapping
|
||||
from shapely.ops import transform as shapely_transform
|
||||
from shapely.ops import unary_union
|
||||
from shapely.validation import make_valid
|
||||
from sqlalchemy.orm import Session
|
||||
@@ -18,12 +22,16 @@ from app.core.errors import AppError
|
||||
from app.models import Area, Dataset, DatasetVersion
|
||||
from app.schemas.dataset import DatasetCreateResponse
|
||||
from app.schemas.operations import VectorOperationResult
|
||||
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
|
||||
from app.services.geojson_service import parse_geojson_payload
|
||||
from app.services.storage_service import StorageService
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
|
||||
|
||||
class VectorOperationsService:
|
||||
CANONICAL_VECTOR_CRS = "EPSG:4326"
|
||||
_CHECKSUM_SHA256 = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE)
|
||||
|
||||
@staticmethod
|
||||
def _require_vector_dataset(dataset: Dataset) -> None:
|
||||
if dataset.dataset_type not in {"vector", "geojson"}:
|
||||
@@ -37,7 +45,8 @@ class VectorOperationsService:
|
||||
if not path.exists():
|
||||
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
stored_bytes = path.read_bytes()
|
||||
payload = json.loads(stored_bytes.decode("utf-8"))
|
||||
except Exception as exc:
|
||||
raise AppError(code="INVALID_GEOJSON", message="Stored dataset is not valid JSON", status_code=400) from exc
|
||||
|
||||
@@ -47,6 +56,55 @@ class VectorOperationsService:
|
||||
features = payload.get("features")
|
||||
if not isinstance(features, list):
|
||||
raise AppError(code="INVALID_GEOJSON", message="Dataset payload is missing features", status_code=400)
|
||||
|
||||
# The normal Dataset storage path is a consumption artifact, not a
|
||||
# provenance source archive. Refuse projected/original source bytes
|
||||
# here rather than letting a spatial operation interpret them as
|
||||
# canonical map coordinates.
|
||||
raw_crs = payload.get("crs")
|
||||
if isinstance(raw_crs, dict):
|
||||
crs_properties = raw_crs.get("properties")
|
||||
raw_crs = crs_properties.get("name") if isinstance(crs_properties, dict) else None
|
||||
stored_crs = str(raw_crs or VectorOperationsService.CANONICAL_VECTOR_CRS).strip().upper()
|
||||
dataset_crs = str(dataset.crs or "").strip().upper()
|
||||
if stored_crs != VectorOperationsService.CANONICAL_VECTOR_CRS or (
|
||||
dataset_crs and dataset_crs != VectorOperationsService.CANONICAL_VECTOR_CRS
|
||||
):
|
||||
raise AppError(
|
||||
code="DATASET_STORAGE_CRS_MISMATCH",
|
||||
message="Vector operations require canonical EPSG:4326 dataset storage.",
|
||||
details={
|
||||
"stored_crs": raw_crs or VectorOperationsService.CANONICAL_VECTOR_CRS,
|
||||
"dataset_crs": dataset.crs,
|
||||
"expected_crs": VectorOperationsService.CANONICAL_VECTOR_CRS,
|
||||
},
|
||||
status_code=409,
|
||||
)
|
||||
|
||||
expected_checksum = str(dataset.checksum_sha256 or "").strip().lower()
|
||||
actual_checksum = sha256(stored_bytes).hexdigest()
|
||||
governed_artifact = bool(
|
||||
getattr(dataset, "data_contract_key", None)
|
||||
or (
|
||||
isinstance(getattr(dataset, "metadata_json", None), dict)
|
||||
and dataset.metadata_json.get("canonical_storage_crs")
|
||||
)
|
||||
)
|
||||
if expected_checksum and VectorOperationsService._CHECKSUM_SHA256.fullmatch(expected_checksum):
|
||||
if expected_checksum != actual_checksum:
|
||||
raise AppError(
|
||||
code="DATASET_STORAGE_CHECKSUM_MISMATCH",
|
||||
message="Vector dataset storage no longer matches its validated checksum.",
|
||||
details={"expected_checksum_sha256": expected_checksum, "actual_checksum_sha256": actual_checksum},
|
||||
status_code=409,
|
||||
)
|
||||
elif governed_artifact:
|
||||
raise AppError(
|
||||
code="DATASET_STORAGE_CHECKSUM_UNVERIFIABLE",
|
||||
message="Governed vector storage requires a valid SHA-256 checksum before use.",
|
||||
details={"checksum_sha256": dataset.checksum_sha256},
|
||||
status_code=409,
|
||||
)
|
||||
return payload, [feature for feature in features if isinstance(feature, dict)]
|
||||
|
||||
@staticmethod
|
||||
@@ -73,6 +131,25 @@ class VectorOperationsService:
|
||||
raise AppError(code="EMPTY_VECTOR_DATASET", message="Dataset has no parseable geometries", status_code=422)
|
||||
return geometries
|
||||
|
||||
@staticmethod
|
||||
def _buffer_in_metres(geometry: BaseGeometry, distance_m: float, source_crs: str) -> BaseGeometry:
|
||||
"""Buffer in a Belgian projected CRS, never in angular degrees."""
|
||||
|
||||
try:
|
||||
input_crs = CRS.from_user_input(source_crs)
|
||||
metric_crs = CRS.from_epsg(31370)
|
||||
if input_crs == metric_crs:
|
||||
return geometry.buffer(distance_m)
|
||||
forward = Transformer.from_crs(input_crs, metric_crs, always_xy=True)
|
||||
backward = Transformer.from_crs(metric_crs, input_crs, always_xy=True)
|
||||
return shapely_transform(backward.transform, shapely_transform(forward.transform, geometry).buffer(distance_m))
|
||||
except Exception as exc:
|
||||
raise AppError(
|
||||
code="INVALID_CRS",
|
||||
message="A valid explicit CRS is required for metre-based vector buffering.",
|
||||
status_code=400,
|
||||
) from exc
|
||||
|
||||
@staticmethod
|
||||
def inspect(db: Session, dataset_id: uuid.UUID) -> VectorOperationResult:
|
||||
dataset = db.get(Dataset, dataset_id)
|
||||
@@ -94,7 +171,7 @@ class VectorOperationsService:
|
||||
feature_count=len(geometries),
|
||||
geometry_type_summary=geometry_type_summary,
|
||||
bounds_json={"min_x": float(bounds[0]), "min_y": float(bounds[1]), "max_x": float(bounds[2]), "max_y": float(bounds[3])},
|
||||
crs=payload.get("crs") if isinstance(payload.get("crs"), str) else dataset.crs,
|
||||
crs=VectorOperationsService.CANONICAL_VECTOR_CRS,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -184,9 +261,13 @@ class VectorOperationsService:
|
||||
if distance_m <= 0:
|
||||
raise AppError(code="INVALID_PARAMETERS", message="distance_m must be greater than 0", status_code=400)
|
||||
|
||||
_, features = VectorOperationsService._load_dataset_payload(source_dataset)
|
||||
payload, features = VectorOperationsService._load_dataset_payload(source_dataset)
|
||||
source_crs = VectorOperationsService.CANONICAL_VECTOR_CRS
|
||||
geometries = VectorOperationsService._extract_geometries(features)
|
||||
buffered_features = [(feature, geometry.buffer(distance_m)) for feature, geometry in geometries]
|
||||
buffered_features = [
|
||||
(feature, VectorOperationsService._buffer_in_metres(geometry, distance_m, source_crs))
|
||||
for feature, geometry in geometries
|
||||
]
|
||||
|
||||
output_features: list[dict[str, Any]] = []
|
||||
for feature, geometry in buffered_features:
|
||||
@@ -431,7 +512,14 @@ class VectorOperationsService:
|
||||
if not output_name_value.strip():
|
||||
output_name_value = f"{default_name}.geojson"
|
||||
|
||||
stored = json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
# All current governed vector storage is EPSG:4326. A legacy source
|
||||
# with another CRS is not relabelled here: the derived contract will
|
||||
# quarantine the result instead of placing non-WGS84 coordinates on
|
||||
# the map as if they were WGS84.
|
||||
output_crs = VectorOperationsService.CANONICAL_VECTOR_CRS
|
||||
output_feature_collection = dict(feature_collection)
|
||||
output_feature_collection["crs"] = output_crs
|
||||
stored = json.dumps(output_feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
storage_info = StorageService.persist_dataset_file(
|
||||
project_id=str(source_dataset.project_id),
|
||||
dataset_id=str(derived_id),
|
||||
@@ -441,7 +529,7 @@ class VectorOperationsService:
|
||||
content_type="application/geo+json",
|
||||
)
|
||||
|
||||
metadata = parse_geojson_payload(json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")))
|
||||
metadata = parse_geojson_payload(json.dumps(output_feature_collection, ensure_ascii=False, separators=(",", ":")))
|
||||
if metadata_extra:
|
||||
metadata.update(metadata_extra)
|
||||
derived_dataset = Dataset(
|
||||
@@ -452,7 +540,7 @@ class VectorOperationsService:
|
||||
dataset_type="vector",
|
||||
source=f"operation:{operation}",
|
||||
dataset_role=dataset_role,
|
||||
source_name=source_name,
|
||||
source_name=source_name or "derived",
|
||||
source_metadata=source_metadata,
|
||||
provenance_metadata=provenance_metadata,
|
||||
imported_at=datetime.now(timezone.utc),
|
||||
@@ -478,29 +566,44 @@ class VectorOperationsService:
|
||||
resolution_json=metadata.get("resolution_json"),
|
||||
bands_json=metadata.get("bands_json"),
|
||||
metadata_json=metadata,
|
||||
status="ready",
|
||||
status="validating",
|
||||
)
|
||||
db.add(derived_dataset)
|
||||
db.add(
|
||||
DatasetVersion(
|
||||
dataset_id=derived_dataset.id,
|
||||
version=1,
|
||||
storage_path=derived_dataset.storage_path,
|
||||
source_version=derived_dataset.source_version,
|
||||
observed_at=derived_dataset.observed_at,
|
||||
valid_from=derived_dataset.valid_from,
|
||||
valid_to=derived_dataset.valid_to,
|
||||
checksum_sha256=derived_dataset.checksum_sha256,
|
||||
source_metadata=derived_dataset.source_metadata,
|
||||
provenance_metadata=derived_dataset.provenance_metadata,
|
||||
)
|
||||
dataset_version = DatasetVersion(
|
||||
dataset_id=derived_dataset.id,
|
||||
version=1,
|
||||
storage_path=derived_dataset.storage_path,
|
||||
source_version=derived_dataset.source_version,
|
||||
observed_at=derived_dataset.observed_at,
|
||||
valid_from=derived_dataset.valid_from,
|
||||
valid_to=derived_dataset.valid_to,
|
||||
checksum_sha256=derived_dataset.checksum_sha256,
|
||||
source_metadata=derived_dataset.source_metadata,
|
||||
provenance_metadata=derived_dataset.provenance_metadata,
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(derived_dataset)
|
||||
if persist_vector_features:
|
||||
db.add(dataset_version)
|
||||
derived_source_key = "map_selection" if source_name == "map_selection" else "derived"
|
||||
is_ready = DerivedDatasetGovernanceService.govern_vector(
|
||||
db,
|
||||
dataset=derived_dataset,
|
||||
dataset_version=dataset_version,
|
||||
feature_collection=output_feature_collection,
|
||||
source_key=derived_source_key,
|
||||
operation=f"vector.{operation}",
|
||||
parent_dataset=source_dataset,
|
||||
operation_parameters={
|
||||
"operation": operation,
|
||||
"output_name": output_name_value,
|
||||
**(metadata_extra or {}),
|
||||
},
|
||||
)
|
||||
if persist_vector_features and is_ready:
|
||||
VectorFeatureService.persist_geojson_features(
|
||||
db=db,
|
||||
dataset_id=derived_dataset.id,
|
||||
payload=feature_collection,
|
||||
payload=output_feature_collection,
|
||||
commit=False,
|
||||
)
|
||||
db.commit()
|
||||
db.refresh(derived_dataset)
|
||||
return derived_id
|
||||
|
||||
@@ -9,6 +9,7 @@ from app.core.config import Settings, get_settings
|
||||
from app.core.errors import AppError
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||
from app.services.yolo_adapter import YoloDetectionAdapter
|
||||
|
||||
|
||||
@@ -41,6 +42,8 @@ class YoloPreflightService:
|
||||
"accelerator_ready": None,
|
||||
"model_path_set": None,
|
||||
"model_file_exists": None,
|
||||
"model_provenance_manifest_path": None,
|
||||
"model_provenance_valid": None,
|
||||
"model_load_requested": check_model_load,
|
||||
"model_load_ok": None,
|
||||
"manifest_path_set": None,
|
||||
@@ -98,6 +101,29 @@ class YoloPreflightService:
|
||||
result["message"] = "YOLO_MODEL_PATH does not point to an existing local model file."
|
||||
return result
|
||||
|
||||
result["checks"]["model_provenance_manifest_path"] = str(
|
||||
RuntimeModelProvenanceService.manifest_path_for_model(model_path)
|
||||
)
|
||||
try:
|
||||
RuntimeModelProvenanceService.validate_for_runtime(
|
||||
model_path=model_path,
|
||||
model_id=resolved_settings.yolo_model_id,
|
||||
task_type="object_detection",
|
||||
expected_model_version=resolved_settings.yolo_model_version,
|
||||
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
|
||||
)
|
||||
except AppError as exc:
|
||||
result["checks"]["model_provenance_valid"] = False
|
||||
result["status"] = "contract_incomplete"
|
||||
result["message"] = (
|
||||
"Configured YOLO weights are not runnable until their immutable runtime provenance sidecar validates: "
|
||||
f"{exc.message}"
|
||||
)
|
||||
result["error_code"] = exc.code
|
||||
result["details"] = exc.details
|
||||
return result
|
||||
result["checks"]["model_provenance_valid"] = True
|
||||
|
||||
if check_model_load:
|
||||
try:
|
||||
yolo_adapter_class(resolved_settings).load_model(model_path)
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
"""Canonical backend-test import boundary.
|
||||
|
||||
Pytest is intentionally runnable from ``backend/`` because that is the CI
|
||||
entrypoint. Some contract tests exercise repository-level deterministic
|
||||
scripts; put the canonical repository root ahead of the legacy
|
||||
``backend/scripts`` helper directory so those imports resolve to the code that
|
||||
is actually shipped by the root Docker build.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
|
||||
repository_root_text = str(REPOSITORY_ROOT)
|
||||
if repository_root_text not in sys.path:
|
||||
sys.path.insert(0, repository_root_text)
|
||||
@@ -0,0 +1,34 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "run_accuracy_phase2_foundation_audit.py"
|
||||
SPEC = importlib.util.spec_from_file_location("accuracy_phase2_foundation_audit", SCRIPT)
|
||||
assert SPEC is not None and SPEC.loader is not None
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
def test_phase2_foundation_audit_enumerates_exact_source_and_contract_policies() -> None:
|
||||
payload = MODULE.collect()
|
||||
|
||||
assert payload["phase"] == "P2"
|
||||
assert payload["migration_revision"] == "202608010001"
|
||||
assert payload["source_registry"]["definition_count"] >= 40
|
||||
assert payload["source_registry"]["required_building_policy"] == {
|
||||
"grb_primary_building_validation": "primary",
|
||||
"buildings_register_classification": "authoritative",
|
||||
"sentinel_2_classification": "contextual",
|
||||
"dhmv_classification": "authoritative",
|
||||
"osm_ground_truth_allowed": False,
|
||||
}
|
||||
assert {(item["key"], item["version"]) for item in payload["data_contracts"]} == {
|
||||
("geointel.vector.geojson", "1.0.0"),
|
||||
("geointel.raster.geotiff", "1.0.0"),
|
||||
("geointel.label.yolo", "1.0.0"),
|
||||
("geointel.label.yolo", "1.1.0"),
|
||||
("geointel.model.pytorch", "1.0.0"),
|
||||
}
|
||||
@@ -0,0 +1,772 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import timedelta
|
||||
from importlib.util import module_from_spec, spec_from_file_location
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import CheckConstraint
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import (
|
||||
Dataset,
|
||||
DatasetLineageEdge,
|
||||
DatasetQuarantine,
|
||||
DatasetVersion,
|
||||
SourceRegistry,
|
||||
SourceSnapshot,
|
||||
)
|
||||
from app.services.coverage_registry_service import SOURCE_DEFINITIONS
|
||||
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||
from app.services.dataset_service import DatasetService
|
||||
from app.services.source_registry_service import (
|
||||
SERVER_OWNED_SOURCE_DEFINITIONS,
|
||||
SourceRegistryService,
|
||||
)
|
||||
|
||||
|
||||
class _Query:
|
||||
def __init__(self, session: "InMemorySession", model: type) -> None:
|
||||
self.session = session
|
||||
self.model = model
|
||||
self.predicates = []
|
||||
|
||||
def filter(self, *predicates):
|
||||
self.predicates.extend(predicates)
|
||||
return self
|
||||
|
||||
def one_or_none(self):
|
||||
matches = self._matches()
|
||||
if len(matches) > 1:
|
||||
raise AssertionError(
|
||||
f"Expected one {self.model.__name__}, found {len(matches)}"
|
||||
)
|
||||
return matches[0] if matches else None
|
||||
|
||||
def all(self):
|
||||
return self._matches()
|
||||
|
||||
def _matches(self):
|
||||
matches = list(self.session.objects.get(self.model, []))
|
||||
for predicate in self.predicates:
|
||||
field_name = predicate.left.key
|
||||
expected = predicate.right.value
|
||||
operator_name = getattr(predicate.operator, "__name__", "")
|
||||
if operator_name == "in_op":
|
||||
matches = [
|
||||
item for item in matches if getattr(item, field_name) in expected
|
||||
]
|
||||
else:
|
||||
matches = [
|
||||
item for item in matches if getattr(item, field_name) == expected
|
||||
]
|
||||
return matches
|
||||
|
||||
|
||||
class InMemorySession:
|
||||
def __init__(self, *objects: object) -> None:
|
||||
self.objects: dict[type, list[object]] = {}
|
||||
self.added: list[object] = []
|
||||
self.flushes = 0
|
||||
for item in objects:
|
||||
self._store(item)
|
||||
|
||||
def query(self, model: type) -> _Query:
|
||||
return _Query(self, model)
|
||||
|
||||
def add(self, item: object) -> None:
|
||||
if getattr(item, "id", None) is None:
|
||||
setattr(item, "id", uuid4())
|
||||
self._store(item)
|
||||
self.added.append(item)
|
||||
|
||||
def flush(self) -> None:
|
||||
self.flushes += 1
|
||||
|
||||
def _store(self, item: object) -> None:
|
||||
self.objects.setdefault(type(item), []).append(item)
|
||||
|
||||
|
||||
def _registry(source_key: str) -> SourceRegistry:
|
||||
definition = SERVER_OWNED_SOURCE_DEFINITIONS[source_key]
|
||||
return SourceRegistry(id=uuid4(), **definition.as_model_values())
|
||||
|
||||
|
||||
def _dataset() -> Dataset:
|
||||
return Dataset(
|
||||
id=uuid4(),
|
||||
project_id=uuid4(),
|
||||
name="candidate.tif",
|
||||
dataset_type="raster",
|
||||
source="governed",
|
||||
status="ready",
|
||||
validation_status="not_validated",
|
||||
provenance_status="incomplete",
|
||||
lineage_status="incomplete",
|
||||
quarantine_status="not_quarantined",
|
||||
)
|
||||
|
||||
|
||||
def test_server_owned_definitions_encode_building_authority_and_non_ground_truth_sources() -> (
|
||||
None
|
||||
):
|
||||
grb = SERVER_OWNED_SOURCE_DEFINITIONS["grb"]
|
||||
buildings_register = SERVER_OWNED_SOURCE_DEFINITIONS[
|
||||
"digitaal_vlaanderen_buildings_addresses_register"
|
||||
]
|
||||
sentinel = SERVER_OWNED_SOURCE_DEFINITIONS["sentinel_2"]
|
||||
dhmv = SERVER_OWNED_SOURCE_DEFINITIONS["digitaal_vlaanderen_dhmv"]
|
||||
osm = SERVER_OWNED_SOURCE_DEFINITIONS["osm"]
|
||||
|
||||
assert grb.classification == "authoritative"
|
||||
assert grb.usage_policy["ground_truth_allowed"] is True
|
||||
assert grb.usage_policy["validation_authority"]["building_validation"] == "primary"
|
||||
assert buildings_register.classification == "authoritative"
|
||||
assert buildings_register.usage_policy["ground_truth_allowed"] is False
|
||||
assert (
|
||||
buildings_register.usage_policy["validation_authority"]["building_validation"]
|
||||
== "corroborative"
|
||||
)
|
||||
assert (
|
||||
buildings_register.usage_policy["validation_authority"][
|
||||
"building_register_validation"
|
||||
]
|
||||
== "primary"
|
||||
)
|
||||
assert sentinel.classification == "contextual"
|
||||
assert dhmv.classification == "authoritative"
|
||||
assert dhmv.usage_policy["ground_truth_allowed"] is False
|
||||
assert (
|
||||
dhmv.usage_policy["validation_authority"]["building_validation"]
|
||||
== "corroborative"
|
||||
)
|
||||
assert (
|
||||
dhmv.usage_policy["validation_authority"]["elevation_validation"] == "primary"
|
||||
)
|
||||
assert osm.classification == "contextual"
|
||||
assert osm.usage_policy["ground_truth_allowed"] is False
|
||||
assert osm.usage_policy["automatic_ground_truth"] is False
|
||||
assert osm.usage_policy["training_allowed"] is False
|
||||
assert {
|
||||
"ngi_adminvector",
|
||||
"rbins_marine_reporting_units",
|
||||
"rbins_msp_2026",
|
||||
"grb",
|
||||
"digitaal_vlaanderen",
|
||||
"vrbg",
|
||||
"digitaal_vlaanderen_buildings_addresses_register",
|
||||
"digitaal_vlaanderen_orthophoto",
|
||||
"spw_orthophoto",
|
||||
"urbis_orthophoto",
|
||||
"digitaal_vlaanderen_dhmv",
|
||||
"spw_terrain",
|
||||
"spw_walous_land_cover",
|
||||
"spw_geoportail",
|
||||
"spw_picc",
|
||||
"urbis",
|
||||
"vmm_flood_hazard",
|
||||
"vmm_vha_bathymetry_profiles",
|
||||
"dov_soil_map",
|
||||
"statbel",
|
||||
"waterinfo",
|
||||
"agentschap_landbouw_zeevisserij_agricultural_parcels",
|
||||
"sentinel_2",
|
||||
"osm",
|
||||
"manual",
|
||||
"fixture",
|
||||
"map_selection",
|
||||
"derived",
|
||||
"training_label",
|
||||
"model",
|
||||
"experimental",
|
||||
"mdk_bcp_bathymetry",
|
||||
}.issubset(SERVER_OWNED_SOURCE_DEFINITIONS)
|
||||
|
||||
for umbrella_key in ("digitaal_vlaanderen", "spw_geoportail"):
|
||||
definition = SERVER_OWNED_SOURCE_DEFINITIONS[umbrella_key]
|
||||
assert definition.classification == "authoritative"
|
||||
assert definition.usage_policy["ground_truth_allowed"] is False
|
||||
assert definition.usage_policy["automatic_ground_truth"] is False
|
||||
|
||||
assert (
|
||||
SERVER_OWNED_SOURCE_DEFINITIONS["mdk_bcp_bathymetry"].ingest_status
|
||||
== "not_configured"
|
||||
)
|
||||
|
||||
|
||||
def test_coverage_and_direct_adapter_source_keys_are_registry_backed() -> None:
|
||||
coverage_source_keys = {
|
||||
definition.contract.source_name for definition in SOURCE_DEFINITIONS
|
||||
}
|
||||
coverage_materialization_keys = {
|
||||
source_key
|
||||
for definition in SOURCE_DEFINITIONS
|
||||
for source_key in definition.materialized_source_names
|
||||
}
|
||||
direct_adapter_source_keys = {
|
||||
"digitaal_vlaanderen",
|
||||
"spw_geoportail",
|
||||
"mdk_bcp_bathymetry",
|
||||
}
|
||||
|
||||
assert (
|
||||
coverage_source_keys
|
||||
| coverage_materialization_keys
|
||||
| direct_adapter_source_keys
|
||||
<= set(SERVER_OWNED_SOURCE_DEFINITIONS)
|
||||
)
|
||||
|
||||
|
||||
def test_new_adapter_source_seed_rows_match_server_owned_registry_semantics() -> None:
|
||||
migration_path = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "alembic"
|
||||
/ "versions"
|
||||
/ "202608010001_source_registry_provenance.py"
|
||||
)
|
||||
spec = spec_from_file_location("phase2_source_registry_migration", migration_path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
migration = module_from_spec(spec)
|
||||
spec.loader.exec_module(migration)
|
||||
seed_rows = {row["source_key"]: row for row in migration._seed_rows()}
|
||||
|
||||
for source_key in ("digitaal_vlaanderen", "spw_geoportail", "mdk_bcp_bathymetry"):
|
||||
expected = SERVER_OWNED_SOURCE_DEFINITIONS[source_key].as_model_values()
|
||||
observed = seed_rows[source_key]
|
||||
for field in (
|
||||
"source_key",
|
||||
"display_name",
|
||||
"classification",
|
||||
"authority_name",
|
||||
"authority_scope_json",
|
||||
"provider_adapter_key",
|
||||
"source_url",
|
||||
"default_crs",
|
||||
"default_units",
|
||||
"geographic_coverage_json",
|
||||
"usage_policy_json",
|
||||
"freshness_status",
|
||||
"ingest_status",
|
||||
"known_limitations_json",
|
||||
):
|
||||
assert observed[field] == expected[field]
|
||||
|
||||
|
||||
def test_ensure_source_is_idempotent_and_rejects_caller_owned_unknown_sources() -> None:
|
||||
grb = _registry("grb")
|
||||
session = InMemorySession(grb)
|
||||
|
||||
assert SourceRegistryService.ensure_server_owned_source(session, "GRB") is grb
|
||||
assert session.added == []
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
SourceRegistryService.ensure_server_owned_source(session, "caller_claimed_grb")
|
||||
|
||||
assert exc_info.value.code == "SOURCE_REGISTRY_ENTRY_NOT_FOUND"
|
||||
|
||||
|
||||
def test_snapshot_is_checksum_bound_and_idempotent() -> None:
|
||||
grb = _registry("grb")
|
||||
session = InMemorySession(grb)
|
||||
checksum = "a" * 64
|
||||
|
||||
snapshot = SourceRegistryService.record_snapshot(
|
||||
session,
|
||||
source_key="grb",
|
||||
snapshot_key="2026-08-01-gbg",
|
||||
checksum_sha256=checksum,
|
||||
source_version="2026-08-01",
|
||||
crs="EPSG:31370",
|
||||
units="metres",
|
||||
)
|
||||
|
||||
assert snapshot.source_registry_id == grb.id
|
||||
assert snapshot.checksum_sha256 == checksum
|
||||
assert snapshot.ingest_status == "ingested"
|
||||
assert (
|
||||
SourceRegistryService.record_snapshot(
|
||||
session,
|
||||
source_key="grb",
|
||||
snapshot_key="2026-08-01-gbg",
|
||||
checksum_sha256=checksum,
|
||||
)
|
||||
is snapshot
|
||||
)
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
SourceRegistryService.record_snapshot(
|
||||
session,
|
||||
source_key="grb",
|
||||
snapshot_key="2026-08-01-gbg",
|
||||
checksum_sha256="b" * 64,
|
||||
)
|
||||
assert exc_info.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT"
|
||||
|
||||
with pytest.raises(AppError) as version_conflict:
|
||||
SourceRegistryService.record_snapshot(
|
||||
session,
|
||||
source_key="grb",
|
||||
snapshot_key="2026-08-01-gbg",
|
||||
checksum_sha256=checksum,
|
||||
source_version="2026-08-02",
|
||||
)
|
||||
assert version_conflict.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT"
|
||||
|
||||
with pytest.raises(AppError) as invalid_checksum:
|
||||
SourceRegistryService.record_snapshot(
|
||||
session,
|
||||
source_key="grb",
|
||||
snapshot_key="bad-checksum",
|
||||
checksum_sha256="not-a-checksum",
|
||||
)
|
||||
assert invalid_checksum.value.code == "SOURCE_SNAPSHOT_CHECKSUM_INVALID"
|
||||
|
||||
|
||||
def test_governed_import_reuses_an_identical_snapshot_without_rewriting_fetched_at() -> None:
|
||||
"""A second project may bind the same immutable source snapshot safely."""
|
||||
|
||||
grb = _registry("grb")
|
||||
session = InMemorySession(grb)
|
||||
checksum = "a" * 64
|
||||
observed_at = None
|
||||
metadata = {
|
||||
"dataset_type": "vector",
|
||||
"bounds_json": {"min_x": 4.0, "min_y": 50.0, "max_x": 4.1, "max_y": 50.1},
|
||||
}
|
||||
source_metadata = {"source_url": "https://example.test/grb", "units": "metres"}
|
||||
|
||||
# Dataset ingest keys are project-scoped, while a source snapshot is
|
||||
# globally keyed by immutable source evidence. This represents the same
|
||||
# source file arriving through two independently resumable imports.
|
||||
project_one, project_two = uuid4(), uuid4()
|
||||
assert (
|
||||
DatasetService._ingest_key(
|
||||
project_id=project_one,
|
||||
source_key="grb",
|
||||
checksum_sha256=checksum,
|
||||
dataset_type="vector",
|
||||
dataset_role="source",
|
||||
area_id=None,
|
||||
reference_layer_name=None,
|
||||
source_version="2026-08-01",
|
||||
)
|
||||
!= DatasetService._ingest_key(
|
||||
project_id=project_two,
|
||||
source_key="grb",
|
||||
checksum_sha256=checksum,
|
||||
dataset_type="vector",
|
||||
dataset_role="source",
|
||||
area_id=None,
|
||||
reference_layer_name=None,
|
||||
source_version="2026-08-01",
|
||||
)
|
||||
)
|
||||
|
||||
first_source, first_snapshot = DatasetService._record_snapshot(
|
||||
session,
|
||||
source_key="grb",
|
||||
checksum_sha256=checksum,
|
||||
source_version="2026-08-01",
|
||||
observed_at=observed_at,
|
||||
valid_from=None,
|
||||
valid_to=None,
|
||||
source_crs="EPSG:31370",
|
||||
source_metadata=source_metadata,
|
||||
metadata=metadata,
|
||||
)
|
||||
original_fetched_at = first_snapshot.fetched_at
|
||||
|
||||
replay_source, replay_snapshot = DatasetService._record_snapshot(
|
||||
session,
|
||||
source_key="grb",
|
||||
checksum_sha256=checksum,
|
||||
source_version="2026-08-01",
|
||||
observed_at=observed_at,
|
||||
valid_from=None,
|
||||
valid_to=None,
|
||||
source_crs="EPSG:31370",
|
||||
source_metadata=source_metadata,
|
||||
metadata=metadata,
|
||||
)
|
||||
|
||||
assert replay_source is first_source
|
||||
assert replay_snapshot is first_snapshot
|
||||
assert replay_snapshot.fetched_at == original_fetched_at
|
||||
assert session.objects[SourceSnapshot] == [first_snapshot]
|
||||
|
||||
# Outside the governed replay path, a contradictory acquisition timestamp
|
||||
# remains immutable evidence and is still rejected.
|
||||
with pytest.raises(AppError) as fetched_at_conflict:
|
||||
SourceRegistryService.record_snapshot(
|
||||
session,
|
||||
source_key="grb",
|
||||
snapshot_key=first_snapshot.snapshot_key,
|
||||
checksum_sha256=checksum,
|
||||
fetched_at=original_fetched_at + timedelta(seconds=1),
|
||||
)
|
||||
assert fetched_at_conflict.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT"
|
||||
|
||||
# Replay mode is narrow: a changed immutable evidence field still fails.
|
||||
with pytest.raises(AppError) as conflict:
|
||||
SourceRegistryService.record_snapshot(
|
||||
session,
|
||||
source_key="grb",
|
||||
snapshot_key=first_snapshot.snapshot_key,
|
||||
checksum_sha256=checksum,
|
||||
crs="EPSG:4326",
|
||||
reuse_existing_snapshot=True,
|
||||
)
|
||||
assert conflict.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT"
|
||||
|
||||
|
||||
def test_snapshot_schema_requires_a_canonical_sha256() -> None:
|
||||
constraints = {
|
||||
constraint.name: str(constraint.sqltext)
|
||||
for constraint in SourceSnapshot.__table__.constraints
|
||||
if isinstance(constraint, CheckConstraint)
|
||||
}
|
||||
|
||||
assert SourceSnapshot.__table__.c.checksum_sha256.nullable is False
|
||||
assert "ck_source_snapshots_checksum_sha256" in constraints
|
||||
assert (
|
||||
"lower(checksum_sha256)" in constraints["ck_source_snapshots_checksum_sha256"]
|
||||
)
|
||||
|
||||
|
||||
def test_complete_provenance_binding_is_required_before_authoritative_validation() -> (
|
||||
None
|
||||
):
|
||||
grb = _registry("grb")
|
||||
snapshot = SourceSnapshot(
|
||||
id=uuid4(),
|
||||
source_registry_id=grb.id,
|
||||
snapshot_key="governed-grb",
|
||||
checksum_sha256="c" * 64,
|
||||
ingest_status="ingested",
|
||||
)
|
||||
dataset = _dataset()
|
||||
|
||||
SourceRegistryService.bind_dataset_provenance(
|
||||
dataset,
|
||||
source=grb,
|
||||
snapshot=snapshot,
|
||||
data_contract_key="vector.grb.buildings",
|
||||
data_contract_version="1.0.0",
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="complete",
|
||||
)
|
||||
|
||||
assert SourceRegistryService.is_dataset_eligible_for_authoritative_validation(
|
||||
dataset,
|
||||
source=grb,
|
||||
snapshot=snapshot,
|
||||
task="building_validation",
|
||||
)
|
||||
|
||||
osm = _registry("osm")
|
||||
dataset.source_registry_id = osm.id
|
||||
snapshot.source_registry_id = osm.id
|
||||
assert not SourceRegistryService.is_dataset_eligible_for_authoritative_validation(
|
||||
dataset,
|
||||
source=osm,
|
||||
snapshot=snapshot,
|
||||
task="building_validation",
|
||||
)
|
||||
|
||||
|
||||
def test_lineage_and_quarantine_are_fail_closed_and_observable() -> None:
|
||||
session = InMemorySession()
|
||||
parent_id = uuid4()
|
||||
child_id = uuid4()
|
||||
edge = SourceRegistryService.record_lineage_edge(
|
||||
session,
|
||||
parent_dataset_id=parent_id,
|
||||
child_dataset_id=child_id,
|
||||
relation_type="derived_from",
|
||||
transformation_name="vector_clip",
|
||||
input_checksum_sha256="d" * 64,
|
||||
output_checksum_sha256="e" * 64,
|
||||
)
|
||||
|
||||
assert isinstance(edge, DatasetLineageEdge)
|
||||
assert (
|
||||
SourceRegistryService.record_lineage_edge(
|
||||
session,
|
||||
parent_dataset_id=parent_id,
|
||||
child_dataset_id=child_id,
|
||||
relation_type="derived_from",
|
||||
transformation_name="vector_clip",
|
||||
input_checksum_sha256="d" * 64,
|
||||
output_checksum_sha256="e" * 64,
|
||||
)
|
||||
is edge
|
||||
)
|
||||
|
||||
with pytest.raises(AppError) as self_reference:
|
||||
SourceRegistryService.record_lineage_edge(
|
||||
session,
|
||||
parent_dataset_id=parent_id,
|
||||
child_dataset_id=parent_id,
|
||||
relation_type="derived_from",
|
||||
transformation_name="vector_clip",
|
||||
)
|
||||
assert self_reference.value.code == "DATASET_LINEAGE_SELF_REFERENCE"
|
||||
|
||||
grandchild_id = uuid4()
|
||||
SourceRegistryService.record_lineage_edge(
|
||||
session,
|
||||
parent_dataset_id=child_id,
|
||||
child_dataset_id=grandchild_id,
|
||||
relation_type="derived_from",
|
||||
transformation_name="vector_buffer",
|
||||
)
|
||||
with pytest.raises(AppError) as cycle:
|
||||
SourceRegistryService.record_lineage_edge(
|
||||
session,
|
||||
parent_dataset_id=grandchild_id,
|
||||
child_dataset_id=parent_id,
|
||||
relation_type="derived_from",
|
||||
transformation_name="vector_union",
|
||||
)
|
||||
assert cycle.value.code == "DATASET_LINEAGE_CYCLE_DETECTED"
|
||||
|
||||
dataset = _dataset()
|
||||
record = SourceRegistryService.quarantine_dataset(
|
||||
session,
|
||||
dataset=dataset,
|
||||
stage="vector_ingest",
|
||||
reason_code="CRS_UNVERIFIED",
|
||||
details={"observed_crs": None},
|
||||
)
|
||||
assert isinstance(record, DatasetQuarantine)
|
||||
assert dataset.status == "quarantined"
|
||||
assert dataset.quarantine_status == "quarantined"
|
||||
assert dataset.validation_status == "failed"
|
||||
|
||||
version_parent = _dataset()
|
||||
version = DatasetVersion(id=uuid4(), dataset_id=version_parent.id, version=1)
|
||||
version_session = InMemorySession(version_parent, version)
|
||||
version_record = SourceRegistryService.quarantine_dataset(
|
||||
version_session,
|
||||
dataset_version=version,
|
||||
stage="dataset_version_validation",
|
||||
reason_code="CHECKSUM_MISMATCH",
|
||||
)
|
||||
assert version_record.dataset_id == version_parent.id
|
||||
assert version_record.dataset_version_id == version.id
|
||||
assert version_parent.status == "quarantined"
|
||||
assert version_parent.quarantine_status == "quarantined"
|
||||
assert version_parent.validation_status == "failed"
|
||||
assert version_parent.provenance_status == "incomplete"
|
||||
assert version_parent.lineage_status == "incomplete"
|
||||
assert version.validation_status == "failed"
|
||||
assert version.provenance_status == "incomplete"
|
||||
|
||||
snapshot = SourceSnapshot(
|
||||
id=uuid4(),
|
||||
source_registry_id=uuid4(),
|
||||
snapshot_key="quarantined-source",
|
||||
checksum_sha256="f" * 64,
|
||||
ingest_status="ingested",
|
||||
)
|
||||
SourceRegistryService.quarantine_dataset(
|
||||
session,
|
||||
source_snapshot=snapshot,
|
||||
stage="source_snapshot_validation",
|
||||
reason_code="CHECKSUM_MISMATCH",
|
||||
)
|
||||
assert snapshot.ingest_status == "quarantined"
|
||||
|
||||
|
||||
def test_quarantine_propagates_transitively_to_descendant_dataset_and_version_consumption_gates() -> (
|
||||
None
|
||||
):
|
||||
"""A -> B -> C must fail closed when the governing A artifact is rejected."""
|
||||
|
||||
source = _registry("grb")
|
||||
snapshot = SourceSnapshot(
|
||||
id=uuid4(),
|
||||
source_registry_id=source.id,
|
||||
snapshot_key="transitive-quarantine-source",
|
||||
checksum_sha256="a" * 64,
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
)
|
||||
|
||||
def governed_dataset(name: str) -> Dataset:
|
||||
dataset = _dataset()
|
||||
dataset.name = name
|
||||
dataset.source = "grb"
|
||||
dataset.source_name = "grb"
|
||||
dataset.dataset_role = "source"
|
||||
dataset.checksum_sha256 = snapshot.checksum_sha256
|
||||
dataset.source_registry_id = source.id
|
||||
dataset.source_snapshot_id = snapshot.id
|
||||
dataset.data_contract_key = "geointel.raster.geotiff"
|
||||
dataset.data_contract_version = "1.0.0"
|
||||
dataset.validation_status = "passed"
|
||||
dataset.provenance_status = "complete"
|
||||
dataset.lineage_status = "complete"
|
||||
dataset.quarantine_status = "not_quarantined"
|
||||
dataset.status = "ready"
|
||||
dataset.source_registry = source
|
||||
dataset.source_snapshot = snapshot
|
||||
return dataset
|
||||
|
||||
parent = governed_dataset("parent.tif")
|
||||
child = governed_dataset("child.tif")
|
||||
grandchild = governed_dataset("grandchild.tif")
|
||||
parent_version = DatasetVersion(
|
||||
id=uuid4(),
|
||||
dataset_id=parent.id,
|
||||
version=1,
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="complete",
|
||||
)
|
||||
child_version = DatasetVersion(
|
||||
id=uuid4(),
|
||||
dataset_id=child.id,
|
||||
version=1,
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="complete",
|
||||
)
|
||||
grandchild_version = DatasetVersion(
|
||||
id=uuid4(),
|
||||
dataset_id=grandchild.id,
|
||||
version=1,
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="complete",
|
||||
)
|
||||
session = InMemorySession(
|
||||
parent,
|
||||
child,
|
||||
grandchild,
|
||||
parent_version,
|
||||
child_version,
|
||||
grandchild_version,
|
||||
)
|
||||
SourceRegistryService.record_lineage_edge(
|
||||
session,
|
||||
parent_dataset_id=parent.id,
|
||||
child_dataset_id=child.id,
|
||||
relation_type="derived_from",
|
||||
transformation_name="clip",
|
||||
)
|
||||
SourceRegistryService.record_lineage_edge(
|
||||
session,
|
||||
parent_dataset_id=child.id,
|
||||
child_dataset_id=grandchild.id,
|
||||
relation_type="derived_from",
|
||||
transformation_name="buffer",
|
||||
)
|
||||
|
||||
assert (
|
||||
DatasetConsumptionGate.evaluate(child, purpose="production_inference").eligible
|
||||
is True
|
||||
)
|
||||
assert (
|
||||
DatasetConsumptionGate.evaluate(
|
||||
grandchild, purpose="production_inference"
|
||||
).eligible
|
||||
is True
|
||||
)
|
||||
|
||||
SourceRegistryService.quarantine_dataset(
|
||||
session,
|
||||
dataset=parent,
|
||||
stage="contract_validation",
|
||||
reason_code="CHECKSUM_MISMATCH",
|
||||
)
|
||||
|
||||
for dataset in (parent, child, grandchild):
|
||||
decision = DatasetConsumptionGate.evaluate(
|
||||
dataset, purpose="production_inference"
|
||||
)
|
||||
assert dataset.status == "quarantined"
|
||||
assert dataset.quarantine_status == "quarantined"
|
||||
assert dataset.validation_status == "failed"
|
||||
assert dataset.provenance_status == "incomplete"
|
||||
assert dataset.lineage_status == "incomplete"
|
||||
assert decision.eligible is False
|
||||
assert "dataset_quarantined" in decision.reasons
|
||||
for dataset_version in (parent_version, child_version, grandchild_version):
|
||||
assert dataset_version.validation_status == "failed"
|
||||
assert dataset_version.provenance_status == "incomplete"
|
||||
assert dataset_version.lineage_status == "incomplete"
|
||||
|
||||
|
||||
def test_ingest_keys_are_scoped_and_migration_keeps_unknown_legacy_unbound() -> None:
|
||||
project_id = uuid4()
|
||||
dataset = _dataset()
|
||||
dataset.project_id = project_id
|
||||
dataset.ingest_key = "grb:2026-08-01:gbg:area-sha"
|
||||
version = DatasetVersion(
|
||||
id=uuid4(),
|
||||
dataset_id=dataset.id,
|
||||
ingest_key=dataset.ingest_key,
|
||||
validation_status="not_validated",
|
||||
provenance_status="incomplete",
|
||||
lineage_status="incomplete",
|
||||
)
|
||||
session = InMemorySession(dataset, version)
|
||||
|
||||
assert (
|
||||
SourceRegistryService.find_dataset_by_ingest_key(
|
||||
session, project_id, dataset.ingest_key
|
||||
)
|
||||
is dataset
|
||||
)
|
||||
assert (
|
||||
SourceRegistryService.find_dataset_version_by_ingest_key(
|
||||
session, dataset.id, dataset.ingest_key
|
||||
)
|
||||
is version
|
||||
)
|
||||
with pytest.raises(AppError) as invalid_key:
|
||||
SourceRegistryService.find_dataset_by_ingest_key(session, project_id, " ")
|
||||
assert invalid_key.value.code == "INGEST_KEY_INVALID"
|
||||
|
||||
migration = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "alembic"
|
||||
/ "versions"
|
||||
/ "202608010001_source_registry_provenance.py"
|
||||
).read_text(encoding="utf-8")
|
||||
assert "uuid_generate_v5" not in migration
|
||||
assert "__unregistered_legacy_source__" in migration
|
||||
assert "uq_datasets_project_ingest_key" in migration
|
||||
assert "uq_dataset_versions_dataset_ingest_key" in migration
|
||||
|
||||
|
||||
def test_migration_contains_database_guards_for_snapshot_pairing_contract_lineage_and_quarantine() -> (
|
||||
None
|
||||
):
|
||||
migration = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "alembic"
|
||||
/ "versions"
|
||||
/ "202608010001_source_registry_provenance.py"
|
||||
).read_text(encoding="utf-8")
|
||||
|
||||
assert "trg_datasets_snapshot_registry_guard" in migration
|
||||
assert "trg_dataset_versions_snapshot_registry_guard" in migration
|
||||
assert "trg_source_registry_write_guard" in migration
|
||||
assert "trg_source_snapshots_evidence_immutable" in migration
|
||||
assert "geointel_phase2_contract_report_guard" in migration
|
||||
assert "trg_datasets_contract_report_guard" in migration
|
||||
assert "trg_dataset_versions_contract_report_guard" in migration
|
||||
assert "matching complete validation report" in migration
|
||||
assert "geointel_phase2_lineage_cycle_guard" in migration
|
||||
assert "trg_dataset_lineage_edges_cycle_guard" in migration
|
||||
assert "geointel_phase2_lineage_edge_immutable_guard" in migration
|
||||
assert "trg_dataset_lineage_edges_immutable" in migration
|
||||
assert "WITH RECURSIVE descendants" in migration
|
||||
assert "geointel_phase2_quarantine_lineage_descendants" in migration
|
||||
assert "geointel_phase2_quarantine_state_guard" in migration
|
||||
assert "trg_dataset_quarantines_state_guard" in migration
|
||||
assert "accepted dataset artifact and provenance evidence is immutable" in migration
|
||||
@@ -154,6 +154,7 @@ def test_guest_login_exposes_models_but_rejects_management_and_cross_project_req
|
||||
other_project = client.get("/api/v1/projects/00000000-0000-0000-0000-000000000999")
|
||||
detection_models = client.get("/api/v1/detection/models")
|
||||
segmentation_models = client.get("/api/v1/segmentation/models")
|
||||
global_source_registry = client.get("/api/v1/source-registry/grb")
|
||||
cross_project_runs = client.get(
|
||||
"/api/v1/detection/runs?project_id=00000000-0000-0000-0000-000000000999"
|
||||
)
|
||||
@@ -179,6 +180,8 @@ def test_guest_login_exposes_models_but_rejects_management_and_cross_project_req
|
||||
assert detection_models.status_code == 200
|
||||
assert detection_models.json()["data"]["models"]
|
||||
assert segmentation_models.status_code == 200
|
||||
assert global_source_registry.status_code == 403
|
||||
assert global_source_registry.json()["error"] == "GUEST_ROUTE_NOT_AVAILABLE"
|
||||
assert cross_project_runs.status_code == 403
|
||||
assert cross_project_runs.json()["error"] == "GUEST_PROJECT_SCOPE_REQUIRED"
|
||||
assert cross_project_coverage.status_code == 403
|
||||
@@ -186,7 +189,7 @@ def test_guest_login_exposes_models_but_rejects_management_and_cross_project_req
|
||||
|
||||
|
||||
def test_guest_session_tokens_fail_closed_without_a_project_scope(monkeypatch) -> None:
|
||||
client = auth_client(monkeypatch, guest_access=True)
|
||||
auth_client(monkeypatch, guest_access=True)
|
||||
settings = get_settings()
|
||||
|
||||
try:
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -13,6 +14,94 @@ assert SPEC and SPEC.loader
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
from training_release_manifest import create_training_release_manifest # noqa: E402
|
||||
|
||||
|
||||
def write_fixture_manifest(path: Path) -> None:
|
||||
policy = "geointel-training-source-eligibility/v1"
|
||||
def eligible(sample_slug: str) -> dict[str, object]:
|
||||
return {
|
||||
"policy_version": policy,
|
||||
"eligible": True,
|
||||
"fixture_mode": True,
|
||||
"raster": {
|
||||
"eligible": True,
|
||||
"reasons": [],
|
||||
"evidence": {
|
||||
"dataset_id": f"raster:{sample_slug}",
|
||||
"checksum_sha256": "a" * 64,
|
||||
"source_registry_id": "fixture-raster",
|
||||
"source_snapshot_id": "fixture-raster-snapshot",
|
||||
},
|
||||
},
|
||||
"reference": {
|
||||
"eligible": True,
|
||||
"reasons": [],
|
||||
"evidence": {
|
||||
"dataset_id": f"reference:{sample_slug}",
|
||||
"checksum_sha256": "b" * 64,
|
||||
"source_registry_id": "fixture-reference",
|
||||
"source_snapshot_id": "fixture-reference-snapshot",
|
||||
},
|
||||
},
|
||||
}
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"training_eligibility": {
|
||||
"policy_version": policy,
|
||||
"status": "eligible",
|
||||
"fixture_mode": True,
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"sample_slug": sample_slug,
|
||||
"split": split,
|
||||
"raster_dataset_id": f"raster:{sample_slug}",
|
||||
"reference_dataset_id": f"reference:{sample_slug}",
|
||||
"training_eligibility": eligible(sample_slug),
|
||||
}
|
||||
for sample_slug, split in (("fixture-train", "train"), ("fixture-val", "val"))
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(path.parent / "corpus-freeze.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": 2,
|
||||
"manifest_sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
|
||||
"immutable": True,
|
||||
"training_eligibility_policy": policy,
|
||||
"fixture_mode": True,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def write_fixture_training_release(tmp_path: Path, manifest: Path) -> Path:
|
||||
dataset_dir = tmp_path / "fixture-dataset"
|
||||
for split, sample_slug in (("train", "fixture-train"), ("val", "fixture-val")):
|
||||
image = dataset_dir / "images" / split / f"{sample_slug}.png"
|
||||
label = dataset_dir / "labels" / split / f"{sample_slug}.txt"
|
||||
image.parent.mkdir(parents=True, exist_ok=True)
|
||||
label.parent.mkdir(parents=True, exist_ok=True)
|
||||
image.write_bytes(split.encode("utf-8"))
|
||||
label.write_text("0 0.5 0.5 0.2 0.2\n", encoding="utf-8")
|
||||
yaml_path = dataset_dir / "dataset.yaml"
|
||||
yaml_path.write_text(
|
||||
f"path: {dataset_dir}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
create_training_release_manifest(
|
||||
train_yaml=yaml_path,
|
||||
corpus_manifest=manifest,
|
||||
fixture_mode=True,
|
||||
)
|
||||
return yaml_path
|
||||
|
||||
|
||||
def test_training_command_is_cuda_deterministic_and_bound_to_frozen_inputs(tmp_path: Path) -> None:
|
||||
command = MODULE.training_command(
|
||||
@@ -78,6 +167,7 @@ def test_failed_iteration_builds_train_only_sampling_for_next_checkpoint(tmp_pat
|
||||
corpus_manifest=tmp_path / "manifest.json",
|
||||
assessment=tmp_path / "assessment.json",
|
||||
output_dir=tmp_path / "iteration-001" / "failure-driven-training",
|
||||
review_audit=tmp_path / "review-audit.json",
|
||||
)
|
||||
assert command[1].endswith("build_failure_driven_yolo_sampling.py")
|
||||
assert command[command.index("--summary") + 1].endswith("train-summary.json")
|
||||
@@ -103,20 +193,23 @@ def test_dry_run_can_gate_existing_checkpoint_without_training(tmp_path: Path) -
|
||||
"status": "ok", "low_variance_positive_tile_count": 0,
|
||||
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
||||
}))
|
||||
manifest = tmp_path / "manifest.json"
|
||||
write_fixture_manifest(manifest)
|
||||
train_yaml = write_fixture_training_release(tmp_path, manifest)
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable, str(SCRIPT),
|
||||
"--initial-model", str(tmp_path / "candidate.pt"),
|
||||
"--train-yaml", str(tmp_path / "dataset.yaml"),
|
||||
"--train-yaml", str(train_yaml),
|
||||
"--train-summary", str(tmp_path / "train-summary.json"),
|
||||
"--dataset-audit", str(audit),
|
||||
"--train-quality-audit", str(quality),
|
||||
"--calibration-summary", str(tmp_path / "cal.json"),
|
||||
"--test-summary", str(tmp_path / "test.json"),
|
||||
"--background-summary", str(tmp_path / "background.json"),
|
||||
"--corpus-manifest", str(tmp_path / "manifest.json"),
|
||||
"--corpus-manifest", str(manifest),
|
||||
"--output-dir", str(tmp_path / "output"),
|
||||
"--evaluate-initial-model", "--dry-run",
|
||||
"--evaluate-initial-model", "--fixture-mode", "--dry-run",
|
||||
], capture_output=True, text=True, check=False,
|
||||
)
|
||||
assert result.returncode == 0
|
||||
@@ -139,6 +232,9 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
|
||||
"status": "ok", "low_variance_positive_tile_count": 0,
|
||||
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
||||
}))
|
||||
manifest = tmp_path / "manifest.json"
|
||||
write_fixture_manifest(manifest)
|
||||
train_yaml = write_fixture_training_release(tmp_path, manifest)
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
@@ -146,7 +242,7 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
|
||||
"--initial-model",
|
||||
str(tmp_path / "base.pt"),
|
||||
"--train-yaml",
|
||||
str(tmp_path / "dataset.yaml"),
|
||||
str(train_yaml),
|
||||
"--train-summary",
|
||||
str(tmp_path / "train-summary.json"),
|
||||
"--dataset-audit",
|
||||
@@ -160,9 +256,10 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
|
||||
"--background-summary",
|
||||
str(tmp_path / "background.json"),
|
||||
"--corpus-manifest",
|
||||
str(tmp_path / "manifest.json"),
|
||||
str(manifest),
|
||||
"--output-dir",
|
||||
str(tmp_path / "output"),
|
||||
"--fixture-mode",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
@@ -172,7 +269,45 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
|
||||
assert "Dataset audit is not eligible for training" in result.stderr
|
||||
|
||||
|
||||
def test_pending_human_review_does_not_block_objective_training() -> None:
|
||||
def test_loop_rejects_manifest_without_source_eligibility_before_cuda_training(tmp_path: Path) -> None:
|
||||
audit = tmp_path / "audit.json"
|
||||
audit.write_text(json.dumps({
|
||||
"status": "needs_human_review", "failures": [],
|
||||
"manifest_immutable": True, "spatial_leakage_status": "ok",
|
||||
}))
|
||||
quality = tmp_path / "quality.json"
|
||||
quality.write_text(json.dumps({
|
||||
"status": "ok", "low_variance_positive_tile_count": 0,
|
||||
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
||||
}))
|
||||
manifest = tmp_path / "manifest.json"
|
||||
manifest.write_text(json.dumps({"samples": [{"sample_slug": "unproven"}]}), encoding="utf-8")
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable, str(SCRIPT),
|
||||
"--initial-model", str(tmp_path / "candidate.pt"),
|
||||
"--train-yaml", str(tmp_path / "dataset.yaml"),
|
||||
"--train-summary", str(tmp_path / "train-summary.json"),
|
||||
"--dataset-audit", str(audit),
|
||||
"--train-quality-audit", str(quality),
|
||||
"--calibration-summary", str(tmp_path / "cal.json"),
|
||||
"--test-summary", str(tmp_path / "test.json"),
|
||||
"--background-summary", str(tmp_path / "background.json"),
|
||||
"--corpus-manifest", str(manifest),
|
||||
"--output-dir", str(tmp_path / "output"),
|
||||
"--evaluate-initial-model", "--dry-run",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert "manifest_training_eligibility_missing" in result.stderr
|
||||
|
||||
|
||||
def test_pending_human_review_blocks_operational_training() -> None:
|
||||
audit = {
|
||||
"status": "needs_human_review",
|
||||
"failures": [],
|
||||
@@ -185,7 +320,58 @@ def test_pending_human_review_does_not_block_objective_training() -> None:
|
||||
"status": "ok", "low_variance_positive_tile_count": 0,
|
||||
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
||||
}
|
||||
assert MODULE.dataset_audit_failures(audit, quality) == []
|
||||
failures = MODULE.dataset_audit_failures(audit, quality)
|
||||
assert "unsupported audit status: needs_human_review" in failures
|
||||
assert "review_complete_not_true" in failures
|
||||
assert "accepted_human_review_evidence_missing" in failures
|
||||
|
||||
|
||||
def test_fixture_mode_can_relax_review_only_after_fixture_manifest_gate() -> None:
|
||||
audit = {
|
||||
"status": "needs_human_review",
|
||||
"failures": [],
|
||||
"manifest_immutable": True,
|
||||
"spatial_leakage_status": "ok",
|
||||
"review_complete": False,
|
||||
}
|
||||
quality = {
|
||||
"status": "ok", "low_variance_positive_tile_count": 0,
|
||||
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
||||
}
|
||||
assert MODULE.dataset_audit_failures(audit, quality, fixture_mode=True) == []
|
||||
|
||||
|
||||
def test_operational_dataset_audit_must_be_the_one_bound_into_the_release(tmp_path: Path) -> None:
|
||||
bound = tmp_path / "bound-audit.json"
|
||||
other = tmp_path / "other-audit.json"
|
||||
bound.write_text("{}", encoding="utf-8")
|
||||
other.write_text("{}", encoding="utf-8")
|
||||
release = {"human_review": {"audit_path": str(bound.resolve())}}
|
||||
|
||||
MODULE.assert_dataset_audit_bound_to_release(
|
||||
release=release,
|
||||
dataset_audit=bound,
|
||||
fixture_mode=False,
|
||||
)
|
||||
try:
|
||||
MODULE.assert_dataset_audit_bound_to_release(
|
||||
release=release,
|
||||
dataset_audit=other,
|
||||
fixture_mode=False,
|
||||
)
|
||||
except MODULE.TrainingReleaseError as exc:
|
||||
assert "does not match" in str(exc)
|
||||
else:
|
||||
raise AssertionError("unbound dataset audit was accepted")
|
||||
|
||||
|
||||
def test_protected_assessment_feedback_is_terminal_and_cannot_seed_another_yaml() -> None:
|
||||
assert MODULE.protected_feedback_roles(
|
||||
{"status": "continue_training_loop", "test": {"aggregate": {}}, "background": None}
|
||||
) == ["test"]
|
||||
assert MODULE.protected_feedback_roles(
|
||||
{"status": "continue_training_loop", "test": None, "background": {"aggregate": {}}}
|
||||
) == ["background"]
|
||||
|
||||
|
||||
def test_training_audit_still_fails_closed_on_automated_integrity_gates() -> None:
|
||||
|
||||
@@ -0,0 +1,449 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from shapely.geometry import box
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.services.data_contract_validation import (
|
||||
AttributeRule,
|
||||
BoundingBox,
|
||||
ContractKind,
|
||||
DataAssetValidationInput,
|
||||
DataContract,
|
||||
DataContractRegistry,
|
||||
DataContractValidator,
|
||||
FreshnessRules,
|
||||
GeometryRecord,
|
||||
GeometryRules,
|
||||
LineageEvidence,
|
||||
LineageRules,
|
||||
RasterRules,
|
||||
RequirementLevel,
|
||||
Resolution,
|
||||
ResolutionRules,
|
||||
TransformationEvidence,
|
||||
ValidationStatus,
|
||||
build_default_data_contract_registry,
|
||||
build_label_validation_input,
|
||||
build_model_validation_input,
|
||||
build_raster_ingest_input,
|
||||
build_vector_ingest_input,
|
||||
validate_registered_asset,
|
||||
)
|
||||
from app.services.data_quarantine_service import AssetUse, DataQuarantineService
|
||||
|
||||
|
||||
FIXTURE_ROOT = Path(__file__).resolve().parents[2] / "tests" / "fixtures" / "data-contracts"
|
||||
NOW = datetime(2026, 8, 1, 12, 0, tzinfo=timezone.utc)
|
||||
CHECKSUM_A = "a" * 64
|
||||
|
||||
|
||||
def _fixture_json(name: str) -> tuple[bytes, object]:
|
||||
raw = (FIXTURE_ROOT / name).read_bytes()
|
||||
return raw, json.loads(raw)
|
||||
|
||||
|
||||
def _checksum(raw: bytes) -> str:
|
||||
return sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def _lineage_with_transform() -> LineageEvidence:
|
||||
return LineageEvidence(
|
||||
transformations=(
|
||||
TransformationEvidence(
|
||||
name="epsg31370-to-epsg4326",
|
||||
version="1.0.0",
|
||||
checksum_sha256=CHECKSUM_A,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _vector_input_from_fixture(name: str, *, source_crs: str = "EPSG:31370", storage_crs: str = "EPSG:4326") -> DataAssetValidationInput:
|
||||
raw, payload = _fixture_json(name)
|
||||
assert isinstance(payload, dict)
|
||||
return build_vector_ingest_input(
|
||||
asset_id=f"fixture:{name}",
|
||||
source_crs=source_crs,
|
||||
storage_crs=storage_crs,
|
||||
feature_collection=payload,
|
||||
checksum_sha256=_checksum(raw),
|
||||
computed_checksum_sha256=_checksum(raw),
|
||||
content=raw,
|
||||
source_registry_id="source:digitaal-vlaanderen:grb",
|
||||
source_snapshot_id="snapshot:grb:2026-07-31",
|
||||
imported_at=NOW,
|
||||
metadata={"license": "Open Data Licence", "provider": "Digitaal Vlaanderen"},
|
||||
observed_at=NOW - timedelta(days=1),
|
||||
source_version="2026.07.31",
|
||||
lineage=_lineage_with_transform() if source_crs != storage_crs else LineageEvidence(),
|
||||
)
|
||||
|
||||
|
||||
def _issue_codes(report) -> set[str]:
|
||||
return {issue.code for issue in report.issues}
|
||||
|
||||
|
||||
def test_default_vector_contract_accepts_transformed_geojson_with_complete_provenance() -> None:
|
||||
report = validate_registered_asset(_vector_input_from_fixture("vector-building-valid.geojson"), now=NOW)
|
||||
|
||||
assert report.validation_status == ValidationStatus.PASSED
|
||||
assert report.quarantine_status == "not_quarantined"
|
||||
assert report.provenance_status == "complete"
|
||||
assert report.lineage_status == "complete"
|
||||
persisted = report.persistence_fields()
|
||||
assert persisted["data_contract_key"] == "geointel.vector.geojson"
|
||||
assert persisted["data_contract_version"] == "1.0.0"
|
||||
assert persisted["validation_report_json"]["report_sha256"] == report.report_sha256
|
||||
|
||||
|
||||
def test_default_vector_contract_quarantines_lambert_coordinates_mislabelled_as_epsg4326() -> None:
|
||||
report = validate_registered_asset(
|
||||
_vector_input_from_fixture(
|
||||
"vector-lambert-mislabelled-as-4326.geojson",
|
||||
source_crs="EPSG:4326",
|
||||
storage_crs="EPSG:4326",
|
||||
),
|
||||
now=NOW,
|
||||
)
|
||||
|
||||
assert report.validation_status == ValidationStatus.FAILED
|
||||
assert report.quarantine_status == "quarantined"
|
||||
assert "CRS_COORDINATE_DOMAIN_VIOLATION" in _issue_codes(report)
|
||||
|
||||
|
||||
def test_vector_contract_checks_geometry_attributes_bounds_and_topology_fail_closed() -> None:
|
||||
contract = DataContract(
|
||||
key="test.vector.buildings",
|
||||
version="1.0.0",
|
||||
kind=ContractKind.VECTOR,
|
||||
accepted_source_crs=frozenset({"EPSG:4326"}),
|
||||
canonical_storage_crs="EPSG:4326",
|
||||
spatial_domain=BoundingBox(2.0, 49.0, 7.0, 52.0),
|
||||
require_bounds=True,
|
||||
geometry_rules=GeometryRules(
|
||||
allowed_geometry_types=frozenset({"Polygon"}),
|
||||
attribute_rules=(AttributeRule("native_id", accepted_types=("integer",)),),
|
||||
forbid_shared_area=True,
|
||||
),
|
||||
)
|
||||
raw = b"overlapping-vector"
|
||||
asset = DataAssetValidationInput(
|
||||
asset_id="vector:bad",
|
||||
data_contract_key=contract.key,
|
||||
data_contract_version=contract.version,
|
||||
kind=ContractKind.VECTOR,
|
||||
source_crs="EPSG:4326",
|
||||
storage_crs="EPSG:4326",
|
||||
bounds=BoundingBox(4.0, 51.0, 4.1, 51.1),
|
||||
checksum_sha256=_checksum(raw),
|
||||
computed_checksum_sha256=_checksum(raw),
|
||||
content=raw,
|
||||
geometry_records=(
|
||||
GeometryRecord(box(4.0, 51.0, 4.05, 51.05), {"native_id": "wrong-type"}),
|
||||
GeometryRecord(box(4.025, 51.025, 4.075, 51.075), {}),
|
||||
),
|
||||
source_registry_id="source:test",
|
||||
source_snapshot_id="snapshot:test",
|
||||
imported_at=NOW,
|
||||
)
|
||||
|
||||
report = DataContractValidator.validate(contract, asset, now=NOW)
|
||||
|
||||
assert report.validation_status == ValidationStatus.FAILED
|
||||
assert {"ATTRIBUTE_TYPE_INVALID", "ATTRIBUTE_REQUIRED", "TOPOLOGY_SHARED_AREA"} <= _issue_codes(report)
|
||||
assert "BOUNDS_GEOMETRY_MISMATCH" in _issue_codes(report)
|
||||
|
||||
|
||||
def test_default_vector_contract_validates_replayable_large_partition_stream_without_materialising_geometry_list() -> None:
|
||||
"""Regional imports may be large but remain fully schema/domain checked.
|
||||
|
||||
The default contract has no source-specific shared-area rule, so the
|
||||
validator must make its bounds/schema passes over a replayable stream
|
||||
without accumulating every Shapely geometry in memory. A stricter
|
||||
source-specific contract can still opt into a bounded topology batch.
|
||||
"""
|
||||
|
||||
class ReplayableRecords:
|
||||
def __init__(self, count: int) -> None:
|
||||
self.count = count
|
||||
self.iterations = 0
|
||||
|
||||
def __iter__(self):
|
||||
self.iterations += 1
|
||||
for index in range(self.count):
|
||||
yield GeometryRecord(
|
||||
box(4.69, 51.09, 4.70, 51.10),
|
||||
{"partition_feature": index},
|
||||
)
|
||||
|
||||
raw = b"partitioned-vector-stream"
|
||||
records = ReplayableRecords(12_000)
|
||||
asset = DataAssetValidationInput(
|
||||
asset_id="vector:partitioned-stream",
|
||||
data_contract_key="geointel.vector.geojson",
|
||||
data_contract_version="1.0.0",
|
||||
kind=ContractKind.VECTOR,
|
||||
source_crs="EPSG:4326",
|
||||
storage_crs="EPSG:4326",
|
||||
bounds=BoundingBox(4.69, 51.09, 4.70, 51.10),
|
||||
checksum_sha256=_checksum(raw),
|
||||
computed_checksum_sha256=_checksum(raw),
|
||||
content=raw,
|
||||
metadata={"license": "Open Data"},
|
||||
geometry_records=records,
|
||||
source_registry_id="source:grb",
|
||||
source_snapshot_id="snapshot:grb:partitioned",
|
||||
imported_at=NOW,
|
||||
observed_at=NOW,
|
||||
source_version="2026-08-01",
|
||||
)
|
||||
|
||||
report = validate_registered_asset(asset, now=NOW)
|
||||
|
||||
assert report.validation_status == ValidationStatus.PASSED
|
||||
assert records.iterations >= 2
|
||||
|
||||
|
||||
def test_raster_contract_accepts_explicit_units_and_quarantines_stale_bad_profile() -> None:
|
||||
raw = b"raster-stage"
|
||||
valid = build_raster_ingest_input(
|
||||
asset_id="raster:valid",
|
||||
source_crs="EPSG:31370",
|
||||
storage_crs="EPSG:31370",
|
||||
raster_profile={"width": 512, "height": 512, "band_count": 3, "dtype": ["uint8"]},
|
||||
bounds=BoundingBox(193_277.5, 205_708.3, 193_777.5, 206_208.3),
|
||||
resolution=Resolution(0.9765625, 0.9765625, "m"),
|
||||
checksum_sha256=_checksum(raw),
|
||||
computed_checksum_sha256=_checksum(raw),
|
||||
content=raw,
|
||||
source_registry_id="source:orthophoto",
|
||||
source_snapshot_id="snapshot:orthophoto:2026.01",
|
||||
imported_at=NOW,
|
||||
metadata={"license": "Open Data"},
|
||||
observed_at=None,
|
||||
temporal_unknown_reason="latest mosaic has no per-pixel observation date",
|
||||
source_version=None,
|
||||
source_version_unknown_reason="provider did not publish an edition",
|
||||
)
|
||||
assert validate_registered_asset(valid, now=NOW).validation_status == ValidationStatus.PASSED
|
||||
|
||||
strict = DataContract(
|
||||
key="test.raster.strict",
|
||||
version="1.0.0",
|
||||
kind=ContractKind.RASTER,
|
||||
accepted_source_crs=frozenset({"EPSG:31370"}),
|
||||
require_bounds=True,
|
||||
raster_rules=RasterRules(allowed_band_counts=frozenset({3}), allowed_dtypes=frozenset({"uint8"})),
|
||||
resolution_rules=ResolutionRules(allowed_units=frozenset({"m"}), min_x=0.2, max_x=1.0, min_y=0.2, max_y=1.0),
|
||||
freshness_rules=FreshnessRules(observed_at=RequirementLevel.REQUIRED, max_age=timedelta(days=30)),
|
||||
lineage_rules=LineageRules(require_transformation_when_crs_changes=False),
|
||||
)
|
||||
invalid = DataAssetValidationInput(
|
||||
asset_id="raster:bad",
|
||||
data_contract_key=strict.key,
|
||||
data_contract_version=strict.version,
|
||||
kind=ContractKind.RASTER,
|
||||
source_crs="EPSG:31370",
|
||||
storage_crs="EPSG:31370",
|
||||
bounds=BoundingBox(100.0, 100.0, 200.0, 200.0),
|
||||
checksum_sha256=_checksum(raw),
|
||||
computed_checksum_sha256=_checksum(raw),
|
||||
content=raw,
|
||||
raster_profile={"width": 0, "height": 10, "band_count": 2, "dtype": ["float32"]},
|
||||
resolution=Resolution(2.0, 0.1, "degree"),
|
||||
source_registry_id="source:raster",
|
||||
source_snapshot_id="snapshot:raster",
|
||||
imported_at=NOW,
|
||||
observed_at=NOW - timedelta(days=31),
|
||||
)
|
||||
report = DataContractValidator.validate(strict, invalid, now=NOW)
|
||||
|
||||
assert report.validation_status == ValidationStatus.FAILED
|
||||
assert {
|
||||
"RASTER_PROFILE_VALUE_INVALID",
|
||||
"RASTER_BAND_COUNT_NOT_ALLOWED",
|
||||
"RASTER_DTYPE_NOT_ALLOWED",
|
||||
"RESOLUTION_UNIT_NOT_ALLOWED",
|
||||
"RESOLUTION_OUT_OF_RANGE",
|
||||
"FRESHNESS_EXCEEDED",
|
||||
} <= _issue_codes(report)
|
||||
|
||||
|
||||
def test_default_label_and_model_contracts_validate_good_and_bad_fixtures() -> None:
|
||||
valid_raw, valid_labels = _fixture_json("labels-valid.json")
|
||||
invalid_raw, invalid_labels = _fixture_json("labels-invalid.json")
|
||||
assert isinstance(valid_labels, list)
|
||||
assert isinstance(invalid_labels, list)
|
||||
lineage = LineageEvidence(upstream_asset_ids=("image:1",), upstream_checksums_sha256=(CHECKSUM_A,))
|
||||
valid_label = build_label_validation_input(
|
||||
asset_id="label:valid",
|
||||
label_records=valid_labels,
|
||||
checksum_sha256=_checksum(valid_raw),
|
||||
computed_checksum_sha256=_checksum(valid_raw),
|
||||
content=valid_raw,
|
||||
source_registry_id="source:labels",
|
||||
source_snapshot_id="snapshot:labels:1",
|
||||
imported_at=NOW,
|
||||
metadata={
|
||||
"image_checksum_sha256": CHECKSUM_A,
|
||||
"class_ontology_version": "buildings-v1",
|
||||
"source_corpus_manifest_sha256": CHECKSUM_A,
|
||||
},
|
||||
temporal_unknown_reason="labels inherit image observation handling",
|
||||
source_version_unknown_reason="label release is represented by its snapshot",
|
||||
lineage=lineage,
|
||||
)
|
||||
valid_report = validate_registered_asset(valid_label, now=NOW)
|
||||
assert valid_report.validation_status == ValidationStatus.PASSED
|
||||
|
||||
invalid_label = build_label_validation_input(
|
||||
asset_id="label:invalid",
|
||||
label_records=invalid_labels,
|
||||
checksum_sha256=_checksum(invalid_raw),
|
||||
computed_checksum_sha256=_checksum(invalid_raw),
|
||||
content=invalid_raw,
|
||||
source_registry_id="source:labels",
|
||||
source_snapshot_id="snapshot:labels:1",
|
||||
imported_at=NOW,
|
||||
metadata={
|
||||
"image_checksum_sha256": "not-a-sha256",
|
||||
"class_ontology_version": "buildings-v1",
|
||||
"source_corpus_manifest_sha256": CHECKSUM_A,
|
||||
},
|
||||
temporal_unknown_reason="labels inherit image observation handling",
|
||||
source_version_unknown_reason="label release is represented by its snapshot",
|
||||
lineage=lineage,
|
||||
)
|
||||
invalid_report = validate_registered_asset(invalid_label, now=NOW)
|
||||
assert invalid_report.validation_status == ValidationStatus.FAILED
|
||||
assert {
|
||||
"LABEL_CLASS_ID_NOT_ALLOWED",
|
||||
"LABEL_NORMALIZED_COORDINATE_INVALID",
|
||||
"METADATA_CHECKSUM_INVALID",
|
||||
} <= _issue_codes(invalid_report)
|
||||
|
||||
pure_background_raw = b""
|
||||
pure_background_metadata = {
|
||||
"image_checksum_sha256": CHECKSUM_A,
|
||||
"class_ontology_version": "buildings-v1",
|
||||
"source_corpus_manifest_sha256": CHECKSUM_A,
|
||||
"label_mode": "pure_background",
|
||||
"sample_slug": "forest-background-aoi",
|
||||
"split": "train",
|
||||
"raster_dataset_id": "dataset:raster:1",
|
||||
"reference_dataset_id": "dataset:reference:1",
|
||||
"review_decision": "accepted",
|
||||
"reviewer_id": "reviewer@example.test",
|
||||
"reviewed_at": "2026-08-01T11:00:00+00:00",
|
||||
"review_artifact_sha256": CHECKSUM_A,
|
||||
}
|
||||
pure_background = build_label_validation_input(
|
||||
asset_id="label:pure-background",
|
||||
label_records=(),
|
||||
label_mode="pure_background",
|
||||
checksum_sha256=_checksum(pure_background_raw),
|
||||
computed_checksum_sha256=_checksum(pure_background_raw),
|
||||
content=pure_background_raw,
|
||||
source_registry_id="source:labels",
|
||||
source_snapshot_id="snapshot:labels:1",
|
||||
imported_at=NOW,
|
||||
metadata=pure_background_metadata,
|
||||
temporal_unknown_reason="labels inherit image observation handling",
|
||||
source_version_unknown_reason="label release is represented by its snapshot",
|
||||
lineage=LineageEvidence(
|
||||
upstream_asset_ids=("dataset:raster:1", "dataset:reference:1"),
|
||||
upstream_checksums_sha256=(CHECKSUM_A, CHECKSUM_A),
|
||||
),
|
||||
)
|
||||
assert validate_registered_asset(pure_background, now=NOW).validation_status == ValidationStatus.PASSED
|
||||
|
||||
unmarked_empty = build_label_validation_input(
|
||||
asset_id="label:unmarked-empty",
|
||||
label_records=(),
|
||||
checksum_sha256=_checksum(pure_background_raw),
|
||||
computed_checksum_sha256=_checksum(pure_background_raw),
|
||||
content=pure_background_raw,
|
||||
source_registry_id="source:labels",
|
||||
source_snapshot_id="snapshot:labels:1",
|
||||
imported_at=NOW,
|
||||
metadata={
|
||||
"image_checksum_sha256": CHECKSUM_A,
|
||||
"class_ontology_version": "buildings-v1",
|
||||
"source_corpus_manifest_sha256": CHECKSUM_A,
|
||||
},
|
||||
temporal_unknown_reason="labels inherit image observation handling",
|
||||
source_version_unknown_reason="label release is represented by its snapshot",
|
||||
lineage=LineageEvidence(
|
||||
upstream_asset_ids=("dataset:raster:1", "dataset:reference:1"),
|
||||
upstream_checksums_sha256=(CHECKSUM_A, CHECKSUM_A),
|
||||
),
|
||||
)
|
||||
assert "PURE_BACKGROUND_MODE_REQUIRED" in _issue_codes(validate_registered_asset(unmarked_empty, now=NOW))
|
||||
|
||||
model_raw = b"model-asset"
|
||||
model = build_model_validation_input(
|
||||
asset_id="model:valid",
|
||||
model_metadata={"model_format": "pytorch", "framework": "torch", "class_mapping": {"0": "building"}},
|
||||
checksum_sha256=_checksum(model_raw),
|
||||
computed_checksum_sha256=_checksum(model_raw),
|
||||
content=model_raw,
|
||||
source_registry_id="source:model-registry",
|
||||
source_snapshot_id="snapshot:model:1",
|
||||
imported_at=NOW,
|
||||
source_version="candidate-1",
|
||||
metadata={"training_manifest_sha256": CHECKSUM_A, "runtime_manifest_sha256": CHECKSUM_A},
|
||||
lineage=lineage,
|
||||
)
|
||||
assert validate_registered_asset(model, now=NOW).validation_status == ValidationStatus.PASSED
|
||||
|
||||
|
||||
def test_unknown_contract_and_quarantine_gate_are_deterministic_and_fail_closed() -> None:
|
||||
unknown = DataAssetValidationInput(
|
||||
asset_id="asset:unknown",
|
||||
data_contract_key="does.not.exist",
|
||||
data_contract_version="9.9.9",
|
||||
kind=ContractKind.VECTOR,
|
||||
)
|
||||
report = DataContractRegistry().validate(unknown, now=NOW)
|
||||
assert report.validation_status == ValidationStatus.FAILED
|
||||
assert report.quarantine_status == "quarantined"
|
||||
assert _issue_codes(report) == {"DATA_CONTRACT_UNKNOWN"}
|
||||
|
||||
first = DataQuarantineService.decide(report)
|
||||
second = DataQuarantineService.decide(report)
|
||||
assert first.idempotency_key == second.idempotency_key
|
||||
assert first.reason_codes == ("DATA_CONTRACT_UNKNOWN",)
|
||||
with pytest.raises(AppError, match="cannot enter this pipeline") as exc_info:
|
||||
DataQuarantineService.require_eligible(first, use=AssetUse.PRODUCTION_INFERENCE)
|
||||
assert exc_info.value.code == "DATASET_QUARANTINED"
|
||||
assert exc_info.value.details["use"] == "production_inference"
|
||||
|
||||
clean_report = validate_registered_asset(_vector_input_from_fixture("vector-building-valid.geojson"), now=NOW)
|
||||
release_request = DataQuarantineService.decide(clean_report, previous=first)
|
||||
assert release_request.quarantine_status == "quarantined"
|
||||
assert release_request.requires_explicit_release is True
|
||||
assert release_request.reason_codes == ("QUARANTINE_RELEASE_REQUIRES_EXPLICIT_PERSISTENCE",)
|
||||
|
||||
|
||||
def test_registry_requires_exact_contract_version_and_fingerprints_schema() -> None:
|
||||
registry = build_default_data_contract_registry()
|
||||
version_mismatch = _vector_input_from_fixture("vector-building-valid.geojson")
|
||||
mismatched = DataAssetValidationInput(
|
||||
**{**version_mismatch.__dict__, "data_contract_version": "2.0.0"},
|
||||
)
|
||||
|
||||
report = registry.validate(mismatched, now=NOW)
|
||||
assert report.validation_status == ValidationStatus.FAILED
|
||||
assert "DATA_CONTRACT_UNKNOWN" in _issue_codes(report)
|
||||
|
||||
contract = registry.resolve("geointel.vector.geojson", "1.0.0")
|
||||
assert contract is not None
|
||||
direct_report = DataContractValidator.validate(contract, mismatched, now=NOW)
|
||||
assert direct_report.validation_status == ValidationStatus.FAILED
|
||||
assert "DATA_CONTRACT_IDENTITY_MISMATCH" in _issue_codes(direct_report)
|
||||
@@ -0,0 +1,316 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from shapely.geometry import box
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import Dataset, SourceRegistry, SourceSnapshot
|
||||
from app.services.coverage_registry_service import CoverageRegistryService, SOURCE_DEFINITIONS
|
||||
import app.services.dataset_consumption_gate_service as gate_module
|
||||
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||
from app.services.export_service import ExportService
|
||||
|
||||
|
||||
def _governed_dataset(
|
||||
*,
|
||||
source_key: str = "grb",
|
||||
classification: str = "authoritative",
|
||||
snapshot_freshness_status: str = "current",
|
||||
) -> Dataset:
|
||||
source_id = uuid4()
|
||||
snapshot_id = uuid4()
|
||||
checksum = "a" * 64
|
||||
source = SourceRegistry(
|
||||
id=source_id,
|
||||
source_key=source_key,
|
||||
display_name=f"{source_key} test source",
|
||||
classification=classification,
|
||||
authority_name="GeoIntel test authority",
|
||||
authority_scope_json={"scope": "test"},
|
||||
usage_policy_json={
|
||||
"ground_truth_allowed": classification == "authoritative",
|
||||
"validation_authority": {"building_validation": "primary"}
|
||||
if classification == "authoritative"
|
||||
else {},
|
||||
},
|
||||
)
|
||||
snapshot = SourceSnapshot(
|
||||
id=snapshot_id,
|
||||
source_registry_id=source_id,
|
||||
snapshot_key="test-snapshot",
|
||||
checksum_sha256=checksum,
|
||||
freshness_status=snapshot_freshness_status,
|
||||
ingest_status="ingested",
|
||||
)
|
||||
dataset = Dataset(
|
||||
id=uuid4(),
|
||||
project_id=uuid4(),
|
||||
name="governed.tif",
|
||||
dataset_type="raster",
|
||||
source=source_key,
|
||||
source_name=source_key,
|
||||
dataset_role="source",
|
||||
checksum_sha256=checksum,
|
||||
source_registry_id=source_id,
|
||||
source_snapshot_id=snapshot_id,
|
||||
data_contract_key="geointel.raster.geotiff",
|
||||
data_contract_version="1.0.0",
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="not_applicable",
|
||||
quarantine_status="not_quarantined",
|
||||
status="ready",
|
||||
)
|
||||
dataset.source_registry = source
|
||||
dataset.source_snapshot = snapshot
|
||||
return dataset
|
||||
|
||||
|
||||
def test_governed_dataset_passes_production_inference_and_authoritative_coverage() -> None:
|
||||
dataset = _governed_dataset()
|
||||
|
||||
inference = DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
|
||||
coverage = DatasetConsumptionGate.assert_eligible(dataset, purpose="authoritative_coverage")
|
||||
|
||||
assert inference.eligible is True
|
||||
assert coverage.eligible is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("field", "value", "error_code"),
|
||||
(
|
||||
("provenance_status", "incomplete", "DATASET_PROVENANCE_INCOMPLETE"),
|
||||
("validation_status", "failed", "DATASET_QUARANTINED"),
|
||||
("quarantine_status", "quarantined", "DATASET_QUARANTINED"),
|
||||
),
|
||||
)
|
||||
def test_explicit_unsafe_states_can_never_be_relaxed(field: str, value: str, error_code: str) -> None:
|
||||
dataset = _governed_dataset()
|
||||
setattr(dataset, field, value)
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
dataset,
|
||||
purpose="production_inference",
|
||||
fixture_mode=True,
|
||||
)
|
||||
|
||||
assert exc_info.value.code == error_code
|
||||
assert field.replace("_status", "") in " ".join(exc_info.value.details["reasons"])
|
||||
|
||||
|
||||
def test_legacy_fixture_can_support_fixture_qa_but_never_authoritative_coverage() -> None:
|
||||
fixture = Dataset(
|
||||
id=uuid4(),
|
||||
project_id=uuid4(),
|
||||
name="fixture.tif",
|
||||
dataset_type="raster",
|
||||
source="fixture",
|
||||
)
|
||||
|
||||
qa = DatasetConsumptionGate.assert_eligible(fixture, purpose="quality_assessment")
|
||||
with pytest.raises(AppError) as inference_error:
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
fixture,
|
||||
purpose="production_inference",
|
||||
fixture_mode=True,
|
||||
)
|
||||
with pytest.raises(AppError) as export_error:
|
||||
DatasetConsumptionGate.assert_eligible(fixture, purpose="export")
|
||||
with pytest.raises(AppError) as fixture_export_error:
|
||||
DatasetConsumptionGate.assert_eligible(fixture, purpose="export", fixture_mode=True)
|
||||
coverage = DatasetConsumptionGate.evaluate(fixture, purpose="authoritative_coverage")
|
||||
|
||||
assert qa.fixture_legacy_exception is True
|
||||
assert inference_error.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert "fixture_qa_only" in inference_error.value.details["reasons"]
|
||||
assert export_error.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert fixture_export_error.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert "fixture_qa_only" in fixture_export_error.value.details["reasons"]
|
||||
assert coverage.eligible is False
|
||||
assert "fixture_not_authoritative_coverage" in coverage.reasons
|
||||
|
||||
|
||||
def test_unprovenanced_persistent_dataset_is_blocked(monkeypatch) -> None:
|
||||
dataset = Dataset(
|
||||
id=uuid4(),
|
||||
project_id=uuid4(),
|
||||
name="manual.tif",
|
||||
dataset_type="raster",
|
||||
source="manual_upload",
|
||||
)
|
||||
monkeypatch.setattr(gate_module, "sa_inspect", lambda _dataset: SimpleNamespace(transient=False))
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
dataset,
|
||||
purpose="production_inference",
|
||||
fixture_mode=True,
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert "phase2_provenance_missing" in exc_info.value.details["reasons"]
|
||||
assert "fixture_source_required" in exc_info.value.details["reasons"]
|
||||
|
||||
|
||||
def test_transient_orm_test_double_can_only_bypass_missing_legacy_fields_for_qa() -> None:
|
||||
transient = Dataset(
|
||||
id=uuid4(),
|
||||
project_id=uuid4(),
|
||||
name="transient-test.tif",
|
||||
dataset_type="raster",
|
||||
source="manual_upload",
|
||||
)
|
||||
|
||||
decision = DatasetConsumptionGate.assert_eligible(transient, purpose="quality_assessment")
|
||||
coverage = DatasetConsumptionGate.evaluate(transient, purpose="authoritative_coverage")
|
||||
with pytest.raises(AppError) as production_error:
|
||||
DatasetConsumptionGate.assert_eligible(transient, purpose="production_inference")
|
||||
|
||||
assert decision.fixture_legacy_exception is True
|
||||
assert production_error.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert coverage.eligible is False
|
||||
assert "phase2_provenance_missing" in coverage.reasons
|
||||
|
||||
|
||||
@pytest.mark.parametrize("purpose", ("production_inference", "derived_processing", "export"))
|
||||
def test_passed_manual_or_experimental_dataset_cannot_cross_production_boundary(purpose: str) -> None:
|
||||
"""A syntactically valid manual upload remains experimental, never production-ready."""
|
||||
|
||||
manual = _governed_dataset(source_key="manual", classification="experimental")
|
||||
manual.source = "manual_upload"
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetConsumptionGate.assert_eligible(manual, purpose=purpose) # type: ignore[arg-type]
|
||||
|
||||
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert "experimental_source_not_allowed_for_purpose" in exc_info.value.details["reasons"]
|
||||
|
||||
|
||||
def test_reference_validation_requires_authoritative_ground_truth_reference() -> None:
|
||||
reference = _governed_dataset()
|
||||
reference.dataset_type = "vector"
|
||||
reference.dataset_role = "reference"
|
||||
|
||||
decision = DatasetConsumptionGate.assert_eligible(
|
||||
reference,
|
||||
purpose="reference_validation",
|
||||
reference_task="building_validation",
|
||||
)
|
||||
assert decision.eligible is True
|
||||
|
||||
reference.source_registry.classification = "corroborative"
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
reference,
|
||||
purpose="reference_validation",
|
||||
reference_task="building_validation",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert "reference_source_not_authoritative" in exc_info.value.details["reasons"]
|
||||
|
||||
|
||||
def test_pending_regional_building_authority_cannot_become_truth_without_approval() -> None:
|
||||
reference = _governed_dataset(source_key="spw_picc", classification="authoritative")
|
||||
reference.dataset_type = "vector"
|
||||
reference.dataset_role = "reference"
|
||||
reference.source_registry.authority_scope_json = {"zone": "Wallonia"}
|
||||
reference.source_registry.usage_policy_json = {
|
||||
"ground_truth_allowed": True,
|
||||
"validation_authority": {"building_validation": "regional_primary_pending_contract"},
|
||||
}
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
reference,
|
||||
purpose="reference_validation",
|
||||
reference_task="building_validation",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert "reference_task_authority_not_approved" in exc_info.value.details["reasons"]
|
||||
|
||||
|
||||
def test_source_snapshot_must_belong_to_the_dataset_source_registry() -> None:
|
||||
dataset = _governed_dataset()
|
||||
dataset.source_snapshot.source_registry_id = uuid4()
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
|
||||
|
||||
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert "source_snapshot_registry_mismatch" in exc_info.value.details["reasons"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("freshness_status", ("unknown", "review_required", "due", "stale"))
|
||||
def test_non_consumable_source_snapshot_freshness_is_blocked_at_production_boundaries(
|
||||
freshness_status: str,
|
||||
) -> None:
|
||||
dataset = _governed_dataset(snapshot_freshness_status=freshness_status)
|
||||
|
||||
for purpose in ("production_inference", "authoritative_coverage"):
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetConsumptionGate.assert_eligible(dataset, purpose=purpose) # type: ignore[arg-type]
|
||||
|
||||
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||
assert "source_snapshot_freshness_not_eligible" in exc_info.value.details["reasons"]
|
||||
|
||||
|
||||
def test_coverage_registry_ignores_explicitly_incomplete_materialization() -> None:
|
||||
definition = next(item for item in SOURCE_DEFINITIONS if item.contract.source_name == "digitaal_vlaanderen")
|
||||
unsafe_materialization = SimpleNamespace(
|
||||
id=uuid4(),
|
||||
status="ready",
|
||||
source_name="grb",
|
||||
validation_status="passed",
|
||||
provenance_status="incomplete",
|
||||
lineage_status="complete",
|
||||
quarantine_status="not_quarantined",
|
||||
)
|
||||
|
||||
matches, fully_covered = CoverageRegistryService._matching_datasets(
|
||||
[unsafe_materialization],
|
||||
definition,
|
||||
"buildings",
|
||||
"flanders",
|
||||
box(4.0, 50.8, 4.1, 50.9),
|
||||
)
|
||||
|
||||
assert matches == []
|
||||
assert fully_covered is False
|
||||
|
||||
|
||||
def test_vector_export_is_fail_closed_before_selection(monkeypatch) -> None:
|
||||
dataset = _governed_dataset()
|
||||
dataset.dataset_type = "vector"
|
||||
dataset.status = "quarantined"
|
||||
queried = False
|
||||
|
||||
class _Session:
|
||||
@staticmethod
|
||||
def get(model, item_id):
|
||||
return dataset if model is Dataset and item_id == dataset.id else None
|
||||
|
||||
def _unexpected_selection(*_args, **_kwargs):
|
||||
nonlocal queried
|
||||
queried = True
|
||||
raise AssertionError("unsafe dataset must be rejected before querying vector features")
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.export_service.VectorFeatureService.select_features_by_bbox",
|
||||
_unexpected_selection,
|
||||
)
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
ExportService.export_vector_selection_geojson(
|
||||
_Session(),
|
||||
dataset.id,
|
||||
{"min_x": 4.0, "min_y": 50.0, "max_x": 4.1, "max_y": 50.1, "crs": "EPSG:4326"},
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "DATASET_QUARANTINED"
|
||||
assert queried is False
|
||||
@@ -74,6 +74,8 @@ def test_all_in_one_dockerfile_copies_operator_scripts_for_runtime_use() -> None
|
||||
"audit_operator_yolo_dataset_quality.py",
|
||||
"render_operator_yolo_label_qa_contact_sheets.py",
|
||||
"train_operator_yolo_detector.sh",
|
||||
"training_dataset_eligibility.py",
|
||||
"training_release_manifest.py",
|
||||
"verify_real_data_detection_qa_workflow.sh",
|
||||
"run_detection_quality_matrix.sh",
|
||||
"run_multi_sample_detection_quality_matrix.sh",
|
||||
|
||||
@@ -3,6 +3,8 @@ from __future__ import annotations
|
||||
import importlib.util
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).parents[2] / "scripts" / "build_failure_driven_yolo_sampling.py"
|
||||
SPEC = importlib.util.spec_from_file_location("failure_sampling", SCRIPT)
|
||||
@@ -22,7 +24,7 @@ def test_dataset_validation_source_preserves_manifest_path(tmp_path: Path):
|
||||
assert MODULE.dataset_validation_source(source) == "/data/source/internal-val.txt"
|
||||
|
||||
|
||||
def test_sampling_repeats_only_failed_region_train_tiles() -> None:
|
||||
def test_sampling_rejects_protected_test_and_background_feedback() -> None:
|
||||
manifest = {
|
||||
"samples": [
|
||||
{"sample_slug": "train-fl", "split": "train", "region": "flanders"},
|
||||
@@ -54,15 +56,10 @@ def test_sampling_repeats_only_failed_region_train_tiles() -> None:
|
||||
},
|
||||
"background": {"pure_empty_false_positives": 2},
|
||||
}
|
||||
paths, metadata = MODULE.build_sampling(
|
||||
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0
|
||||
)
|
||||
assert paths.count(str(Path("/tmp/fl-pos.png").resolve())) == 3
|
||||
assert paths.count(str(Path("/tmp/fl-neg.png").resolve())) == 4
|
||||
assert paths.count(str(Path("/tmp/wa-pos.png").resolve())) == 1
|
||||
assert not any("protected" in path for path in paths)
|
||||
assert metadata["protected_samples_in_training"] == []
|
||||
assert metadata["weak_recall_regions"] == ["flanders"]
|
||||
with pytest.raises(ValueError, match="protected test/background evidence"):
|
||||
MODULE.build_sampling(
|
||||
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0
|
||||
)
|
||||
|
||||
|
||||
def test_sampling_can_use_calibration_before_test_is_opened() -> None:
|
||||
@@ -171,7 +168,7 @@ def test_sampling_targets_failed_calibration_contexts_without_using_protected_ti
|
||||
assert metadata["recall_dominant_regions"] == ["flanders"]
|
||||
|
||||
|
||||
def test_recall_dominance_does_not_suppress_negatives_when_background_gate_failed() -> None:
|
||||
def test_sampling_rejects_background_feedback_after_a_protected_background_opening() -> None:
|
||||
manifest = {"samples": [
|
||||
{"sample_slug": "positive", "split": "train", "region": "flanders", "context": "industrial"},
|
||||
{"sample_slug": "negative", "split": "train", "region": "flanders", "context": "industrial-hard-negative"},
|
||||
@@ -190,12 +187,10 @@ def test_recall_dominance_does_not_suppress_negatives_when_background_gate_faile
|
||||
"background": {"pure_empty_false_positives": 1},
|
||||
}
|
||||
|
||||
paths, metadata = MODULE.build_sampling(
|
||||
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0,
|
||||
)
|
||||
|
||||
assert paths.count(str(Path("/tmp/negative.png").resolve())) == 4
|
||||
assert metadata["recall_dominant_regions"] == []
|
||||
with pytest.raises(ValueError, match="protected background evidence"):
|
||||
MODULE.build_sampling(
|
||||
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0,
|
||||
)
|
||||
|
||||
|
||||
def test_region_cap_drops_only_repeats_and_preserves_every_unique_tile() -> None:
|
||||
|
||||
@@ -0,0 +1,765 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import UTC, datetime
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
import pytest
|
||||
from pyproj import Transformer
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import (
|
||||
Area,
|
||||
Dataset,
|
||||
DatasetQuarantine,
|
||||
DatasetVersion,
|
||||
Project,
|
||||
SourceRegistry,
|
||||
SourceSnapshot,
|
||||
VectorFeature,
|
||||
)
|
||||
from app.services.dataset_service import DatasetService, _PartitionedGeoJsonRecords
|
||||
from app.services.vector_operations_service import VectorOperationsService
|
||||
|
||||
|
||||
class _Query:
|
||||
def __init__(self, session: "_Session", model: type) -> None:
|
||||
self.session = session
|
||||
self.model = model
|
||||
self.predicates = []
|
||||
|
||||
def filter(self, *predicates):
|
||||
self.predicates.extend(predicates)
|
||||
return self
|
||||
|
||||
def one_or_none(self):
|
||||
matches = self._matches()
|
||||
if len(matches) > 1:
|
||||
raise AssertionError(
|
||||
f"expected one {self.model.__name__}, found {len(matches)}"
|
||||
)
|
||||
return matches[0] if matches else None
|
||||
|
||||
def all(self):
|
||||
return self._matches()
|
||||
|
||||
def _matches(self):
|
||||
matches = list(self.session.rows.get(self.model, []))
|
||||
for predicate in self.predicates:
|
||||
field_name = predicate.left.key
|
||||
expected = predicate.right.value
|
||||
operator_name = getattr(predicate.operator, "__name__", "")
|
||||
if operator_name == "in_op":
|
||||
matches = [
|
||||
item for item in matches if getattr(item, field_name) in expected
|
||||
]
|
||||
else:
|
||||
matches = [
|
||||
item for item in matches if getattr(item, field_name) == expected
|
||||
]
|
||||
return matches
|
||||
|
||||
|
||||
class _Session:
|
||||
"""Small ORM-shaped harness that exercises the real governed path."""
|
||||
|
||||
def __init__(self, project: Project) -> None:
|
||||
self.rows: dict[type, list[object]] = {Project: [project]}
|
||||
self.commits = 0
|
||||
self.rollbacks = 0
|
||||
self.flushes = 0
|
||||
|
||||
def get(self, model: type, item_id: UUID):
|
||||
return next(
|
||||
(
|
||||
item
|
||||
for item in self.rows.get(model, [])
|
||||
if getattr(item, "id", None) == item_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
def query(self, model: type) -> _Query:
|
||||
return _Query(self, model)
|
||||
|
||||
def add(self, item: object) -> None:
|
||||
if getattr(item, "id", None) is None:
|
||||
setattr(item, "id", uuid4())
|
||||
self.rows.setdefault(type(item), []).append(item)
|
||||
|
||||
def flush(self) -> None:
|
||||
self.flushes += 1
|
||||
|
||||
def commit(self) -> None:
|
||||
self.commits += 1
|
||||
|
||||
def rollback(self) -> None:
|
||||
self.rollbacks += 1
|
||||
|
||||
def refresh(self, _item: object) -> None:
|
||||
return None
|
||||
|
||||
def expunge(self, _item: object) -> None:
|
||||
return None
|
||||
|
||||
|
||||
def _storage_info(tmp_path: Path, content: bytes) -> dict[str, object]:
|
||||
path = tmp_path / "grb-buildings.geojson"
|
||||
path.write_bytes(content)
|
||||
return {
|
||||
"storage_path": str(path),
|
||||
"original_filename": path.name,
|
||||
"stored_filename": path.name,
|
||||
"content_type": "application/geo+json",
|
||||
"size_bytes": len(content),
|
||||
"checksum_sha256": sha256(content).hexdigest(),
|
||||
}
|
||||
|
||||
|
||||
def _valid_payload() -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": "gbg-1",
|
||||
"properties": {"id": "gbg-1"},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _grb_payload_without_required_id() -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {"unrelated": "not a GRB identity"},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
|
||||
],
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
).encode("utf-8")
|
||||
|
||||
|
||||
def _lambert_grb_payload() -> tuple[bytes, tuple[float, float, float, float]]:
|
||||
"""Create a valid GRB-shaped source artifact in its declared native CRS."""
|
||||
|
||||
longitude, latitude = 4.70, 51.10
|
||||
max_longitude, max_latitude = 4.7001, 51.1001
|
||||
to_lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
||||
lambert_ring = [
|
||||
to_lambert.transform(longitude, latitude),
|
||||
to_lambert.transform(max_longitude, latitude),
|
||||
to_lambert.transform(max_longitude, max_latitude),
|
||||
to_lambert.transform(longitude, latitude),
|
||||
]
|
||||
return (
|
||||
json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"crs": {"type": "name", "properties": {"name": "EPSG:31370"}},
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": "GBG.lambert.1",
|
||||
"properties": {"id": "GBG.lambert.1"},
|
||||
"geometry": {"type": "Polygon", "coordinates": [lambert_ring]},
|
||||
}
|
||||
],
|
||||
}
|
||||
).encode("utf-8"),
|
||||
(longitude, latitude, max_longitude, max_latitude),
|
||||
)
|
||||
|
||||
|
||||
def test_governed_vector_import_persists_snapshot_contract_and_queryable_features(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
project = Project(id=uuid4(), name="Phase 2 governed ingest")
|
||||
db = _Session(project)
|
||||
raw = _valid_payload()
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||
lambda **_kwargs: _storage_info(tmp_path, raw),
|
||||
)
|
||||
|
||||
result = DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="grb-buildings.geojson",
|
||||
content=raw,
|
||||
source="grb_wfs",
|
||||
source_name="grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={
|
||||
"license": "Open data",
|
||||
"source_url": "https://example.invalid/grb",
|
||||
},
|
||||
provenance_metadata={"adapter": "test"},
|
||||
temporal_series_key="grb:2026-08",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01",
|
||||
temporal_granularity="snapshot",
|
||||
)
|
||||
|
||||
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
|
||||
snapshot = db.rows[SourceSnapshot][0]
|
||||
source = db.rows[SourceRegistry][0]
|
||||
assert result.status == "ready"
|
||||
assert dataset.source_name == "grb"
|
||||
assert dataset.source_registry_id == source.id
|
||||
assert dataset.source_snapshot_id == snapshot.id
|
||||
assert dataset.validation_status == "passed"
|
||||
assert dataset.provenance_status == "complete"
|
||||
assert dataset.lineage_status == "complete"
|
||||
assert dataset.quarantine_status == "not_quarantined"
|
||||
assert dataset.crs == "EPSG:4326"
|
||||
assert snapshot.checksum_sha256 == sha256(raw).hexdigest()
|
||||
assert len(db.rows[VectorFeature]) == 1
|
||||
assert db.commits == 1
|
||||
|
||||
# A retry with identical governed evidence is idempotent and does not
|
||||
# create a second source snapshot, dataset or vector feature.
|
||||
repeated = DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="grb-buildings.geojson",
|
||||
content=raw,
|
||||
source="grb_wfs",
|
||||
source_name="grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={"adapter": "test"},
|
||||
temporal_series_key="grb:2026-08",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01",
|
||||
temporal_granularity="snapshot",
|
||||
)
|
||||
assert repeated.id == result.id
|
||||
assert len(db.rows[Dataset]) == 1
|
||||
assert len(db.rows[SourceSnapshot]) == 1
|
||||
assert len(db.rows[VectorFeature]) == 1
|
||||
|
||||
|
||||
def test_governed_lambert_geojson_persists_canonical_consumption_bytes_and_provenance_evidence(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""Projected source bytes must never be the file that vector operations consume."""
|
||||
|
||||
project = Project(id=uuid4(), name="Canonical GeoJSON storage")
|
||||
db = _Session(project)
|
||||
raw, (longitude, latitude, max_longitude, max_latitude) = _lambert_grb_payload()
|
||||
consumption_path = tmp_path / "consumption" / "grb-buildings.geojson"
|
||||
provenance_path = tmp_path / "provenance" / "grb-buildings.geojson"
|
||||
|
||||
def _persist_dataset_file(**kwargs):
|
||||
stored = kwargs["content"]
|
||||
consumption_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
consumption_path.write_bytes(stored)
|
||||
return _storage_info(consumption_path.parent, stored)
|
||||
|
||||
def _persist_file(storage_path, content, original_filename, content_type):
|
||||
del storage_path, original_filename, content_type
|
||||
provenance_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
provenance_path.write_bytes(content)
|
||||
return _storage_info(provenance_path.parent, content)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||
_persist_dataset_file,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_file",
|
||||
_persist_file,
|
||||
)
|
||||
|
||||
result = DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="grb-lambert.geojson",
|
||||
content=raw,
|
||||
source="grb_wfs",
|
||||
source_name="grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={"adapter": "test"},
|
||||
temporal_series_key="grb:lambert:2026-08",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01-lambert",
|
||||
temporal_granularity="snapshot",
|
||||
)
|
||||
|
||||
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
|
||||
dataset_version = db.rows[DatasetVersion][0]
|
||||
snapshot = db.rows[SourceSnapshot][0]
|
||||
canonical_bytes = Path(str(dataset.storage_path)).read_bytes()
|
||||
canonical_payload = json.loads(canonical_bytes)
|
||||
source_artifact = dataset.provenance_metadata["source_artifact"]
|
||||
|
||||
assert result.status == "ready"
|
||||
assert dataset.crs == "EPSG:4326"
|
||||
assert canonical_payload["crs"]["properties"]["name"] == "EPSG:4326"
|
||||
assert canonical_payload["features"][0]["geometry"]["coordinates"][0][0] == pytest.approx(
|
||||
[longitude, latitude], abs=0.000001
|
||||
)
|
||||
assert sha256(canonical_bytes).hexdigest() == dataset.checksum_sha256
|
||||
assert dataset_version.checksum_sha256 == dataset.checksum_sha256
|
||||
assert snapshot.checksum_sha256 == dataset.checksum_sha256
|
||||
assert source_artifact["retention"] == "provenance_evidence_only"
|
||||
assert source_artifact["checksum_sha256"] == sha256(raw).hexdigest()
|
||||
assert source_artifact["storage_path"] != dataset.storage_path
|
||||
assert Path(source_artifact["storage_path"]).read_bytes() == raw
|
||||
assert dataset.provenance_metadata["canonical_consumption_artifact"] == {
|
||||
"checksum_sha256": dataset.checksum_sha256,
|
||||
"crs": "EPSG:4326",
|
||||
"storage_role": "dataset_consumption",
|
||||
}
|
||||
|
||||
inspection = VectorOperationsService.inspect(db, dataset.id)
|
||||
assert inspection.crs == "EPSG:4326"
|
||||
assert inspection.bounds_json == {
|
||||
"min_x": pytest.approx(longitude, abs=0.000001),
|
||||
"min_y": pytest.approx(latitude, abs=0.000001),
|
||||
"max_x": pytest.approx(max_longitude, abs=0.000001),
|
||||
"max_y": pytest.approx(max_latitude, abs=0.000001),
|
||||
}
|
||||
response_payload = DatasetService.get_dataset_geojson(db, dataset.id)
|
||||
assert response_payload["features"][0]["geometry"]["coordinates"][0][0] == pytest.approx(
|
||||
[longitude, latitude], abs=0.000001
|
||||
)
|
||||
|
||||
# The storage identity is enforced at the operation boundary too; a
|
||||
# replacement with different canonical bytes is not silently processed.
|
||||
Path(str(dataset.storage_path)).write_bytes(canonical_bytes + b"\n")
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
VectorOperationsService.inspect(db, dataset.id)
|
||||
assert exc_info.value.code == "DATASET_STORAGE_CHECKSUM_MISMATCH"
|
||||
|
||||
|
||||
def test_metadata_refresh_refuses_mutated_governed_artifact(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""A passed snapshot cannot be silently re-described from mutable storage."""
|
||||
|
||||
project = Project(id=uuid4(), name="Phase 2 immutable refresh")
|
||||
db = _Session(project)
|
||||
raw = _valid_payload()
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||
lambda **_kwargs: _storage_info(tmp_path, raw),
|
||||
)
|
||||
result = DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="grb-buildings.geojson",
|
||||
content=raw,
|
||||
source="grb_wfs",
|
||||
source_name="grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={"adapter": "test"},
|
||||
temporal_series_key="grb:2026-08",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01",
|
||||
temporal_granularity="snapshot",
|
||||
)
|
||||
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
|
||||
original_checksum = dataset.checksum_sha256
|
||||
original_metadata = dict(dataset.metadata_json or {})
|
||||
original_commit_count = db.commits
|
||||
|
||||
# Simulate an out-of-band storage replacement at the same path. The
|
||||
# refresh endpoint must not parse it into an already-passed contract row.
|
||||
Path(str(dataset.storage_path)).write_bytes(_grb_payload_without_required_id())
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetService.refresh_metadata(db, dataset.id)
|
||||
|
||||
assert exc_info.value.code == "GOVERNED_DATASET_REINGEST_REQUIRED"
|
||||
assert dataset.status == "ready"
|
||||
assert dataset.validation_status == "passed"
|
||||
assert dataset.checksum_sha256 == original_checksum
|
||||
assert dataset.metadata_json == original_metadata
|
||||
assert db.commits == original_commit_count
|
||||
|
||||
|
||||
def test_governed_import_quarantines_bad_artifacts_and_refuses_unknown_source(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
project = Project(id=uuid4(), name="Phase 2 quarantine")
|
||||
db = _Session(project)
|
||||
raw = b'{"type":"FeatureCollection","features":[]}'
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||
lambda **_kwargs: _storage_info(tmp_path, raw),
|
||||
)
|
||||
|
||||
quarantined = DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="empty.geojson",
|
||||
content=raw,
|
||||
source="grb_wfs",
|
||||
source_name="grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={},
|
||||
temporal_series_key="grb:2026-08-empty",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01-empty",
|
||||
temporal_granularity="snapshot",
|
||||
)
|
||||
assert quarantined.status == "quarantined"
|
||||
assert quarantined.validation_status == "failed"
|
||||
assert quarantined.quarantine_status == "quarantined"
|
||||
assert len(db.rows[DatasetQuarantine]) == 1
|
||||
assert db.rows[SourceSnapshot][0].ingest_status == "quarantined"
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="unregistered.geojson",
|
||||
content=_valid_payload(),
|
||||
source="caller_controlled",
|
||||
source_name="caller_claimed_grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={},
|
||||
)
|
||||
assert exc_info.value.code == "SOURCE_REGISTRY_ENTRY_NOT_FOUND"
|
||||
|
||||
|
||||
def test_governed_grb_vector_quarantines_missing_server_owned_required_attribute(
|
||||
monkeypatch, tmp_path: Path
|
||||
) -> None:
|
||||
project = Project(id=uuid4(), name="Phase 2 source schema")
|
||||
db = _Session(project)
|
||||
raw = _grb_payload_without_required_id()
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||
lambda **kwargs: _storage_info(tmp_path, kwargs["content"]),
|
||||
)
|
||||
|
||||
quarantined = DatasetService.import_vector_bytes(
|
||||
db,
|
||||
project_id=project.id,
|
||||
filename="grb-missing-id.geojson",
|
||||
content=raw,
|
||||
source="grb_wfs",
|
||||
source_name="grb",
|
||||
dataset_role="reference",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={"adapter": "test"},
|
||||
temporal_series_key="grb:missing-id",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01-missing-id",
|
||||
temporal_granularity="snapshot",
|
||||
)
|
||||
|
||||
dataset = next(item for item in db.rows[Dataset] if item.id == quarantined.id)
|
||||
assert quarantined.status == "quarantined"
|
||||
assert dataset.validation_status == "failed"
|
||||
assert dataset.quarantine_status == "quarantined"
|
||||
issue = dataset.validation_report_json["issues"][0]
|
||||
assert issue["code"] == "SOURCE_SCHEMA_REQUIRED_ATTRIBUTE_MISSING"
|
||||
assert issue["category"] == "source_schema"
|
||||
assert len(db.rows[DatasetQuarantine]) == 1
|
||||
|
||||
|
||||
def test_partitioned_vector_ingest_is_idempotent_and_quarantines_noncanonical_partition_coordinates(
|
||||
monkeypatch,
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
project = Project(id=uuid4(), name="Partitioned governed ingest")
|
||||
area = Area(id=uuid4(), project_id=project.id, name="Partitioned AOI")
|
||||
db = _Session(project)
|
||||
db.rows[Area] = [area]
|
||||
|
||||
feature = {
|
||||
"type": "Feature",
|
||||
"id": "GBG.1",
|
||||
"properties": {"id": "GBG.1", "source_feature_id": "GBG.1"},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
|
||||
],
|
||||
},
|
||||
}
|
||||
partition_payload = {"type": "FeatureCollection", "features": [feature]}
|
||||
partition_path = tmp_path / "partition-01.geojson"
|
||||
partition_path.write_text(json.dumps(partition_payload), encoding="utf-8")
|
||||
artifact_payload = {
|
||||
"type": "FeatureCollection",
|
||||
"crs": "EPSG:4326",
|
||||
"features": [feature],
|
||||
}
|
||||
artifact_path = tmp_path / "grb-partitioned.geojson"
|
||||
artifact_raw = json.dumps(artifact_payload).encode("utf-8")
|
||||
artifact_path.write_bytes(artifact_raw)
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
|
||||
lambda **_kwargs: _storage_info(tmp_path, artifact_raw),
|
||||
)
|
||||
|
||||
result = DatasetService.import_partitioned_vector_artifact(
|
||||
db,
|
||||
project_id=project.id,
|
||||
area_id=area.id,
|
||||
artifact_path=artifact_path,
|
||||
partition_paths=[partition_path],
|
||||
original_filename="grb-partitioned.geojson",
|
||||
source="operator_official_import",
|
||||
dataset_role="reference",
|
||||
source_name="grb",
|
||||
reference_layer_name="buildings",
|
||||
metadata_json={
|
||||
"feature_count": 1,
|
||||
"crs": "EPSG:4326",
|
||||
"bounds_json": {
|
||||
"min_x": 4.69,
|
||||
"min_y": 51.09,
|
||||
"max_x": 4.70,
|
||||
"max_y": 51.10,
|
||||
},
|
||||
},
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={
|
||||
"artifact_sha256": sha256(artifact_raw).hexdigest(),
|
||||
"partition_checksums": {
|
||||
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
|
||||
},
|
||||
},
|
||||
temporal_series_key="grb:partitioned:test",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01",
|
||||
)
|
||||
|
||||
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
|
||||
assert result.status == "ready"
|
||||
assert dataset.validation_status == "passed"
|
||||
assert dataset.provenance_status == "complete"
|
||||
assert dataset.source_name == "grb"
|
||||
assert len(db.rows[SourceSnapshot]) == 1
|
||||
assert len(db.rows[VectorFeature]) == 1
|
||||
assert dataset.metadata_json["partitioned_geometry_audit"][
|
||||
"partition_checksums_sha256"
|
||||
] == {partition_path.name: sha256(partition_path.read_bytes()).hexdigest()}
|
||||
assert dataset.provenance_metadata["partition_checksum_manifest_sha256"]
|
||||
assert dataset.provenance_metadata["partitioned_artifact_binding_sha256"]
|
||||
|
||||
repeated = DatasetService.import_partitioned_vector_artifact(
|
||||
db,
|
||||
project_id=project.id,
|
||||
area_id=area.id,
|
||||
artifact_path=artifact_path,
|
||||
partition_paths=[partition_path],
|
||||
original_filename="grb-partitioned.geojson",
|
||||
source="operator_official_import",
|
||||
dataset_role="reference",
|
||||
source_name="grb",
|
||||
reference_layer_name="buildings",
|
||||
metadata_json={
|
||||
"feature_count": 1,
|
||||
"crs": "EPSG:4326",
|
||||
"bounds_json": {
|
||||
"min_x": 4.69,
|
||||
"min_y": 51.09,
|
||||
"max_x": 4.70,
|
||||
"max_y": 51.10,
|
||||
},
|
||||
},
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={
|
||||
"artifact_sha256": sha256(artifact_raw).hexdigest(),
|
||||
"partition_checksums": {
|
||||
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
|
||||
},
|
||||
},
|
||||
temporal_series_key="grb:partitioned:test",
|
||||
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||
source_version="2026-08-01",
|
||||
)
|
||||
assert repeated.id == result.id
|
||||
assert len(db.rows[Dataset]) == 1
|
||||
assert len(db.rows[VectorFeature]) == 1
|
||||
|
||||
lambert_feature = {
|
||||
**feature,
|
||||
"id": "GBG.lambert",
|
||||
"properties": {"id": "GBG.lambert"},
|
||||
"geometry": {
|
||||
"type": "Polygon",
|
||||
"coordinates": [
|
||||
[[150000, 170000], [150010, 170000], [150010, 170010], [150000, 170000]]
|
||||
],
|
||||
},
|
||||
}
|
||||
lambert_partition = tmp_path / "partition-lambert.geojson"
|
||||
lambert_partition.write_text(
|
||||
json.dumps({"type": "FeatureCollection", "features": [lambert_feature]}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
lambert_artifact = tmp_path / "grb-lambert.geojson"
|
||||
lambert_raw = json.dumps(
|
||||
{"type": "FeatureCollection", "features": [lambert_feature]}
|
||||
).encode("utf-8")
|
||||
lambert_artifact.write_bytes(lambert_raw)
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
|
||||
lambda **_kwargs: _storage_info(tmp_path, lambert_raw),
|
||||
)
|
||||
|
||||
quarantined = DatasetService.import_partitioned_vector_artifact(
|
||||
db,
|
||||
project_id=project.id,
|
||||
area_id=area.id,
|
||||
artifact_path=lambert_artifact,
|
||||
partition_paths=[lambert_partition],
|
||||
original_filename="grb-lambert.geojson",
|
||||
source="operator_official_import",
|
||||
dataset_role="reference",
|
||||
source_name="grb",
|
||||
reference_layer_name="buildings",
|
||||
metadata_json={
|
||||
"feature_count": 1,
|
||||
"crs": "EPSG:4326",
|
||||
"bounds_json": {
|
||||
"min_x": 150000,
|
||||
"min_y": 170000,
|
||||
"max_x": 150010,
|
||||
"max_y": 170010,
|
||||
},
|
||||
},
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={
|
||||
"artifact_sha256": sha256(lambert_raw).hexdigest(),
|
||||
"partition_checksums": {
|
||||
lambert_partition.name: sha256(
|
||||
lambert_partition.read_bytes()
|
||||
).hexdigest()
|
||||
},
|
||||
},
|
||||
temporal_series_key="grb:partitioned:lambert",
|
||||
observed_at=datetime(2026, 8, 2, tzinfo=UTC),
|
||||
source_version="2026-08-02",
|
||||
)
|
||||
assert quarantined.status == "quarantined"
|
||||
assert quarantined.validation_status == "failed"
|
||||
assert quarantined.quarantine_status == "quarantined"
|
||||
|
||||
missing_manifest_partition = tmp_path / "partition-missing-manifest.geojson"
|
||||
missing_manifest_partition.write_text(
|
||||
json.dumps(partition_payload), encoding="utf-8"
|
||||
)
|
||||
missing_manifest_artifact = tmp_path / "grb-missing-manifest.geojson"
|
||||
missing_manifest_raw = json.dumps(
|
||||
{"type": "FeatureCollection", "features": [feature]}
|
||||
).encode("utf-8")
|
||||
missing_manifest_artifact.write_bytes(missing_manifest_raw)
|
||||
monkeypatch.setattr(
|
||||
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
|
||||
lambda **_kwargs: _storage_info(tmp_path, missing_manifest_raw),
|
||||
)
|
||||
|
||||
missing_manifest = DatasetService.import_partitioned_vector_artifact(
|
||||
db,
|
||||
project_id=project.id,
|
||||
area_id=area.id,
|
||||
artifact_path=missing_manifest_artifact,
|
||||
partition_paths=[missing_manifest_partition],
|
||||
original_filename="grb-missing-manifest.geojson",
|
||||
source="operator_official_import",
|
||||
dataset_role="reference",
|
||||
source_name="grb",
|
||||
reference_layer_name="buildings",
|
||||
metadata_json={
|
||||
"feature_count": 1,
|
||||
"crs": "EPSG:4326",
|
||||
"bounds_json": {
|
||||
"min_x": 4.69,
|
||||
"min_y": 51.09,
|
||||
"max_x": 4.70,
|
||||
"max_y": 51.10,
|
||||
},
|
||||
},
|
||||
source_metadata={"license": "Open data"},
|
||||
provenance_metadata={
|
||||
"artifact_sha256": sha256(missing_manifest_raw).hexdigest()
|
||||
},
|
||||
temporal_series_key="grb:partitioned:missing-manifest",
|
||||
observed_at=datetime(2026, 8, 3, tzinfo=UTC),
|
||||
source_version="2026-08-03",
|
||||
)
|
||||
assert missing_manifest.status == "quarantined"
|
||||
assert (
|
||||
missing_manifest.validation_report_json["issues"][0]["code"]
|
||||
== "PARTITION_CHECKSUM_MANIFEST_REQUIRED"
|
||||
)
|
||||
|
||||
|
||||
def test_partitioned_geometry_audit_handles_more_than_generic_topology_limit_without_materializing_geometries(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
feature_count = 10_001
|
||||
partition_path = tmp_path / "large-partition.geojson"
|
||||
partition_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": f"GBG.{index}",
|
||||
"properties": {"id": f"GBG.{index}"},
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [4.0 + index / 10_000_000, 51.0],
|
||||
},
|
||||
}
|
||||
for index in range(feature_count)
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
audit = _PartitionedGeoJsonRecords(
|
||||
[partition_path],
|
||||
expected_feature_count=feature_count,
|
||||
declared_partition_checksums={
|
||||
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
|
||||
},
|
||||
).audit()
|
||||
|
||||
assert audit.feature_count == feature_count
|
||||
assert audit.bounds_json["min_x"] == 4.0
|
||||
assert audit.bounds_json["max_x"] > audit.bounds_json["min_x"]
|
||||
assert audit.representative_record.geometry.geom_type == "MultiPoint"
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
|
||||
from PIL import Image
|
||||
@@ -13,35 +14,114 @@ SCRIPT = Path(__file__).parents[2] / "scripts" / "build_grayscale_yolo_dataset.p
|
||||
|
||||
def test_grayscale_builder_preserves_labels_and_split(tmp_path: Path) -> None:
|
||||
source = tmp_path / "source"
|
||||
(source / "images" / "train").mkdir(parents=True)
|
||||
(source / "labels" / "train").mkdir(parents=True)
|
||||
image = source / "images" / "train" / "tile.png"
|
||||
label = source / "labels" / "train" / "tile.txt"
|
||||
Image.new("RGB", (8, 8), (255, 0, 0)).save(image)
|
||||
label.write_text("0 0.5 0.5 0.5 0.5\n", encoding="utf-8")
|
||||
entries = []
|
||||
for split, sample_slug, colour in (("train", "fixture-train", (255, 0, 0)), ("val", "fixture-val", (0, 255, 0))):
|
||||
image = source / "images" / split / f"{sample_slug}.png"
|
||||
label = source / "labels" / split / f"{sample_slug}.txt"
|
||||
image.parent.mkdir(parents=True, exist_ok=True)
|
||||
label.parent.mkdir(parents=True, exist_ok=True)
|
||||
Image.new("RGB", (8, 8), colour).save(image)
|
||||
label.write_text("0 0.5 0.5 0.5 0.5\n", encoding="utf-8")
|
||||
entries.append((split, sample_slug, image, label))
|
||||
manifest = source / "operator_samples_manifest.json"
|
||||
policy = "geointel-training-source-eligibility/v1"
|
||||
manifest.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"immutable": True,
|
||||
"training_eligibility": {"policy_version": policy, "status": "eligible", "fixture_mode": True},
|
||||
"samples": [
|
||||
{
|
||||
"sample_slug": sample_slug,
|
||||
"split": split,
|
||||
"raster_dataset_id": f"raster:{sample_slug}",
|
||||
"reference_dataset_id": f"reference:{sample_slug}",
|
||||
"training_eligibility": {
|
||||
"policy_version": policy,
|
||||
"eligible": True,
|
||||
"fixture_mode": True,
|
||||
"raster": {"eligible": True, "reasons": [], "evidence": {"dataset_id": f"raster:{sample_slug}", "checksum_sha256": "a" * 64, "source_registry_id": "fixture-raster", "source_snapshot_id": "fixture-raster-snapshot"}},
|
||||
"reference": {"eligible": True, "reasons": [], "evidence": {"dataset_id": f"reference:{sample_slug}", "checksum_sha256": "b" * 64, "source_registry_id": "fixture-reference", "source_snapshot_id": "fixture-reference-snapshot"}},
|
||||
},
|
||||
}
|
||||
for split, sample_slug, _image, _label in entries
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(source / "corpus-freeze.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": 2,
|
||||
"immutable": True,
|
||||
"fixture_mode": True,
|
||||
"training_eligibility_policy": policy,
|
||||
"manifest_sha256": sha256(manifest.read_bytes()).hexdigest(),
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
dataset_yaml = source / "dataset.yaml"
|
||||
dataset_yaml.write_text(
|
||||
f"path: {source}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
release_script = SCRIPT.parent / "training_release_manifest.py"
|
||||
subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(release_script),
|
||||
"create",
|
||||
"--train-yaml",
|
||||
str(dataset_yaml),
|
||||
"--corpus-manifest",
|
||||
str(manifest),
|
||||
"--fixture-mode",
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
release_path = dataset_yaml.with_name(dataset_yaml.name + ".geointel-training-release.json")
|
||||
asset_path = dataset_yaml.with_name(dataset_yaml.name + ".geointel-training-assets.json")
|
||||
assets = json.loads(asset_path.read_text(encoding="utf-8"))
|
||||
summary = source / "yolo_tile_dataset_summary.json"
|
||||
summary.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"dataset_yaml": str(dataset_yaml.resolve()),
|
||||
"training_release_manifest": str(release_path.resolve()),
|
||||
"training_release_manifest_sha256": sha256(release_path.read_bytes()).hexdigest(),
|
||||
"training_asset_manifest": str(asset_path.resolve()),
|
||||
"source_manifest_sha256": sha256(manifest.read_bytes()).hexdigest(),
|
||||
"tiles": [
|
||||
{
|
||||
"split": "train",
|
||||
"image_path": str(image),
|
||||
"label_path": str(label),
|
||||
}
|
||||
]
|
||||
{"split": entry["split"], "image_path": entry["image_path"], "label_path": entry["label_path"]}
|
||||
for entry in assets["entries"]
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
output = tmp_path / "gray"
|
||||
subprocess.run(
|
||||
[sys.executable, str(SCRIPT), "--summary", str(summary), "--output-dir", str(output)],
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
"--summary",
|
||||
str(summary),
|
||||
"--train-yaml",
|
||||
str(dataset_yaml),
|
||||
"--corpus-manifest",
|
||||
str(manifest),
|
||||
"--fixture-mode",
|
||||
"--output-dir",
|
||||
str(output),
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
converted = Image.open(output / "images" / "train" / "tile.png")
|
||||
converted = Image.open(output / "images" / "train" / "fixture-train.png")
|
||||
r, g, b = converted.getpixel((0, 0))
|
||||
assert r == g == b
|
||||
assert (output / "labels" / "train" / "tile.txt").read_text() == label.read_text()
|
||||
assert (output / "labels" / "train" / "fixture-train.txt").read_text() == "0 0.5 0.5 0.5 0.5\n"
|
||||
evidence = json.loads((output / "grayscale-dataset-evidence.json").read_text())
|
||||
assert evidence["converted_tile_count"] == 1
|
||||
assert evidence["converted_tile_count"] == 2
|
||||
assert evidence["training_eligible"] is False
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
@@ -10,9 +11,10 @@ from fastapi.testclient import TestClient
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import AppError
|
||||
from app.main import app
|
||||
from app.models import AnalysisRun, Dataset, Detection, Job, Project
|
||||
from app.models import AnalysisRun, Dataset, Detection, Job, Project, SourceRegistry, SourceSnapshot
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||
|
||||
|
||||
class FakeSession:
|
||||
@@ -63,15 +65,47 @@ class MockYoloAdapter:
|
||||
def _project_and_raster_dataset():
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
source_registry_id = uuid4()
|
||||
source_snapshot_id = uuid4()
|
||||
checksum = "a" * 64
|
||||
project = Project(id=project_id, name="Geel")
|
||||
source_registry = SourceRegistry(
|
||||
id=source_registry_id,
|
||||
source_key="test-derived-raster",
|
||||
display_name="Governed test-derived raster",
|
||||
classification="derived",
|
||||
authority_name="GeoIntel test fixture",
|
||||
usage_policy_json={"ground_truth_allowed": False},
|
||||
)
|
||||
source_snapshot = SourceSnapshot(
|
||||
id=source_snapshot_id,
|
||||
source_registry_id=source_registry_id,
|
||||
snapshot_key="test-derived-raster-v1",
|
||||
checksum_sha256=checksum,
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
)
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name="source.tif",
|
||||
dataset_type="raster",
|
||||
source="user_upload",
|
||||
source="test-derived-raster",
|
||||
source_name="test-derived-raster",
|
||||
storage_path="storage/uploads/source.tif",
|
||||
checksum_sha256=checksum,
|
||||
source_registry_id=source_registry_id,
|
||||
source_snapshot_id=source_snapshot_id,
|
||||
data_contract_key="geointel.raster.geotiff",
|
||||
data_contract_version="1.0.0",
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="not_applicable",
|
||||
quarantine_status="not_quarantined",
|
||||
status="ready",
|
||||
)
|
||||
dataset.source_registry = source_registry
|
||||
dataset.source_snapshot = source_snapshot
|
||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||
return db, project_id, dataset_id
|
||||
|
||||
@@ -101,6 +135,72 @@ def _manifest(tmp_path: Path) -> Path:
|
||||
return manifest_path
|
||||
|
||||
|
||||
def _write_model_sidecar(
|
||||
model_path: Path,
|
||||
settings: Settings,
|
||||
*,
|
||||
db: FakeSession | None = None,
|
||||
) -> None:
|
||||
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
|
||||
source_registry_id = uuid4()
|
||||
source_snapshot_id = uuid4()
|
||||
source_version = settings.yolo_model_version or "test-v1"
|
||||
if db is not None:
|
||||
source_registry = SourceRegistry(
|
||||
id=source_registry_id,
|
||||
source_key="model",
|
||||
display_name="Governed test model artifact",
|
||||
classification="experimental",
|
||||
authority_name="GeoIntel test fixture",
|
||||
freshness_status="current",
|
||||
ingest_status="configured",
|
||||
)
|
||||
source_snapshot = SourceSnapshot(
|
||||
id=source_snapshot_id,
|
||||
source_registry_id=source_registry_id,
|
||||
snapshot_key=f"model-{source_version}",
|
||||
source_version=source_version,
|
||||
checksum_sha256=model_sha256,
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
)
|
||||
db.objects[(SourceRegistry, source_registry_id)] = source_registry
|
||||
db.objects[(SourceSnapshot, source_snapshot_id)] = source_snapshot
|
||||
payload = {
|
||||
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
|
||||
"data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"},
|
||||
"model": {
|
||||
"model_id": settings.yolo_model_id,
|
||||
"task_type": "object_detection",
|
||||
"sha256": model_sha256,
|
||||
"model_format": "pytorch",
|
||||
"framework": "ultralytics/pytorch",
|
||||
"class_mapping": {"0": "building"},
|
||||
"source_version": source_version,
|
||||
},
|
||||
"source": {
|
||||
"source_registry_id": str(source_registry_id),
|
||||
"source_snapshot_id": str(source_snapshot_id),
|
||||
"source_registry_key": "model",
|
||||
"source_snapshot_checksum_sha256": model_sha256,
|
||||
},
|
||||
"lineage": {
|
||||
"upstream_asset_ids": ["test-training-corpus"],
|
||||
"upstream_checksums_sha256": ["a" * 64],
|
||||
"transformations": [
|
||||
{"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64}
|
||||
],
|
||||
},
|
||||
"metadata": {"training_manifest_sha256": "c" * 64},
|
||||
"imported_at": "2026-08-01T10:00:00+00:00",
|
||||
}
|
||||
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
|
||||
RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text(
|
||||
json.dumps(payload, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def test_model_asset_catalog_lists_supported_local_model_files(tmp_path: Path) -> None:
|
||||
model_file = tmp_path / "building-detector.pt"
|
||||
model_file.write_bytes(b"local model")
|
||||
@@ -190,6 +290,7 @@ def test_detection_run_persists_selected_model_asset_parameters(tmp_path: Path)
|
||||
yolo_models_dir=str(tmp_path),
|
||||
yolo_max_tiles=4,
|
||||
)
|
||||
_write_model_sidecar(model_file, settings, db=db)
|
||||
|
||||
result = DetectionService.run_detection(
|
||||
db=db,
|
||||
|
||||
@@ -0,0 +1,644 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
from app.models import Dataset, DatasetVersion, SourceRegistry, SourceSnapshot
|
||||
from app.services.data_contract_validation import (
|
||||
build_vector_ingest_input,
|
||||
validate_registered_asset,
|
||||
)
|
||||
from app.services.demo_workflow_service import DemoWorkflowService
|
||||
from app.services.derived_dataset_governance_service import (
|
||||
DerivedDatasetGovernanceService,
|
||||
)
|
||||
from app.services.raster_operations_service import RasterOperationsService
|
||||
from app.services.storage_service import StorageService
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
from app.services.vector_operations_service import VectorOperationsService
|
||||
|
||||
|
||||
_CHECKSUM = "a" * 64
|
||||
|
||||
|
||||
class _FakeSession:
|
||||
def __init__(self, rows=None) -> None:
|
||||
self.rows = rows or {}
|
||||
self.added = []
|
||||
|
||||
def get(self, model, row_id):
|
||||
row = self.rows.get((model, row_id))
|
||||
if row is not None:
|
||||
return row
|
||||
return next(
|
||||
(
|
||||
item
|
||||
for item in self.added
|
||||
if isinstance(item, model) and item.id == row_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
def add(self, row) -> None:
|
||||
self.added.append(row)
|
||||
|
||||
def commit(self) -> None:
|
||||
return None
|
||||
|
||||
def refresh(self, row) -> None:
|
||||
return None
|
||||
|
||||
|
||||
class _GovernedSession:
|
||||
"""Small ORM-shaped session for the real governance branch.
|
||||
|
||||
Registry persistence is monkeypatched below; the test exercises the
|
||||
service's orchestration and report decisions without needing PostGIS.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.added = []
|
||||
self.flushes = 0
|
||||
|
||||
class _EmptyQuery:
|
||||
def filter(self, *_args, **_kwargs):
|
||||
return self
|
||||
|
||||
@staticmethod
|
||||
def all():
|
||||
return []
|
||||
|
||||
@staticmethod
|
||||
def one_or_none():
|
||||
return None
|
||||
|
||||
def query(self, *_args, **_kwargs):
|
||||
# Governance now performs a bounded lineage traversal during quarantine.
|
||||
# This focused harness intentionally has no persisted siblings/edges.
|
||||
return self._EmptyQuery()
|
||||
|
||||
def add(self, row) -> None:
|
||||
self.added.append(row)
|
||||
|
||||
def flush(self) -> None:
|
||||
self.flushes += 1
|
||||
|
||||
|
||||
def _governed_parent() -> Dataset:
|
||||
source_id = uuid4()
|
||||
snapshot_id = uuid4()
|
||||
source = SourceRegistry(
|
||||
id=source_id,
|
||||
source_key="grb",
|
||||
display_name="GRB parent fixture",
|
||||
classification="authoritative",
|
||||
authority_name="Digitaal Vlaanderen",
|
||||
authority_scope_json={"zone": "Flanders"},
|
||||
usage_policy_json={"ground_truth_allowed": True},
|
||||
)
|
||||
snapshot = SourceSnapshot(
|
||||
id=snapshot_id,
|
||||
source_registry_id=source_id,
|
||||
snapshot_key="governed-parent",
|
||||
checksum_sha256=_CHECKSUM,
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
)
|
||||
dataset = Dataset(
|
||||
id=uuid4(),
|
||||
project_id=uuid4(),
|
||||
name="governed.geojson",
|
||||
dataset_type="vector",
|
||||
source="grb",
|
||||
source_name="grb",
|
||||
status="ready",
|
||||
checksum_sha256=_CHECKSUM,
|
||||
data_contract_key="geointel.vector.geojson",
|
||||
data_contract_version="1.0.0",
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="not_applicable",
|
||||
quarantine_status="not_quarantined",
|
||||
source_registry_id=source_id,
|
||||
source_snapshot_id=snapshot_id,
|
||||
)
|
||||
dataset.source_registry = source
|
||||
dataset.source_snapshot = snapshot
|
||||
return dataset
|
||||
|
||||
|
||||
def test_lineage_evidence_quarantines_ungoverned_parent_without_inventing_a_checksum() -> (
|
||||
None
|
||||
):
|
||||
parent = _governed_parent()
|
||||
valid_lineage = DerivedDatasetGovernanceService._lineage_evidence(
|
||||
parent, "vector.clip", {"area_id": "a"}
|
||||
)
|
||||
valid_report = validate_registered_asset(
|
||||
build_vector_ingest_input(
|
||||
asset_id="derived-valid",
|
||||
source_crs="EPSG:4326",
|
||||
storage_crs="EPSG:4326",
|
||||
feature_collection={
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||
"properties": {},
|
||||
}
|
||||
],
|
||||
},
|
||||
checksum_sha256=_CHECKSUM,
|
||||
computed_checksum_sha256=_CHECKSUM,
|
||||
source_registry_id="derived-source",
|
||||
source_snapshot_id="derived-snapshot",
|
||||
imported_at=datetime.now(timezone.utc),
|
||||
metadata={
|
||||
"license": "internal derived artifact",
|
||||
"bounds_json": {
|
||||
"min_x": 5.0,
|
||||
"min_y": 51.0,
|
||||
"max_x": 5.0,
|
||||
"max_y": 51.0,
|
||||
},
|
||||
},
|
||||
temporal_unknown_reason="derived input has no precise observation timestamp",
|
||||
source_version_unknown_reason="transform version is recorded separately",
|
||||
lineage=valid_lineage,
|
||||
)
|
||||
)
|
||||
|
||||
assert valid_report.validation_status.value == "passed"
|
||||
assert valid_lineage.upstream_asset_ids == (str(parent.id),)
|
||||
assert valid_lineage.upstream_checksums_sha256 == (_CHECKSUM,)
|
||||
|
||||
parent.validation_status = "not_validated"
|
||||
rejected_lineage = DerivedDatasetGovernanceService._lineage_evidence(
|
||||
parent, "vector.clip", {}
|
||||
)
|
||||
rejected_report = validate_registered_asset(
|
||||
build_vector_ingest_input(
|
||||
asset_id="derived-rejected",
|
||||
source_crs="EPSG:4326",
|
||||
storage_crs="EPSG:4326",
|
||||
feature_collection={
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||
"properties": {},
|
||||
}
|
||||
],
|
||||
},
|
||||
checksum_sha256=_CHECKSUM,
|
||||
computed_checksum_sha256=_CHECKSUM,
|
||||
source_registry_id="derived-source",
|
||||
source_snapshot_id="derived-snapshot",
|
||||
imported_at=datetime.now(timezone.utc),
|
||||
metadata={
|
||||
"license": "internal derived artifact",
|
||||
"bounds_json": {
|
||||
"min_x": 5.0,
|
||||
"min_y": 51.0,
|
||||
"max_x": 5.0,
|
||||
"max_y": 51.0,
|
||||
},
|
||||
},
|
||||
temporal_unknown_reason="derived input has no precise observation timestamp",
|
||||
source_version_unknown_reason="transform version is recorded separately",
|
||||
lineage=rejected_lineage,
|
||||
)
|
||||
)
|
||||
|
||||
assert rejected_lineage.upstream_checksums_sha256 == (
|
||||
"parent_dataset_not_governed",
|
||||
)
|
||||
assert rejected_report.validation_status.value == "failed"
|
||||
assert rejected_report.quarantine_status.value == "quarantined"
|
||||
assert any(
|
||||
issue.code == "UPSTREAM_CHECKSUM_FORMAT_INVALID"
|
||||
for issue in rejected_report.issues
|
||||
)
|
||||
|
||||
|
||||
def test_govern_vector_binds_snapshot_contract_and_lineage_before_marking_ready(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
from app.services.source_registry_service import SourceRegistryService
|
||||
|
||||
db = _GovernedSession()
|
||||
parent = _governed_parent()
|
||||
parent_version_id = uuid4()
|
||||
dataset = Dataset(
|
||||
id=uuid4(),
|
||||
project_id=parent.project_id,
|
||||
name="derived.geojson",
|
||||
dataset_type="vector",
|
||||
source="operation:clip",
|
||||
source_name="derived",
|
||||
checksum_sha256=_CHECKSUM,
|
||||
imported_at=datetime.now(timezone.utc),
|
||||
crs="EPSG:4326",
|
||||
metadata_json={
|
||||
"bounds_json": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.0, "max_y": 51.0}
|
||||
},
|
||||
status="validating",
|
||||
)
|
||||
version = DatasetVersion(
|
||||
id=uuid4(), dataset_id=dataset.id, version=1, checksum_sha256=_CHECKSUM
|
||||
)
|
||||
source = SimpleNamespace(id=uuid4(), license_name="internal derived artifact")
|
||||
snapshot = SimpleNamespace(id=uuid4(), source_registry_id=source.id)
|
||||
edges = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
SourceRegistryService,
|
||||
"ensure_server_owned_source",
|
||||
lambda *_args, **_kwargs: source,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
SourceRegistryService, "record_snapshot", lambda *_args, **_kwargs: snapshot
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
DerivedDatasetGovernanceService,
|
||||
"_latest_parent_version_id",
|
||||
lambda *_args: parent_version_id,
|
||||
)
|
||||
|
||||
def _bind(target, **kwargs):
|
||||
target.source_registry_id = kwargs["source"].id
|
||||
target.source_snapshot_id = kwargs["snapshot"].id
|
||||
target.data_contract_key = kwargs["data_contract_key"]
|
||||
target.data_contract_version = kwargs["data_contract_version"]
|
||||
target.validation_status = kwargs["validation_status"]
|
||||
target.provenance_status = kwargs["provenance_status"]
|
||||
target.lineage_status = kwargs["lineage_status"]
|
||||
return target
|
||||
|
||||
monkeypatch.setattr(SourceRegistryService, "bind_dataset_provenance", _bind)
|
||||
monkeypatch.setattr(SourceRegistryService, "bind_dataset_version_provenance", _bind)
|
||||
monkeypatch.setattr(
|
||||
SourceRegistryService,
|
||||
"record_lineage_edge",
|
||||
lambda *_args, **kwargs: edges.append(kwargs),
|
||||
)
|
||||
|
||||
ready = DerivedDatasetGovernanceService.govern_vector(
|
||||
db,
|
||||
dataset=dataset,
|
||||
dataset_version=version,
|
||||
feature_collection={
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||
"properties": {},
|
||||
}
|
||||
],
|
||||
},
|
||||
source_key="derived",
|
||||
operation="vector.clip",
|
||||
parent_dataset=parent,
|
||||
operation_parameters={"area_id": "a"},
|
||||
)
|
||||
|
||||
assert ready is True
|
||||
assert dataset.status == "ready"
|
||||
assert dataset.validation_status == "passed"
|
||||
assert dataset.provenance_status == "complete"
|
||||
assert dataset.source_registry_id == source.id
|
||||
assert version.source_snapshot_id == snapshot.id
|
||||
assert db.flushes >= 1
|
||||
assert edges[0]["parent_dataset_id"] == parent.id
|
||||
assert edges[0]["parent_dataset_version_id"] == parent_version_id
|
||||
assert edges[0]["child_dataset_version_id"] == version.id
|
||||
|
||||
|
||||
def test_govern_vector_quarantines_output_when_parent_is_manual_or_experimental(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
from app.services.source_registry_service import SourceRegistryService
|
||||
|
||||
db = _GovernedSession()
|
||||
parent = _governed_parent()
|
||||
parent.source = "manual"
|
||||
parent.source_name = "manual"
|
||||
parent.source_registry.source_key = "manual"
|
||||
parent.source_registry.classification = "experimental"
|
||||
parent.source_registry.usage_policy_json = {"ground_truth_allowed": False}
|
||||
dataset = Dataset(
|
||||
id=uuid4(),
|
||||
project_id=parent.project_id,
|
||||
name="manual-derived.geojson",
|
||||
dataset_type="vector",
|
||||
source="operation:clip",
|
||||
source_name="derived",
|
||||
checksum_sha256=_CHECKSUM,
|
||||
imported_at=datetime.now(timezone.utc),
|
||||
crs="EPSG:4326",
|
||||
metadata_json={
|
||||
"bounds_json": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.0, "max_y": 51.0}
|
||||
},
|
||||
status="validating",
|
||||
)
|
||||
version = DatasetVersion(
|
||||
id=uuid4(), dataset_id=dataset.id, version=1, checksum_sha256=_CHECKSUM
|
||||
)
|
||||
source = SimpleNamespace(id=uuid4(), license_name="internal derived artifact")
|
||||
snapshot = SimpleNamespace(
|
||||
id=uuid4(), source_registry_id=source.id, ingest_status="ingested"
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
SourceRegistryService,
|
||||
"ensure_server_owned_source",
|
||||
lambda *_args, **_kwargs: source,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
SourceRegistryService, "record_snapshot", lambda *_args, **_kwargs: snapshot
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
DerivedDatasetGovernanceService,
|
||||
"_latest_parent_version_id",
|
||||
lambda *_args: None,
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
SourceRegistryService, "record_lineage_edge", lambda *_args, **_kwargs: None
|
||||
)
|
||||
|
||||
ready = DerivedDatasetGovernanceService.govern_vector(
|
||||
db,
|
||||
dataset=dataset,
|
||||
dataset_version=version,
|
||||
feature_collection={
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||
"properties": {},
|
||||
}
|
||||
],
|
||||
},
|
||||
source_key="derived",
|
||||
operation="vector.clip",
|
||||
parent_dataset=parent,
|
||||
)
|
||||
|
||||
assert ready is False
|
||||
assert dataset.status == "quarantined"
|
||||
assert dataset.quarantine_status == "quarantined"
|
||||
assert dataset.validation_status == "failed"
|
||||
assert any(
|
||||
issue["code"] == "PARENT_DATASET_NOT_ELIGIBLE_FOR_DERIVED_PROCESSING"
|
||||
for issue in dataset.validation_report_json["issues"]
|
||||
)
|
||||
assert snapshot.ingest_status == "quarantined"
|
||||
|
||||
|
||||
def test_vector_selection_uses_map_selection_registry_and_skips_features_when_quarantined(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
source = _governed_parent()
|
||||
source.area_id = None
|
||||
source.storage_path = str(tmp_path / "source.geojson")
|
||||
db = _FakeSession({(Dataset, source.id): source})
|
||||
output_path = tmp_path / "selection.geojson"
|
||||
calls = []
|
||||
persisted_features = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
VectorFeatureService,
|
||||
"select_features_by_bbox",
|
||||
lambda *_args, **_kwargs: {
|
||||
"selection_bbox": {
|
||||
"min_x": 4.9,
|
||||
"min_y": 50.9,
|
||||
"max_x": 5.2,
|
||||
"max_y": 51.2,
|
||||
"crs": "EPSG:4326",
|
||||
},
|
||||
"feature_count": 1,
|
||||
"limit": 250,
|
||||
"truncated": False,
|
||||
"geojson": {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": "source-feature",
|
||||
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||
"properties": {
|
||||
"vector_feature_id": "source-feature",
|
||||
"dataset_id": str(source.id),
|
||||
},
|
||||
}
|
||||
],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
def _persist_dataset_file(**kwargs):
|
||||
output_path.write_bytes(kwargs["content"])
|
||||
return {
|
||||
"original_filename": kwargs["original_filename"],
|
||||
"stored_filename": output_path.name,
|
||||
"content_type": kwargs["content_type"],
|
||||
"size_bytes": len(kwargs["content"]),
|
||||
"checksum_sha256": _CHECKSUM,
|
||||
"storage_path": str(output_path),
|
||||
}
|
||||
|
||||
monkeypatch.setattr(StorageService, "persist_dataset_file", _persist_dataset_file)
|
||||
|
||||
def _quarantine(db, **kwargs):
|
||||
calls.append(kwargs)
|
||||
kwargs["dataset"].status = "quarantined"
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_vector", _quarantine)
|
||||
monkeypatch.setattr(
|
||||
VectorFeatureService,
|
||||
"persist_geojson_features",
|
||||
lambda **kwargs: persisted_features.append(kwargs),
|
||||
)
|
||||
|
||||
response = VectorOperationsService.derive_selection_dataset(
|
||||
db=db,
|
||||
dataset_id=source.id,
|
||||
bbox={
|
||||
"min_x": 4.9,
|
||||
"min_y": 50.9,
|
||||
"max_x": 5.2,
|
||||
"max_y": 51.2,
|
||||
"crs": "EPSG:4326",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status == "quarantined"
|
||||
assert calls[0]["source_key"] == "map_selection"
|
||||
assert calls[0]["parent_dataset"] is source
|
||||
assert calls[0]["operation"] == "vector.selection"
|
||||
assert persisted_features == []
|
||||
|
||||
|
||||
def test_vector_buffer_uses_projected_metres_instead_of_wgs84_degrees(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
source = _governed_parent()
|
||||
source.storage_path = str(tmp_path / "source.geojson")
|
||||
source.crs = "EPSG:4326"
|
||||
Path(source.storage_path).write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "FeatureCollection",
|
||||
"crs": "EPSG:4326",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||
"properties": {},
|
||||
},
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
# A governed consumption artifact must carry the checksum of these exact
|
||||
# bytes; vector operations deliberately refuse a stale fixture checksum.
|
||||
source.checksum_sha256 = sha256(Path(source.storage_path).read_bytes()).hexdigest()
|
||||
source.source_snapshot.checksum_sha256 = source.checksum_sha256
|
||||
db = _FakeSession({(Dataset, source.id): source})
|
||||
captured = {}
|
||||
|
||||
def _persist(**kwargs):
|
||||
captured.update(kwargs)
|
||||
return uuid4()
|
||||
|
||||
monkeypatch.setattr(VectorOperationsService, "_persist_derived_dataset", _persist)
|
||||
VectorOperationsService.buffer(
|
||||
db, source.id, distance_m=100.0, dissolve=False, output_name=None
|
||||
)
|
||||
|
||||
coordinates = captured["feature_collection"]["features"][0]["geometry"][
|
||||
"coordinates"
|
||||
][0]
|
||||
longitudes = [coordinate[0] for coordinate in coordinates]
|
||||
latitudes = [coordinate[1] for coordinate in coordinates]
|
||||
assert max(longitudes) - min(longitudes) < 0.01
|
||||
assert max(latitudes) - min(latitudes) < 0.01
|
||||
|
||||
|
||||
def test_raster_operation_uses_derived_registry_before_commit(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
source = _governed_parent()
|
||||
source.dataset_type = "raster"
|
||||
source.storage_path = str(tmp_path / "source.tif")
|
||||
output_path = tmp_path / "derived.tif"
|
||||
output_path.write_bytes(b"derived-raster")
|
||||
db = _FakeSession()
|
||||
calls = []
|
||||
|
||||
def _govern(db, **kwargs):
|
||||
calls.append(kwargs)
|
||||
kwargs["dataset"].status = "quarantined"
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_raster", _govern)
|
||||
|
||||
result = RasterOperationsService._persist_derived_dataset(
|
||||
db,
|
||||
source_dataset=source,
|
||||
source_dataset_id=source.id,
|
||||
operation="ndvi",
|
||||
output_path=str(output_path),
|
||||
output_name="derived.tif",
|
||||
metadata={
|
||||
"crs": "EPSG:31370",
|
||||
"bounds": [100000.0, 100000.0, 100001.0, 100001.0],
|
||||
"resolution": [1.0, 1.0],
|
||||
"width": 1,
|
||||
"height": 1,
|
||||
"band_count": 1,
|
||||
"dtype": ["float32"],
|
||||
"operation_parameters": {"nir_band": 4, "red_band": 3},
|
||||
},
|
||||
)
|
||||
|
||||
derived = next(item for item in db.added if isinstance(item, Dataset))
|
||||
assert result == derived.id
|
||||
assert derived.status == "quarantined"
|
||||
assert derived.source_name == "derived"
|
||||
assert calls[0]["source_key"] == "derived"
|
||||
assert calls[0]["parent_dataset"] is source
|
||||
assert calls[0]["operation"] == "raster.ndvi"
|
||||
|
||||
|
||||
def test_demo_fixture_creation_is_governed_and_does_not_persist_features_when_rejected(
|
||||
monkeypatch, tmp_path
|
||||
) -> None:
|
||||
project_id = uuid4()
|
||||
area_id = uuid4()
|
||||
db = _FakeSession()
|
||||
calls = []
|
||||
features = []
|
||||
|
||||
monkeypatch.setattr(
|
||||
StorageService,
|
||||
"persist_dataset_file",
|
||||
lambda **kwargs: {
|
||||
"storage_path": str(tmp_path / kwargs["original_filename"]),
|
||||
"original_filename": kwargs["original_filename"],
|
||||
"stored_filename": kwargs["original_filename"],
|
||||
"content_type": kwargs["content_type"],
|
||||
"size_bytes": len(kwargs["content"]),
|
||||
"checksum_sha256": _CHECKSUM,
|
||||
},
|
||||
)
|
||||
|
||||
def _quarantine(db, **kwargs):
|
||||
calls.append(kwargs)
|
||||
kwargs["dataset"].status = "quarantined"
|
||||
return False
|
||||
|
||||
monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_vector", _quarantine)
|
||||
monkeypatch.setattr(
|
||||
VectorFeatureService,
|
||||
"persist_geojson_features",
|
||||
lambda **kwargs: features.append(kwargs),
|
||||
)
|
||||
payload = {
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||
"properties": {},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
dataset = DemoWorkflowService._create_dataset(
|
||||
db,
|
||||
project_id=project_id,
|
||||
area_id=area_id,
|
||||
filename="fixture.geojson",
|
||||
payload=payload,
|
||||
raw=json.dumps(payload).encode("utf-8"),
|
||||
role="source",
|
||||
source_name="fixture",
|
||||
reference_layer_name=None,
|
||||
)
|
||||
|
||||
assert dataset.status == "quarantined"
|
||||
assert calls[0]["source_key"] == "fixture"
|
||||
assert calls[0]["operation"] == "demo.fixture_vector"
|
||||
assert features == []
|
||||
@@ -1,11 +1,11 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
from app.models import Area, Dataset
|
||||
from app.models import Dataset, SourceRegistry, SourceSnapshot
|
||||
from app.services.qa_service import QaService
|
||||
|
||||
|
||||
@@ -46,6 +46,47 @@ def _write_features(path: Path, features: list[dict]) -> None:
|
||||
path.write_text(json.dumps({"type": "FeatureCollection", "features": features}), encoding="utf-8")
|
||||
|
||||
|
||||
def _authoritative_reference(dataset: Dataset) -> Dataset:
|
||||
"""Give QA reference fixtures the same durable authority proof as GRB."""
|
||||
|
||||
source_id = uuid4()
|
||||
snapshot_id = uuid4()
|
||||
checksum = sha256(Path(str(dataset.storage_path)).read_bytes()).hexdigest()
|
||||
source = SourceRegistry(
|
||||
id=source_id,
|
||||
source_key="grb",
|
||||
display_name="GRB test reference",
|
||||
classification="authoritative",
|
||||
authority_name="Digitaal Vlaanderen",
|
||||
authority_scope_json={"zone": "Flanders"},
|
||||
usage_policy_json={"ground_truth_allowed": True, "validation_authority": {"building_validation": "primary"}},
|
||||
)
|
||||
snapshot = SourceSnapshot(
|
||||
id=snapshot_id,
|
||||
source_registry_id=source_id,
|
||||
snapshot_key=f"qa-grb-{dataset.id}",
|
||||
checksum_sha256=checksum,
|
||||
ingest_status="ingested",
|
||||
freshness_status="current",
|
||||
)
|
||||
dataset.source = "grb"
|
||||
dataset.source_name = "grb"
|
||||
dataset.dataset_role = "reference"
|
||||
dataset.status = "ready"
|
||||
dataset.checksum_sha256 = checksum
|
||||
dataset.source_registry_id = source_id
|
||||
dataset.source_snapshot_id = snapshot_id
|
||||
dataset.data_contract_key = "geointel.vector.geojson"
|
||||
dataset.data_contract_version = "1.0.0"
|
||||
dataset.validation_status = "passed"
|
||||
dataset.provenance_status = "complete"
|
||||
dataset.lineage_status = "complete"
|
||||
dataset.quarantine_status = "not_quarantined"
|
||||
dataset.source_registry = source
|
||||
dataset.source_snapshot = snapshot
|
||||
return dataset
|
||||
|
||||
|
||||
def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None:
|
||||
project_id = uuid4()
|
||||
candidate_id = uuid4()
|
||||
@@ -66,7 +107,7 @@ def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None:
|
||||
crs="EPSG:4326",
|
||||
metadata_json={"crs_assumed": False},
|
||||
)
|
||||
reference = Dataset(
|
||||
reference = _authoritative_reference(Dataset(
|
||||
id=reference_id,
|
||||
project_id=project_id,
|
||||
name="reference.geojson",
|
||||
@@ -75,7 +116,7 @@ def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None:
|
||||
storage_path=str(reference_path),
|
||||
crs="EPSG:4326",
|
||||
metadata_json={"crs_assumed": False},
|
||||
)
|
||||
))
|
||||
|
||||
result = QaService.compare_candidate_with_reference(
|
||||
db=FakeSession([candidate, reference]),
|
||||
@@ -117,7 +158,7 @@ def test_qa_compare_candidate_with_reference_returns_feature_level_evidence(tmp_
|
||||
crs="EPSG:4326",
|
||||
metadata_json={"crs_assumed": False},
|
||||
)
|
||||
reference = Dataset(
|
||||
reference = _authoritative_reference(Dataset(
|
||||
id=reference_id,
|
||||
project_id=project_id,
|
||||
name="reference.geojson",
|
||||
@@ -126,7 +167,7 @@ def test_qa_compare_candidate_with_reference_returns_feature_level_evidence(tmp_
|
||||
storage_path=str(reference_path),
|
||||
crs="EPSG:4326",
|
||||
metadata_json={"crs_assumed": False},
|
||||
)
|
||||
))
|
||||
|
||||
result = QaService.compare_candidate_with_reference(
|
||||
db=FakeSession([candidate, reference]),
|
||||
|
||||
@@ -49,6 +49,54 @@ def scope_area(name: str, geometry):
|
||||
return SimpleNamespace(name=name, geometry=geometry)
|
||||
|
||||
|
||||
def governed_materialization(
|
||||
*,
|
||||
source_name: str,
|
||||
reference_layer_name: str | None,
|
||||
source_metadata: dict[str, object],
|
||||
dataset_id=None,
|
||||
) -> SimpleNamespace:
|
||||
"""Build a complete authoritative materialization for coverage tests.
|
||||
|
||||
Coverage is a production-facing statement. These fixtures must therefore
|
||||
carry the same registry, immutable snapshot, checksum and freshness state
|
||||
that a materialized official dataset needs in production.
|
||||
"""
|
||||
|
||||
source_registry_id = uuid4()
|
||||
source_snapshot_id = uuid4()
|
||||
checksum_sha256 = "a" * 64
|
||||
return SimpleNamespace(
|
||||
id=dataset_id or uuid4(),
|
||||
status="ready",
|
||||
source=source_name,
|
||||
source_name=source_name,
|
||||
reference_layer_name=reference_layer_name,
|
||||
source_metadata=dict(source_metadata),
|
||||
checksum_sha256=checksum_sha256,
|
||||
source_registry_id=source_registry_id,
|
||||
source_snapshot_id=source_snapshot_id,
|
||||
data_contract_key="geointel.vector.geojson",
|
||||
data_contract_version="1.0.0",
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="not_applicable",
|
||||
quarantine_status="not_quarantined",
|
||||
source_registry=SimpleNamespace(
|
||||
source_key=source_name,
|
||||
classification="authoritative",
|
||||
authority_scope_json={"scope": "coverage test"},
|
||||
usage_policy_json={},
|
||||
),
|
||||
source_snapshot=SimpleNamespace(
|
||||
source_registry_id=source_registry_id,
|
||||
checksum_sha256=checksum_sha256,
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_coverage_catalog_uses_normalized_contracts_and_does_not_change_provider_registry() -> None:
|
||||
catalog = CoverageRegistryService.catalog()
|
||||
|
||||
@@ -135,9 +183,8 @@ def test_coverage_resolver_only_reports_operational_for_materialized_ready_datas
|
||||
assert without_materialized.items[0].materialized_dataset_ids == []
|
||||
|
||||
dataset_id = uuid4()
|
||||
materialized = SimpleNamespace(
|
||||
id=dataset_id,
|
||||
status="ready",
|
||||
materialized = governed_materialization(
|
||||
dataset_id=dataset_id,
|
||||
source_name="ngi_adminvector",
|
||||
reference_layer_name="belgium_regions",
|
||||
source_metadata={"coverage_zones": ["belgium", "flanders", "wallonia", "brussels"]},
|
||||
@@ -156,9 +203,8 @@ def test_coverage_resolver_only_reports_operational_for_materialized_ready_datas
|
||||
def test_statbel_population_materialization_does_not_masquerade_as_admin_data() -> None:
|
||||
project_id = uuid4()
|
||||
statbel_id = uuid4()
|
||||
statbel = SimpleNamespace(
|
||||
id=statbel_id,
|
||||
status="ready",
|
||||
statbel = governed_materialization(
|
||||
dataset_id=statbel_id,
|
||||
source_name="statbel",
|
||||
reference_layer_name="population",
|
||||
source_metadata={"coverage_zones": ["belgium", "flanders", "wallonia", "brussels"]},
|
||||
@@ -187,9 +233,8 @@ def test_statbel_population_materialization_does_not_masquerade_as_admin_data()
|
||||
def test_bounded_api_materialization_only_covers_its_persisted_bbox() -> None:
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
dataset = SimpleNamespace(
|
||||
id=dataset_id,
|
||||
status="ready",
|
||||
dataset = governed_materialization(
|
||||
dataset_id=dataset_id,
|
||||
source_name="spw_picc",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={
|
||||
@@ -230,8 +275,24 @@ def test_bounded_partition_union_can_be_operational() -> None:
|
||||
left_id = uuid4()
|
||||
right_id = uuid4()
|
||||
datasets = [
|
||||
SimpleNamespace(id=left_id, status="ready", source_name="spw_picc", reference_layer_name="buildings", source_metadata={"coverage_zones": ["wallonia"], "bbox_epsg4326": [4.50, 50.50, 4.60, 50.60]}),
|
||||
SimpleNamespace(id=right_id, status="ready", source_name="spw_picc", reference_layer_name="buildings", source_metadata={"coverage_zones": ["wallonia"], "bbox_epsg4326": [4.60, 50.50, 4.70, 50.60]}),
|
||||
governed_materialization(
|
||||
dataset_id=left_id,
|
||||
source_name="spw_picc",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={
|
||||
"coverage_zones": ["wallonia"],
|
||||
"bbox_epsg4326": [4.50, 50.50, 4.60, 50.60],
|
||||
},
|
||||
),
|
||||
governed_materialization(
|
||||
dataset_id=right_id,
|
||||
source_name="spw_picc",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={
|
||||
"coverage_zones": ["wallonia"],
|
||||
"bbox_epsg4326": [4.60, 50.50, 4.70, 50.60],
|
||||
},
|
||||
),
|
||||
]
|
||||
session = FakeSession(
|
||||
project=SimpleNamespace(id=project_id),
|
||||
@@ -252,9 +313,7 @@ def test_spw_bathymetry_materialization_is_source_specific() -> None:
|
||||
scope_area("Wallonia", box(2.5, 49.5, 6.4, 50.8)),
|
||||
]
|
||||
selection = CoverageBBox(minx=4.851, miny=50.451, maxx=4.869, maxy=50.469)
|
||||
spw_picc = SimpleNamespace(
|
||||
id=uuid4(),
|
||||
status="ready",
|
||||
spw_picc = governed_materialization(
|
||||
source_name="spw_picc",
|
||||
reference_layer_name="buildings",
|
||||
source_metadata={
|
||||
@@ -272,9 +331,8 @@ def test_spw_bathymetry_materialization_is_source_specific() -> None:
|
||||
assert without_bathymetry.items[0].materialized_dataset_ids == []
|
||||
|
||||
bathymetry_id = uuid4()
|
||||
bathymetry = SimpleNamespace(
|
||||
id=bathymetry_id,
|
||||
status="ready",
|
||||
bathymetry = governed_materialization(
|
||||
dataset_id=bathymetry_id,
|
||||
source_name="spw_bathymetry",
|
||||
reference_layer_name=None,
|
||||
source_metadata={
|
||||
@@ -309,9 +367,8 @@ def test_vha_bathymetry_profiles_are_operational_only_inside_the_persisted_selec
|
||||
assert without_profiles.items[0].source_names == ["vmm_vha_bathymetry_profiles"]
|
||||
|
||||
profile_id = uuid4()
|
||||
profiles = SimpleNamespace(
|
||||
id=profile_id,
|
||||
status="ready",
|
||||
profiles = governed_materialization(
|
||||
dataset_id=profile_id,
|
||||
source_name="vmm_vha_bathymetry_profiles",
|
||||
reference_layer_name="bathymetry_profile_points",
|
||||
source_metadata={
|
||||
@@ -360,9 +417,8 @@ def test_flemish_materialization_is_theme_specific() -> None:
|
||||
project_id = uuid4()
|
||||
project = SimpleNamespace(id=project_id)
|
||||
orthophoto_id = uuid4()
|
||||
orthophoto = SimpleNamespace(
|
||||
id=orthophoto_id,
|
||||
status="ready",
|
||||
orthophoto = governed_materialization(
|
||||
dataset_id=orthophoto_id,
|
||||
source_name="digitaal_vlaanderen_orthophoto",
|
||||
reference_layer_name="orthophoto",
|
||||
source_metadata={"coverage_zones": ["flanders"]},
|
||||
@@ -429,5 +485,5 @@ def test_frontend_prefers_materialized_national_workspace_and_resolves_drawn_bbo
|
||||
assert "externalApi.resolveCoverage" in coverage_hook
|
||||
assert "coverage.outside_supported_scope" in map_workspace
|
||||
assert "coverageStatusLabel" in map_workspace
|
||||
assert "coverageSelectionAvailable" in map_workspace
|
||||
assert "activeThemeSupportsCurrentSelection" in map_workspace
|
||||
assert "activeThemeAvailable && !regionalPartitionedThemeActive" in map_workspace
|
||||
|
||||
@@ -44,8 +44,10 @@ def _project_and_dataset():
|
||||
project_id=project_id,
|
||||
name="ortho.tif",
|
||||
dataset_type="raster",
|
||||
source="user_upload",
|
||||
source="fixture",
|
||||
source_name="fixture",
|
||||
storage_path="storage/uploads/ortho.tif",
|
||||
source_metadata={"fixture": True},
|
||||
)
|
||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||
return db, project_id, dataset_id
|
||||
|
||||
@@ -0,0 +1,325 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.models import DatasetQuarantine, SourceRegistry, SourceSnapshot
|
||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||
|
||||
|
||||
class FakeSession:
|
||||
"""Explicit database double for production-runtime provenance tests."""
|
||||
|
||||
def __init__(self, objects: dict[tuple[type, object], object] | None = None) -> None:
|
||||
self.objects = objects or {}
|
||||
|
||||
def get(self, model, item_id):
|
||||
return self.objects.get((model, item_id))
|
||||
|
||||
|
||||
def _write_sidecar(
|
||||
model_path: Path,
|
||||
*,
|
||||
model_id: str = "yolo-configured",
|
||||
task_type: str = "object_detection",
|
||||
framework: str = "ultralytics/pytorch",
|
||||
source_version: str = "test-v1",
|
||||
source_registry_id: str | None = None,
|
||||
source_snapshot_id: str | None = None,
|
||||
) -> Path:
|
||||
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
|
||||
payload = {
|
||||
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
|
||||
"data_contract": {
|
||||
"key": "geointel.model.pytorch",
|
||||
"version": "1.0.0",
|
||||
},
|
||||
"model": {
|
||||
"model_id": model_id,
|
||||
"task_type": task_type,
|
||||
"sha256": model_sha256,
|
||||
"model_format": "pytorch",
|
||||
"framework": framework,
|
||||
"class_mapping": {"0": "building"},
|
||||
"source_version": source_version,
|
||||
},
|
||||
"source": {
|
||||
"source_registry_id": source_registry_id or str(uuid4()),
|
||||
"source_snapshot_id": source_snapshot_id or str(uuid4()),
|
||||
"source_registry_key": "model",
|
||||
"source_snapshot_checksum_sha256": model_sha256,
|
||||
},
|
||||
"lineage": {
|
||||
"upstream_asset_ids": ["training-corpus:test-v1"],
|
||||
"upstream_checksums_sha256": ["a" * 64],
|
||||
"transformations": [
|
||||
{
|
||||
"name": "pytorch-training",
|
||||
"version": "1.0.0",
|
||||
"checksum_sha256": "b" * 64,
|
||||
}
|
||||
],
|
||||
},
|
||||
"metadata": {
|
||||
"training_manifest_sha256": "c" * 64,
|
||||
},
|
||||
"imported_at": "2026-08-01T10:00:00+00:00",
|
||||
}
|
||||
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
|
||||
sidecar_path = RuntimeModelProvenanceService.manifest_path_for_model(model_path)
|
||||
sidecar_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
|
||||
return sidecar_path
|
||||
|
||||
|
||||
def _governed_model_database(
|
||||
*,
|
||||
source_registry_id,
|
||||
source_snapshot_id,
|
||||
model_checksum: str,
|
||||
source_version: str = "test-v1",
|
||||
) -> tuple[FakeSession, SourceRegistry, SourceSnapshot]:
|
||||
registry = SourceRegistry(
|
||||
id=source_registry_id,
|
||||
source_key="model",
|
||||
display_name="Governed test model artifacts",
|
||||
classification="experimental",
|
||||
authority_name="GeoIntel test fixture",
|
||||
freshness_status="current",
|
||||
ingest_status="configured",
|
||||
)
|
||||
snapshot = SourceSnapshot(
|
||||
id=source_snapshot_id,
|
||||
source_registry_id=source_registry_id,
|
||||
snapshot_key=f"model-{source_version}",
|
||||
source_version=source_version,
|
||||
checksum_sha256=model_checksum,
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
)
|
||||
return (
|
||||
FakeSession(
|
||||
{
|
||||
(SourceRegistry, source_registry_id): registry,
|
||||
(SourceSnapshot, source_snapshot_id): snapshot,
|
||||
}
|
||||
),
|
||||
registry,
|
||||
snapshot,
|
||||
)
|
||||
|
||||
|
||||
def test_runtime_model_provenance_accepts_byte_bound_pytorch_sidecar_for_structural_preflight(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"trusted local model bytes")
|
||||
sidecar_path = _write_sidecar(model_path, source_version="v1")
|
||||
|
||||
evidence = RuntimeModelProvenanceService.validate_for_runtime(
|
||||
model_path=model_path,
|
||||
model_id="yolo-configured",
|
||||
task_type="object_detection",
|
||||
expected_model_version="v1",
|
||||
allowed_frameworks=("ultralytics/pytorch",),
|
||||
)
|
||||
|
||||
assert evidence.model_sha256 == sha256(model_path.read_bytes()).hexdigest()
|
||||
assert evidence.manifest_path == str(sidecar_path.resolve())
|
||||
assert evidence.data_contract_key == "geointel.model.pytorch"
|
||||
assert evidence.data_contract_version == "1.0.0"
|
||||
assert len(evidence.validation_report_sha256) == 64
|
||||
|
||||
|
||||
def test_production_runtime_requires_db_bound_model_source_snapshot(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"governed local model bytes")
|
||||
source_registry_id = uuid4()
|
||||
source_snapshot_id = uuid4()
|
||||
_write_sidecar(
|
||||
model_path,
|
||||
source_version="v1",
|
||||
source_registry_id=str(source_registry_id),
|
||||
source_snapshot_id=str(source_snapshot_id),
|
||||
)
|
||||
db, _, _ = _governed_model_database(
|
||||
source_registry_id=source_registry_id,
|
||||
source_snapshot_id=source_snapshot_id,
|
||||
model_checksum=sha256(model_path.read_bytes()).hexdigest(),
|
||||
source_version="v1",
|
||||
)
|
||||
|
||||
evidence = RuntimeModelProvenanceService.validate_for_production_runtime(
|
||||
db=db,
|
||||
model_path=model_path,
|
||||
model_id="yolo-configured",
|
||||
task_type="object_detection",
|
||||
expected_model_version="v1",
|
||||
allowed_frameworks=("ultralytics/pytorch",),
|
||||
)
|
||||
|
||||
assert evidence.source_registry_id == str(source_registry_id)
|
||||
assert evidence.source_snapshot_id == str(source_snapshot_id)
|
||||
assert evidence.source_snapshot_checksum_sha256 == evidence.model_sha256
|
||||
|
||||
|
||||
def test_production_runtime_rejects_missing_database_source_binding(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"unbound model bytes")
|
||||
_write_sidecar(model_path)
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
RuntimeModelProvenanceService.validate_for_production_runtime(
|
||||
db=FakeSession(),
|
||||
model_path=model_path,
|
||||
model_id="yolo-configured",
|
||||
task_type="object_detection",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND"
|
||||
|
||||
|
||||
def test_production_runtime_requires_a_database_session(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"model bytes")
|
||||
_write_sidecar(model_path)
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
RuntimeModelProvenanceService.validate_for_production_runtime(
|
||||
db=None,
|
||||
model_path=model_path,
|
||||
model_id="yolo-configured",
|
||||
task_type="object_detection",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "MODEL_PROVENANCE_DATABASE_REQUIRED"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("mutation", "expected_code"),
|
||||
(
|
||||
("registry_unsafe", "MODEL_PROVENANCE_SOURCE_REGISTRY_UNSAFE"),
|
||||
("snapshot_registry_mismatch", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_REGISTRY_MISMATCH"),
|
||||
("snapshot_missing", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_NOT_FOUND"),
|
||||
("snapshot_quarantined", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_UNSAFE"),
|
||||
("snapshot_checksum_mismatch", "MODEL_PROVENANCE_DATABASE_SNAPSHOT_CHECKSUM_MISMATCH"),
|
||||
("active_quarantine", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_QUARANTINED"),
|
||||
),
|
||||
)
|
||||
def test_production_runtime_rejects_unsafe_or_inconsistent_database_snapshot(
|
||||
tmp_path: Path,
|
||||
mutation: str,
|
||||
expected_code: str,
|
||||
) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"governed model bytes")
|
||||
source_registry_id = uuid4()
|
||||
source_snapshot_id = uuid4()
|
||||
_write_sidecar(
|
||||
model_path,
|
||||
source_registry_id=str(source_registry_id),
|
||||
source_snapshot_id=str(source_snapshot_id),
|
||||
)
|
||||
db, registry, snapshot = _governed_model_database(
|
||||
source_registry_id=source_registry_id,
|
||||
source_snapshot_id=source_snapshot_id,
|
||||
model_checksum=sha256(model_path.read_bytes()).hexdigest(),
|
||||
)
|
||||
if mutation == "registry_unsafe":
|
||||
registry.ingest_status = "quarantined"
|
||||
elif mutation == "snapshot_registry_mismatch":
|
||||
snapshot.source_registry_id = uuid4()
|
||||
elif mutation == "snapshot_missing":
|
||||
db.objects.pop((SourceSnapshot, source_snapshot_id))
|
||||
elif mutation == "snapshot_quarantined":
|
||||
snapshot.ingest_status = "quarantined"
|
||||
elif mutation == "snapshot_checksum_mismatch":
|
||||
snapshot.checksum_sha256 = "f" * 64
|
||||
elif mutation == "active_quarantine":
|
||||
snapshot.quarantines = [
|
||||
DatasetQuarantine(
|
||||
source_snapshot_id=source_snapshot_id,
|
||||
stage="test",
|
||||
reason_code="test_active_quarantine",
|
||||
status="quarantined",
|
||||
)
|
||||
]
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
RuntimeModelProvenanceService.validate_for_production_runtime(
|
||||
db=db,
|
||||
model_path=model_path,
|
||||
model_id="yolo-configured",
|
||||
task_type="object_detection",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == expected_code
|
||||
|
||||
|
||||
def test_runtime_model_provenance_rejects_missing_sidecar(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"unmanifested local model bytes")
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
RuntimeModelProvenanceService.validate_for_runtime(
|
||||
model_path=model_path,
|
||||
model_id="yolo-configured",
|
||||
task_type="object_detection",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "MODEL_PROVENANCE_MANIFEST_MISSING"
|
||||
|
||||
|
||||
def test_runtime_model_provenance_rejects_model_bytes_tampered_after_manifest(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"original local model bytes")
|
||||
_write_sidecar(model_path)
|
||||
model_path.write_bytes(b"tampered local model bytes")
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
RuntimeModelProvenanceService.validate_for_runtime(
|
||||
model_path=model_path,
|
||||
model_id="yolo-configured",
|
||||
task_type="object_detection",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "MODEL_PROVENANCE_MODEL_CHECKSUM_MISMATCH"
|
||||
|
||||
|
||||
def test_runtime_model_provenance_rejects_tampered_manifest_contents(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"original local model bytes")
|
||||
sidecar_path = _write_sidecar(model_path)
|
||||
payload = json.loads(sidecar_path.read_text(encoding="utf-8"))
|
||||
payload["model"]["class_mapping"]["1"] = "road"
|
||||
sidecar_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
RuntimeModelProvenanceService.validate_for_runtime(
|
||||
model_path=model_path,
|
||||
model_id="yolo-configured",
|
||||
task_type="object_detection",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "MODEL_PROVENANCE_MANIFEST_CHECKSUM_MISMATCH"
|
||||
|
||||
|
||||
def test_runtime_model_provenance_rejects_other_contract_even_if_structurally_valid(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"local model bytes")
|
||||
sidecar_path = _write_sidecar(model_path)
|
||||
payload = json.loads(sidecar_path.read_text(encoding="utf-8"))
|
||||
payload["data_contract"]["key"] = "geointel.vector.geojson"
|
||||
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
|
||||
sidecar_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
|
||||
|
||||
with pytest.raises(AppError) as exc_info:
|
||||
RuntimeModelProvenanceService.validate_for_runtime(
|
||||
model_path=model_path,
|
||||
model_id="yolo-configured",
|
||||
task_type="object_detection",
|
||||
)
|
||||
|
||||
assert exc_info.value.code == "MODEL_PROVENANCE_MANIFEST_INVALID"
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
@@ -8,9 +9,10 @@ import pytest
|
||||
from geoalchemy2.shape import to_shape
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.models import Dataset, Project, Segmentation
|
||||
from app.models import Dataset, Project, Segmentation, SourceRegistry, SourceSnapshot
|
||||
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
|
||||
from app.services.model_registry_service import ModelRegistryService
|
||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||
from app.services.segmentation_service import SegmentationService
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
@@ -80,18 +82,58 @@ class MissingDependencySegAdapter(AvailableSegAdapter):
|
||||
return False
|
||||
|
||||
|
||||
class NeverLoadSegAdapter(AvailableSegAdapter):
|
||||
load_calls = 0
|
||||
|
||||
def load_model(self, model_path: Path):
|
||||
type(self).load_calls += 1
|
||||
raise AssertionError("unmanifested weights must not reach adapter.load_model")
|
||||
|
||||
|
||||
def _project_and_dataset(dataset_type: str = "raster"):
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
source_id = uuid4()
|
||||
snapshot_id = uuid4()
|
||||
checksum = "a" * 64
|
||||
project = Project(id=project_id, name="Mol")
|
||||
source = SourceRegistry(
|
||||
id=source_id,
|
||||
source_key="digitaal_vlaanderen_orthophoto",
|
||||
display_name="Governed orthophoto test source",
|
||||
classification="contextual",
|
||||
authority_name="Digitaal Vlaanderen",
|
||||
authority_scope_json={"zone": "Flanders", "role": "imagery"},
|
||||
)
|
||||
snapshot = SourceSnapshot(
|
||||
id=snapshot_id,
|
||||
source_registry_id=source_id,
|
||||
snapshot_key="configured-segmentation-orthophoto",
|
||||
checksum_sha256=checksum,
|
||||
ingest_status="ingested",
|
||||
freshness_status="current",
|
||||
)
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name="ortho.tif",
|
||||
dataset_type=dataset_type,
|
||||
source="user_upload",
|
||||
source="digitaal_vlaanderen_orthophoto",
|
||||
source_name="digitaal_vlaanderen_orthophoto",
|
||||
storage_path="storage/uploads/ortho.tif",
|
||||
checksum_sha256=checksum,
|
||||
source_registry_id=source_id,
|
||||
source_snapshot_id=snapshot_id,
|
||||
data_contract_key="geointel.raster.geotiff",
|
||||
data_contract_version="1.0.0",
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="not_applicable",
|
||||
quarantine_status="not_quarantined",
|
||||
status="ready",
|
||||
)
|
||||
dataset.source_registry = source
|
||||
dataset.source_snapshot = snapshot
|
||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||
return db, project_id, dataset_id
|
||||
|
||||
@@ -108,6 +150,102 @@ def _settings(tmp_path: Path, **overrides) -> Settings:
|
||||
return Settings(**values)
|
||||
|
||||
|
||||
def _write_model_sidecar(
|
||||
model_path: Path,
|
||||
*,
|
||||
model_id: str,
|
||||
framework: str,
|
||||
source_version: str | None,
|
||||
db: FakeSession | None = None,
|
||||
) -> None:
|
||||
"""Create explicit local test evidence; no production code creates sidecars."""
|
||||
|
||||
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
|
||||
source_registry_id = uuid4()
|
||||
source_snapshot_id = uuid4()
|
||||
resolved_source_version = source_version or "test-v1"
|
||||
if db is not None:
|
||||
source_registry = SourceRegistry(
|
||||
id=source_registry_id,
|
||||
source_key="model",
|
||||
display_name="Governed test model artifact",
|
||||
classification="experimental",
|
||||
authority_name="GeoIntel test fixture",
|
||||
freshness_status="current",
|
||||
ingest_status="configured",
|
||||
)
|
||||
source_snapshot = SourceSnapshot(
|
||||
id=source_snapshot_id,
|
||||
source_registry_id=source_registry_id,
|
||||
snapshot_key=f"model-{model_id}-{resolved_source_version}",
|
||||
source_version=resolved_source_version,
|
||||
checksum_sha256=model_sha256,
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
)
|
||||
db.objects[(SourceRegistry, source_registry_id)] = source_registry
|
||||
db.objects[(SourceSnapshot, source_snapshot_id)] = source_snapshot
|
||||
payload = {
|
||||
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
|
||||
"data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"},
|
||||
"model": {
|
||||
"model_id": model_id,
|
||||
"task_type": "segmentation",
|
||||
"sha256": model_sha256,
|
||||
"model_format": "pytorch",
|
||||
"framework": framework,
|
||||
"class_mapping": {"0": "segment"},
|
||||
"source_version": resolved_source_version,
|
||||
},
|
||||
"source": {
|
||||
"source_registry_id": str(source_registry_id),
|
||||
"source_snapshot_id": str(source_snapshot_id),
|
||||
"source_registry_key": "model",
|
||||
"source_snapshot_checksum_sha256": model_sha256,
|
||||
},
|
||||
"lineage": {
|
||||
"upstream_asset_ids": ["test-training-corpus"],
|
||||
"upstream_checksums_sha256": ["a" * 64],
|
||||
"transformations": [
|
||||
{"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64}
|
||||
],
|
||||
},
|
||||
"metadata": {"training_manifest_sha256": "c" * 64},
|
||||
"imported_at": "2026-08-01T10:00:00+00:00",
|
||||
}
|
||||
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
|
||||
RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text(
|
||||
json.dumps(payload, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _write_configured_model_sidecars(
|
||||
tmp_path: Path,
|
||||
settings: Settings,
|
||||
*,
|
||||
include_yolo: bool = True,
|
||||
include_sam: bool = True,
|
||||
db: FakeSession | None = None,
|
||||
) -> None:
|
||||
if include_yolo:
|
||||
_write_model_sidecar(
|
||||
tmp_path / "seg.pt",
|
||||
model_id=settings.yolo_seg_model_id,
|
||||
framework="ultralytics/pytorch",
|
||||
source_version=settings.yolo_seg_model_version,
|
||||
db=db,
|
||||
)
|
||||
if include_sam:
|
||||
_write_model_sidecar(
|
||||
tmp_path / "sam.pt",
|
||||
model_id=settings.sam_model_id,
|
||||
framework="ultralytics/sam",
|
||||
source_version=settings.sam_model_version,
|
||||
db=db,
|
||||
)
|
||||
|
||||
|
||||
def _manifest(tmp_path: Path, tile_count: int = 1) -> Path:
|
||||
tiles = []
|
||||
for index in range(tile_count):
|
||||
@@ -172,10 +310,31 @@ def test_segmentation_models_report_dependency_unavailable(tmp_path: Path) -> No
|
||||
assert models["sam-configured"].status == "dependency_unavailable"
|
||||
|
||||
|
||||
def test_segmentation_models_require_runtime_provenance_sidecars(tmp_path: Path) -> None:
|
||||
(tmp_path / "seg.pt").write_bytes(b"unmanifested yolo segmentation weights")
|
||||
(tmp_path / "sam.pt").write_bytes(b"unmanifested sam weights")
|
||||
settings = _settings(tmp_path)
|
||||
|
||||
models = {
|
||||
model.model_id: model
|
||||
for model in ModelRegistryService.list_segmentation_model_capabilities(
|
||||
settings=settings,
|
||||
yolo_seg_adapter_class=AvailableSegAdapter,
|
||||
sam_adapter_class=ClassAgnosticSamAdapter,
|
||||
)
|
||||
}
|
||||
|
||||
assert models["yolo-seg-configured"].configured is False
|
||||
assert models["yolo-seg-configured"].status == "contract_incomplete"
|
||||
assert models["sam-configured"].configured is False
|
||||
assert models["sam-configured"].status == "contract_incomplete"
|
||||
|
||||
|
||||
def test_segmentation_models_report_configured_with_local_weights(tmp_path: Path) -> None:
|
||||
(tmp_path / "seg.pt").write_bytes(b"weights")
|
||||
(tmp_path / "sam.pt").write_bytes(b"weights")
|
||||
settings = _settings(tmp_path)
|
||||
_write_configured_model_sidecars(tmp_path, settings)
|
||||
|
||||
models = {
|
||||
model.model_id: model
|
||||
@@ -204,6 +363,7 @@ def test_configured_segmentation_requires_tile_manifest(tmp_path: Path) -> None:
|
||||
(tmp_path / "seg.pt").write_bytes(b"weights")
|
||||
db, project_id, dataset_id = _project_and_dataset()
|
||||
settings = _settings(tmp_path)
|
||||
_write_configured_model_sidecars(tmp_path, settings, include_sam=False, db=db)
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
SegmentationService.run_segmentation(
|
||||
@@ -220,10 +380,60 @@ def test_configured_segmentation_requires_tile_manifest(tmp_path: Path) -> None:
|
||||
assert getattr(exc_info.value, "code", None) == "SEGMENTATION_TILE_MANIFEST_REQUIRED"
|
||||
|
||||
|
||||
def test_configured_segmentation_fails_before_adapter_load_without_sidecar(tmp_path: Path) -> None:
|
||||
(tmp_path / "seg.pt").write_bytes(b"unmanifested weights")
|
||||
db, project_id, dataset_id = _project_and_dataset()
|
||||
settings = _settings(tmp_path)
|
||||
NeverLoadSegAdapter.load_calls = 0
|
||||
|
||||
response = SegmentationService.run_segmentation(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
model_id="yolo-seg-configured",
|
||||
confidence_threshold=0.5,
|
||||
tile_manifest_path=str(_manifest(tmp_path)),
|
||||
settings=settings,
|
||||
yolo_seg_adapter_class=NeverLoadSegAdapter,
|
||||
sam_adapter_class=ClassAgnosticSamAdapter,
|
||||
)
|
||||
|
||||
assert response.status == "failed"
|
||||
assert response.error_code == "SEGMENTATION_MODEL_UNAVAILABLE"
|
||||
assert NeverLoadSegAdapter.load_calls == 0
|
||||
|
||||
|
||||
def test_configured_segmentation_rejects_unbound_model_snapshot_before_adapter_load(tmp_path: Path) -> None:
|
||||
(tmp_path / "seg.pt").write_bytes(b"structurally valid but unbound weights")
|
||||
db, project_id, dataset_id = _project_and_dataset()
|
||||
settings = _settings(tmp_path)
|
||||
# The sidecar passes catalog validation but its source registry/snapshot
|
||||
# was never registered in this production-session fixture.
|
||||
_write_configured_model_sidecars(tmp_path, settings, include_sam=False)
|
||||
NeverLoadSegAdapter.load_calls = 0
|
||||
|
||||
response = SegmentationService.run_segmentation(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
model_id="yolo-seg-configured",
|
||||
confidence_threshold=0.5,
|
||||
tile_manifest_path=str(_manifest(tmp_path)),
|
||||
settings=settings,
|
||||
yolo_seg_adapter_class=NeverLoadSegAdapter,
|
||||
sam_adapter_class=ClassAgnosticSamAdapter,
|
||||
)
|
||||
|
||||
assert response.status == "failed"
|
||||
assert response.error_code == "MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND"
|
||||
assert NeverLoadSegAdapter.load_calls == 0
|
||||
|
||||
|
||||
def test_configured_yolo_seg_run_persists_georeferenced_masks(tmp_path: Path) -> None:
|
||||
(tmp_path / "seg.pt").write_bytes(b"weights")
|
||||
db, project_id, dataset_id = _project_and_dataset()
|
||||
settings = _settings(tmp_path)
|
||||
_write_configured_model_sidecars(tmp_path, settings, include_sam=False, db=db)
|
||||
manifest_path = _manifest(tmp_path)
|
||||
|
||||
response = SegmentationService.run_segmentation(
|
||||
@@ -255,12 +465,14 @@ def test_configured_yolo_seg_run_persists_georeferenced_masks(tmp_path: Path) ->
|
||||
assert segmentation.area_m2 is not None and segmentation.area_m2 > 0
|
||||
assert segmentation.provenance_json["inference"] == "local"
|
||||
assert segmentation.provenance_json["model_id"] == "yolo-seg-configured"
|
||||
assert segmentation.provenance_json["runtime_model_provenance"]["data_contract_key"] == "geointel.model.pytorch"
|
||||
|
||||
|
||||
def test_configured_sam_run_is_class_agnostic(tmp_path: Path) -> None:
|
||||
(tmp_path / "sam.pt").write_bytes(b"weights")
|
||||
db, project_id, dataset_id = _project_and_dataset()
|
||||
settings = _settings(tmp_path)
|
||||
_write_configured_model_sidecars(tmp_path, settings, include_yolo=False, db=db)
|
||||
manifest_path = _manifest(tmp_path)
|
||||
|
||||
response = SegmentationService.run_segmentation(
|
||||
|
||||
@@ -11,7 +11,7 @@ from shapely.geometry import box
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.main import app
|
||||
from app.models import Area, Dataset, Export
|
||||
from app.models import Area, Dataset, Export, SourceRegistry, SourceSnapshot
|
||||
from app.schemas.export import ExportCreateResponse
|
||||
from app.services.export_service import ExportService
|
||||
from app.services.storage_service import StorageService
|
||||
@@ -45,18 +45,56 @@ class FakeSession:
|
||||
return row
|
||||
|
||||
|
||||
def _govern_fixture_dataset(dataset: Dataset) -> Dataset:
|
||||
source_id = uuid4()
|
||||
snapshot_id = uuid4()
|
||||
checksum = "a" * 64
|
||||
source = SourceRegistry(
|
||||
id=source_id,
|
||||
source_key="grb",
|
||||
display_name="GRB map export test source",
|
||||
classification="authoritative",
|
||||
authority_name="Digitaal Vlaanderen",
|
||||
authority_scope_json={"zone": "Flanders"},
|
||||
usage_policy_json={"ground_truth_allowed": True},
|
||||
)
|
||||
snapshot = SourceSnapshot(
|
||||
id=snapshot_id,
|
||||
source_registry_id=source_id,
|
||||
snapshot_key=f"map-export-grb-{dataset.id}",
|
||||
checksum_sha256=checksum,
|
||||
ingest_status="ingested",
|
||||
freshness_status="current",
|
||||
)
|
||||
dataset.source = "grb"
|
||||
dataset.source_name = "grb"
|
||||
dataset.checksum_sha256 = checksum
|
||||
dataset.source_registry_id = source_id
|
||||
dataset.source_snapshot_id = snapshot_id
|
||||
dataset.data_contract_key = "geointel.vector.geojson"
|
||||
dataset.data_contract_version = "1.0.0"
|
||||
dataset.validation_status = "passed"
|
||||
dataset.provenance_status = "complete"
|
||||
dataset.lineage_status = "not_applicable"
|
||||
dataset.quarantine_status = "not_quarantined"
|
||||
dataset.status = "ready"
|
||||
dataset.source_registry = source
|
||||
dataset.source_snapshot = snapshot
|
||||
return dataset
|
||||
|
||||
|
||||
def test_vector_selection_geojson_export_persists_handoff_artifact(tmp_path, monkeypatch) -> None:
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
export_path = tmp_path / "exports" / "selection.geojson"
|
||||
dataset = Dataset(
|
||||
dataset = _govern_fixture_dataset(Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name="candidate.geojson",
|
||||
dataset_type="vector",
|
||||
source="fixture",
|
||||
status="ready",
|
||||
)
|
||||
))
|
||||
db = FakeSession({(Dataset, dataset_id): dataset})
|
||||
selection_bbox = {"min_x": 4.9, "min_y": 50.9, "max_x": 5.2, "max_y": 51.2, "crs": "EPSG:4326"}
|
||||
selection_payload = {
|
||||
@@ -135,14 +173,14 @@ def test_vector_selection_export_uses_exact_area_scope_when_requested(tmp_path,
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
area_id = uuid4()
|
||||
dataset = Dataset(
|
||||
dataset = _govern_fixture_dataset(Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name="regional-buildings.geojson",
|
||||
dataset_type="vector",
|
||||
source="fixture",
|
||||
status="ready",
|
||||
)
|
||||
))
|
||||
area_shape = box(5.0, 51.1, 5.2, 51.3)
|
||||
area_geometry = from_shape(area_shape, srid=4326)
|
||||
area = SimpleNamespace(id=area_id, project_id=project_id, name="Gemeente Mol", geometry=area_geometry)
|
||||
@@ -184,7 +222,7 @@ def test_area_constrained_bbox_uses_intersection_and_disables_full_area_fast_pat
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
area_id = uuid4()
|
||||
dataset = Dataset(
|
||||
dataset = _govern_fixture_dataset(Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
area_id=area_id,
|
||||
@@ -193,7 +231,7 @@ def test_area_constrained_bbox_uses_intersection_and_disables_full_area_fast_pat
|
||||
source="fixture",
|
||||
source_metadata={"geometry_clipped_to_area": True},
|
||||
status="ready",
|
||||
)
|
||||
))
|
||||
area_shape = box(5.0, 51.0, 5.2, 51.2)
|
||||
area = SimpleNamespace(
|
||||
id=area_id,
|
||||
|
||||
@@ -7,7 +7,7 @@ from uuid import uuid4
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.main import app
|
||||
from app.models import Dataset, VectorFeature
|
||||
from app.models import Dataset
|
||||
from app.schemas.dataset import DatasetCreateResponse
|
||||
from app.services.storage_service import StorageService
|
||||
from app.services.vector_feature_service import VectorFeatureService
|
||||
@@ -124,7 +124,7 @@ def test_vector_selection_derive_persists_queryable_derived_dataset(tmp_path, mo
|
||||
assert response.provenance_metadata["source_dataset_id"] == str(dataset_id)
|
||||
assert response.provenance_metadata["source_table"] == "vector_features"
|
||||
assert persisted_features[0]["dataset_id"] == derived.id
|
||||
assert persisted_features[0]["commit"] is True
|
||||
assert persisted_features[0]["commit"] is False
|
||||
derived_payload = json.loads(output_path.read_text(encoding="utf-8"))
|
||||
props = derived_payload["features"][0]["properties"]
|
||||
assert props["source_vector_feature_id"] == "source-row-1"
|
||||
|
||||
@@ -75,5 +75,7 @@ def test_operator_yolo_train_smoke_script_contract() -> None:
|
||||
assert '"dataset_summary_sha256"' in script
|
||||
assert '"base_model_sha256"' in script
|
||||
assert '"trained_model_sha256"' in script
|
||||
assert "training_release_manifest.py" in script
|
||||
assert "verify" in script
|
||||
assert "download" not in script.lower()
|
||||
assert "fixture_mode" not in script
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from hashlib import sha256
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -9,6 +10,7 @@ from fastapi.testclient import TestClient
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.main import app
|
||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||
from app.services.yolo_preflight_service import YoloPreflightService
|
||||
|
||||
|
||||
@@ -57,6 +59,43 @@ def _manifest(tmp_path: Path, tile_count: int = 1) -> Path:
|
||||
return manifest_path
|
||||
|
||||
|
||||
def _write_model_sidecar(model_path: Path, settings: Settings) -> None:
|
||||
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
|
||||
payload = {
|
||||
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
|
||||
"data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"},
|
||||
"model": {
|
||||
"model_id": settings.yolo_model_id,
|
||||
"task_type": "object_detection",
|
||||
"sha256": model_sha256,
|
||||
"model_format": "pytorch",
|
||||
"framework": "ultralytics/pytorch",
|
||||
"class_mapping": {"0": "building"},
|
||||
"source_version": settings.yolo_model_version or "test-v1",
|
||||
},
|
||||
"source": {
|
||||
"source_registry_id": "11111111-1111-4111-8111-111111111111",
|
||||
"source_snapshot_id": "22222222-2222-4222-8222-222222222222",
|
||||
"source_registry_key": "model",
|
||||
"source_snapshot_checksum_sha256": model_sha256,
|
||||
},
|
||||
"lineage": {
|
||||
"upstream_asset_ids": ["test-training-corpus"],
|
||||
"upstream_checksums_sha256": ["a" * 64],
|
||||
"transformations": [
|
||||
{"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64}
|
||||
],
|
||||
},
|
||||
"metadata": {"training_manifest_sha256": "c" * 64},
|
||||
"imported_at": "2026-08-01T10:00:00+00:00",
|
||||
}
|
||||
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
|
||||
RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text(
|
||||
json.dumps(payload, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def test_yolo_preflight_reports_disabled_without_loading_model(tmp_path: Path, monkeypatch) -> None:
|
||||
monkeypatch.setenv("YOLO_CONFIG_DIR", str(tmp_path / "ultralytics"))
|
||||
|
||||
@@ -99,9 +138,11 @@ def test_yolo_preflight_validates_model_and_manifest_without_importing_yolo(tmp_
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"weights")
|
||||
manifest_path = _manifest(tmp_path, tile_count=2)
|
||||
settings = Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4)
|
||||
_write_model_sidecar(model_path, settings)
|
||||
|
||||
result = YoloPreflightService.run(
|
||||
settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4),
|
||||
settings=settings,
|
||||
tile_manifest_path=str(manifest_path),
|
||||
yolo_adapter_class=AvailableAdapter,
|
||||
)
|
||||
@@ -109,6 +150,7 @@ def test_yolo_preflight_validates_model_and_manifest_without_importing_yolo(tmp_
|
||||
assert result["status"] == "ready"
|
||||
assert result["checks"]["dependencies_available"] is True
|
||||
assert result["checks"]["model_file_exists"] is True
|
||||
assert result["checks"]["model_provenance_valid"] is True
|
||||
assert result["checks"]["manifest_valid"] is True
|
||||
assert result["tile_count"] == 2
|
||||
assert result["will_download_models"] is False
|
||||
@@ -120,9 +162,11 @@ def test_yolo_preflight_marks_assumed_dependencies_in_runtime_details(tmp_path:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"weights")
|
||||
manifest_path = _manifest(tmp_path, tile_count=1)
|
||||
settings = Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4)
|
||||
_write_model_sidecar(model_path, settings)
|
||||
|
||||
result = YoloPreflightService.run(
|
||||
settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4),
|
||||
settings=settings,
|
||||
tile_manifest_path=str(manifest_path),
|
||||
yolo_adapter_class=MissingDependencyAdapter,
|
||||
assume_dependencies=True,
|
||||
@@ -138,9 +182,11 @@ def test_yolo_preflight_can_explicitly_smoke_load_local_model(tmp_path: Path) ->
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"weights")
|
||||
manifest_path = _manifest(tmp_path)
|
||||
settings = Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4)
|
||||
_write_model_sidecar(model_path, settings)
|
||||
|
||||
result = YoloPreflightService.run(
|
||||
settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4),
|
||||
settings=settings,
|
||||
tile_manifest_path=str(manifest_path),
|
||||
yolo_adapter_class=AvailableAdapter,
|
||||
check_model_load=True,
|
||||
@@ -156,9 +202,11 @@ def test_yolo_preflight_can_explicitly_smoke_load_local_model(tmp_path: Path) ->
|
||||
def test_yolo_preflight_reports_explicit_model_load_failure(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"weights")
|
||||
settings = Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4)
|
||||
_write_model_sidecar(model_path, settings)
|
||||
|
||||
result = YoloPreflightService.run(
|
||||
settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4),
|
||||
settings=settings,
|
||||
tile_manifest_path=str(_manifest(tmp_path)),
|
||||
yolo_adapter_class=FailingLoadAdapter,
|
||||
check_model_load=True,
|
||||
@@ -173,6 +221,7 @@ def test_yolo_preflight_script_outputs_json(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"weights")
|
||||
manifest_path = _manifest(tmp_path)
|
||||
_write_model_sidecar(model_path, Settings(yolo_enabled=True, yolo_model_path=str(model_path)))
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
@@ -201,6 +250,7 @@ def test_yolo_preflight_script_uses_environment_configuration(tmp_path: Path, mo
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"weights")
|
||||
manifest_path = _manifest(tmp_path)
|
||||
_write_model_sidecar(model_path, Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4))
|
||||
monkeypatch.setenv("YOLO_ENABLED", "true")
|
||||
monkeypatch.setenv("YOLO_MODEL_PATH", str(model_path))
|
||||
monkeypatch.setenv("YOLO_MAX_TILES", "4")
|
||||
@@ -227,6 +277,21 @@ def test_yolo_preflight_script_uses_environment_configuration(tmp_path: Path, mo
|
||||
assert payload["max_tiles"] == 4
|
||||
|
||||
|
||||
def test_yolo_preflight_refuses_unmanifested_local_weights(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"unmanifested weights")
|
||||
|
||||
result = YoloPreflightService.run(
|
||||
settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path)),
|
||||
tile_manifest_path=str(_manifest(tmp_path)),
|
||||
yolo_adapter_class=AvailableAdapter,
|
||||
)
|
||||
|
||||
assert result["status"] == "contract_incomplete"
|
||||
assert result["checks"]["model_provenance_valid"] is False
|
||||
assert result["error_code"] == "MODEL_PROVENANCE_MANIFEST_MISSING"
|
||||
|
||||
|
||||
def test_yolo_preflight_script_rejects_assumed_dependencies_for_model_load(tmp_path: Path) -> None:
|
||||
result = subprocess.run(
|
||||
[
|
||||
|
||||
@@ -8,7 +8,7 @@ from fastapi.testclient import TestClient
|
||||
|
||||
from app.core.errors import AppError
|
||||
from app.main import app
|
||||
from app.models import Area, Dataset, Export, Project, QualityCheck
|
||||
from app.models import Area, Dataset, Export, Project, QualityCheck, SourceRegistry, SourceSnapshot
|
||||
from app.schemas.export import ExportCreateResponse
|
||||
from app.services.export_service import ExportService
|
||||
from app.services.storage_service import StorageService
|
||||
@@ -66,13 +66,57 @@ class FakeSession:
|
||||
return row
|
||||
|
||||
|
||||
def _govern_fixture_dataset(dataset: Dataset) -> Dataset:
|
||||
"""Give an export fixture a governed authoritative source identity.
|
||||
|
||||
Export is a production boundary: test data must model a source that could
|
||||
cross it, rather than using the deliberately QA-only ``fixture`` source.
|
||||
"""
|
||||
|
||||
source_id = uuid4()
|
||||
snapshot_id = uuid4()
|
||||
checksum = "a" * 64
|
||||
source = SourceRegistry(
|
||||
id=source_id,
|
||||
source_key="grb",
|
||||
display_name="GRB export test source",
|
||||
classification="authoritative",
|
||||
authority_name="Digitaal Vlaanderen",
|
||||
authority_scope_json={"zone": "Flanders"},
|
||||
usage_policy_json={"ground_truth_allowed": True},
|
||||
)
|
||||
snapshot = SourceSnapshot(
|
||||
id=snapshot_id,
|
||||
source_registry_id=source_id,
|
||||
snapshot_key=f"export-grb-{dataset.id}",
|
||||
checksum_sha256=checksum,
|
||||
ingest_status="ingested",
|
||||
freshness_status="current",
|
||||
)
|
||||
dataset.source = "grb"
|
||||
dataset.source_name = "grb"
|
||||
dataset.checksum_sha256 = checksum
|
||||
dataset.source_registry_id = source_id
|
||||
dataset.source_snapshot_id = snapshot_id
|
||||
dataset.data_contract_key = "geointel.vector.geojson"
|
||||
dataset.data_contract_version = "1.0.0"
|
||||
dataset.validation_status = "passed"
|
||||
dataset.provenance_status = "complete"
|
||||
dataset.lineage_status = "not_applicable"
|
||||
dataset.quarantine_status = "not_quarantined"
|
||||
dataset.status = "ready"
|
||||
dataset.source_registry = source
|
||||
dataset.source_snapshot = snapshot
|
||||
return dataset
|
||||
|
||||
|
||||
def test_dataset_geojson_export_persists_export_and_writes_artifact(tmp_path, monkeypatch) -> None:
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
dataset_path = tmp_path / "input.geojson"
|
||||
dataset_path.write_text(json.dumps({"type": "FeatureCollection", "features": []}), encoding="utf-8")
|
||||
export_path = tmp_path / "exports" / "buildings.geojson"
|
||||
dataset = Dataset(
|
||||
dataset = _govern_fixture_dataset(Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name="buildings.geojson",
|
||||
@@ -80,7 +124,7 @@ def test_dataset_geojson_export_persists_export_and_writes_artifact(tmp_path, mo
|
||||
source="fixture",
|
||||
storage_path=str(dataset_path),
|
||||
status="ready",
|
||||
)
|
||||
))
|
||||
db = FakeSession({(Dataset, dataset_id): dataset})
|
||||
monkeypatch.setattr(StorageService, "dataset_export_path", lambda *_args: str(export_path))
|
||||
|
||||
|
||||
@@ -18,7 +18,7 @@ from app.core.config import Settings
|
||||
from app.core.errors import AppError
|
||||
from app.db.session import get_db
|
||||
from app.main import app
|
||||
from app.models import Area, Dataset, DatasetVersion, Job, Project
|
||||
from app.models import Area, Dataset, DatasetVersion, Job, Project, SourceRegistry, SourceSnapshot
|
||||
from app.schemas.orthophoto import OrthophotoAcquireRequest
|
||||
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
|
||||
|
||||
@@ -41,6 +41,15 @@ class FakeSession:
|
||||
def add(self, row):
|
||||
self.added.append(row)
|
||||
|
||||
def flush(self):
|
||||
# The governed importer persists source identities and immutable
|
||||
# snapshots before the Dataset. Mirror the database-generated UUIDs
|
||||
# so this harness exercises that Phase 2 path rather than the legacy
|
||||
# no-registry fallback.
|
||||
for row in self.added:
|
||||
if getattr(row, "id", None) is None:
|
||||
row.id = uuid4()
|
||||
|
||||
def commit(self):
|
||||
return None
|
||||
|
||||
@@ -50,13 +59,23 @@ class FakeSession:
|
||||
def refresh(self, row):
|
||||
return row
|
||||
|
||||
def query(self, _model):
|
||||
return FakeQuery(self.query_result)
|
||||
def query(self, model):
|
||||
rows = [
|
||||
row
|
||||
for (row_model, _row_id), row in self.rows.items()
|
||||
if row_model is model and isinstance(row, model)
|
||||
]
|
||||
rows.extend(row for row in self.added if isinstance(row, model))
|
||||
if isinstance(self.query_result, model):
|
||||
rows.append(self.query_result)
|
||||
elif isinstance(self.query_result, list):
|
||||
rows.extend(row for row in self.query_result if isinstance(row, model))
|
||||
return FakeQuery(rows)
|
||||
|
||||
|
||||
class FakeQuery:
|
||||
def __init__(self, result):
|
||||
self.result = result
|
||||
def __init__(self, results):
|
||||
self.results = list(results)
|
||||
|
||||
def filter(self, *_args):
|
||||
return self
|
||||
@@ -65,7 +84,10 @@ class FakeQuery:
|
||||
return self
|
||||
|
||||
def first(self):
|
||||
return self.result
|
||||
return self.results[0] if self.results else None
|
||||
|
||||
def one_or_none(self):
|
||||
return self.first()
|
||||
|
||||
|
||||
class FakeImageResponse:
|
||||
@@ -222,12 +244,23 @@ def test_regional_orthophoto_products_bind_provider_and_governed_scope(
|
||||
)
|
||||
|
||||
dataset = next(row for row in db.added if isinstance(row, Dataset))
|
||||
source = next(row for row in db.added if isinstance(row, SourceRegistry))
|
||||
snapshot = next(row for row in db.added if isinstance(row, SourceSnapshot))
|
||||
assert result["provider"] == provider
|
||||
assert result["layer"] == layer
|
||||
assert dataset.source_name == provider
|
||||
assert dataset.source_metadata["coverage_zone"] == coverage_zone
|
||||
assert dataset.source_metadata["license_note"]
|
||||
assert dataset.provenance_metadata["request_url"].startswith(prepared["product"].wms_url)
|
||||
assert dataset.source_registry_id == source.id
|
||||
assert dataset.source_snapshot_id == snapshot.id
|
||||
assert dataset.validation_status == "passed"
|
||||
assert dataset.provenance_status == "complete"
|
||||
assert dataset.quarantine_status == "not_quarantined"
|
||||
assert snapshot.source_registry_id == source.id
|
||||
assert snapshot.checksum_sha256 == dataset.checksum_sha256
|
||||
assert snapshot.ingest_status == "ingested"
|
||||
assert snapshot.freshness_status == "current"
|
||||
|
||||
prepared = OrthophotoAcquisitionService._prepared_request(_selection_payload(product_key="1971"), settings)
|
||||
assert prepared["params"]["LAYERS"] == "OKZPAN71VL"
|
||||
@@ -291,6 +324,8 @@ def test_orthophoto_acquisition_persists_georeferenced_raster_and_provenance(tmp
|
||||
assert len(datasets) == 1
|
||||
assert len(versions) == 1
|
||||
dataset = datasets[0]
|
||||
source = next(row for row in db.added if isinstance(row, SourceRegistry))
|
||||
snapshot = next(row for row in db.added if isinstance(row, SourceSnapshot))
|
||||
assert result["output_dataset_id"] == str(dataset.id)
|
||||
assert result["reused"] is False
|
||||
assert dataset.project_id == project_id
|
||||
@@ -301,6 +336,15 @@ def test_orthophoto_acquisition_persists_georeferenced_raster_and_provenance(tmp
|
||||
assert dataset.crs == "EPSG:31370"
|
||||
assert dataset.provenance_metadata["acquisition"] == "explicit_bounded_map_selection"
|
||||
assert dataset.provenance_metadata["request_hash"] == prepared["request_hash"]
|
||||
assert dataset.source_registry_id == source.id
|
||||
assert dataset.source_snapshot_id == snapshot.id
|
||||
assert dataset.validation_status == "passed"
|
||||
assert dataset.provenance_status == "complete"
|
||||
assert dataset.lineage_status == "not_applicable"
|
||||
assert dataset.quarantine_status == "not_quarantined"
|
||||
assert snapshot.source_registry_id == source.id
|
||||
assert snapshot.checksum_sha256 == dataset.checksum_sha256
|
||||
assert snapshot.freshness_status == "current"
|
||||
assert dataset.source_metadata["attribution"].startswith("Bron: Orthofotomozaiek Vlaanderen")
|
||||
assert dataset.storage_path is not None
|
||||
with rasterio.open(dataset.storage_path) as stored:
|
||||
|
||||
@@ -17,7 +17,7 @@ from app.core.config import Settings
|
||||
from app.core.errors import AppError
|
||||
from app.db.session import get_db
|
||||
from app.main import app
|
||||
from app.models import Area, Dataset, DatasetVersion, Job, Project
|
||||
from app.models import Area, Dataset, DatasetVersion, Job, Project, SourceRegistry, SourceSnapshot
|
||||
from app.schemas.dhmv import DhmvAcquireRequest, TerrainPartitionSelectionRequest, TerrainSelectionRequest
|
||||
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
|
||||
from app.services.terrain_analysis_service import TerrainAnalysisService
|
||||
@@ -27,8 +27,8 @@ ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
class FakeQuery:
|
||||
def __init__(self, result=None):
|
||||
self.result = result
|
||||
def __init__(self, results=None):
|
||||
self.results = list(results or [])
|
||||
|
||||
def filter(self, *_args):
|
||||
return self
|
||||
@@ -37,10 +37,13 @@ class FakeQuery:
|
||||
return self
|
||||
|
||||
def first(self):
|
||||
return self.result
|
||||
return self.results[0] if self.results else None
|
||||
|
||||
def one_or_none(self):
|
||||
return self.first()
|
||||
|
||||
def all(self):
|
||||
return self.result if isinstance(self.result, list) else []
|
||||
return list(self.results)
|
||||
|
||||
|
||||
class FakeSession:
|
||||
@@ -58,6 +61,14 @@ class FakeSession:
|
||||
def add(self, row):
|
||||
self.added.append(row)
|
||||
|
||||
def flush(self):
|
||||
# Exercise the governed source/snapshot import path with database-like
|
||||
# primary-key assignment instead of silently falling back to legacy
|
||||
# fixture behavior.
|
||||
for row in self.added:
|
||||
if getattr(row, "id", None) is None:
|
||||
row.id = uuid4()
|
||||
|
||||
def commit(self):
|
||||
return None
|
||||
|
||||
@@ -67,8 +78,18 @@ class FakeSession:
|
||||
def refresh(self, row):
|
||||
return row
|
||||
|
||||
def query(self, _model):
|
||||
return FakeQuery(self.query_result)
|
||||
def query(self, model):
|
||||
rows = [
|
||||
row
|
||||
for (row_model, _row_id), row in self.rows.items()
|
||||
if row_model is model and isinstance(row, model)
|
||||
]
|
||||
rows.extend(row for row in self.added if isinstance(row, model))
|
||||
if isinstance(self.query_result, model):
|
||||
rows.append(self.query_result)
|
||||
elif isinstance(self.query_result, list):
|
||||
rows.extend(row for row in self.query_result if isinstance(row, model))
|
||||
return FakeQuery(rows)
|
||||
|
||||
|
||||
class FakeResponse:
|
||||
@@ -318,6 +339,8 @@ def test_dhmv_acquisition_clips_validates_and_persists_via_dataset_service(tmp_p
|
||||
|
||||
dataset = next(item for item in db.added if isinstance(item, Dataset))
|
||||
version = next(item for item in db.added if isinstance(item, DatasetVersion))
|
||||
source = next(item for item in db.added if isinstance(item, SourceRegistry))
|
||||
snapshot = next(item for item in db.added if isinstance(item, SourceSnapshot))
|
||||
assert result["output_dataset_id"] == str(dataset.id)
|
||||
assert dataset.source_name == "digitaal_vlaanderen_dhmv"
|
||||
assert dataset.area_id == area_id
|
||||
@@ -331,6 +354,16 @@ def test_dhmv_acquisition_clips_validates_and_persists_via_dataset_service(tmp_p
|
||||
assert dataset.provenance_metadata["water_depth_available"] is False
|
||||
assert dataset.provenance_metadata["water_volume_available"] is False
|
||||
assert len(dataset.provenance_metadata["response_sha256"]) == 64
|
||||
assert dataset.source_registry_id == source.id
|
||||
assert dataset.source_snapshot_id == snapshot.id
|
||||
assert dataset.validation_status == "passed"
|
||||
assert dataset.provenance_status == "complete"
|
||||
assert dataset.lineage_status == "not_applicable"
|
||||
assert dataset.quarantine_status == "not_quarantined"
|
||||
assert snapshot.source_registry_id == source.id
|
||||
assert snapshot.checksum_sha256 == dataset.checksum_sha256
|
||||
assert snapshot.ingest_status == "ingested"
|
||||
assert snapshot.freshness_status == "current"
|
||||
with rasterio.open(dataset.storage_path) as stored:
|
||||
assert stored.crs.to_epsg() == 31370
|
||||
assert stored.count == 1
|
||||
|
||||
@@ -1,8 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
@@ -10,12 +11,13 @@ from fastapi.testclient import TestClient
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.main import app
|
||||
from app.models import Dataset, Export
|
||||
from app.models import Dataset, Export, SourceRegistry, SourceSnapshot
|
||||
from app.schemas.export import ExportCreateResponse, MapResultExportRequest
|
||||
from app.schemas.project import ProjectRead
|
||||
from app.services.export_service import ExportService
|
||||
from app.services.project_service import ProjectService
|
||||
from app.services.storage_service import StorageService
|
||||
from app.services.source_registry_service import SourceRegistryService
|
||||
from app.services.temporal_analysis_service import TemporalAnalysisService
|
||||
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
|
||||
|
||||
@@ -48,6 +50,71 @@ def bbox_payload() -> dict:
|
||||
}
|
||||
|
||||
|
||||
def governed_dataset(
|
||||
*,
|
||||
project_id,
|
||||
dataset_id,
|
||||
name: str,
|
||||
dataset_type: str,
|
||||
source_key: str,
|
||||
dataset_role: str = "source",
|
||||
) -> Dataset:
|
||||
"""Build an in-memory stand-in for a passed governed dataset.
|
||||
|
||||
Map-result export is an operational consumption boundary. These tests
|
||||
must therefore model the same source registry/snapshot, checksum and
|
||||
passed-contract evidence supplied by a real adapter rather than relying
|
||||
on an old transient Dataset fixture.
|
||||
"""
|
||||
|
||||
source = SourceRegistry(
|
||||
id=uuid4(),
|
||||
**SourceRegistryService.definition_for(source_key).as_model_values(),
|
||||
)
|
||||
checksum = sha256(f"{dataset_id}:{source_key}:{dataset_type}".encode("utf-8")).hexdigest()
|
||||
snapshot = SourceSnapshot(
|
||||
id=uuid4(),
|
||||
source_registry_id=source.id,
|
||||
snapshot_key=f"test:{source_key}:{checksum}",
|
||||
checksum_sha256=checksum,
|
||||
fetched_at=datetime.now(UTC),
|
||||
crs="EPSG:31370",
|
||||
units=source.default_units,
|
||||
spatial_resolution_json={"x": 1.0, "y": 1.0, "unit": "m"},
|
||||
temporal_coverage_json={"status": "test-fixture"},
|
||||
geographic_coverage_json={"zone": "Flanders"},
|
||||
observed_schema_json={"dataset_type": dataset_type},
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
known_limitations_json=["In-memory governed fixture used only by this export test."],
|
||||
snapshot_metadata_json={"fixture_mode": True},
|
||||
)
|
||||
return Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name=name,
|
||||
dataset_type=dataset_type,
|
||||
source="governed test fixture",
|
||||
dataset_role=dataset_role,
|
||||
source_name=source.source_key,
|
||||
source_registry_id=source.id,
|
||||
source_snapshot_id=snapshot.id,
|
||||
source_registry=source,
|
||||
source_snapshot=snapshot,
|
||||
data_contract_key=("geointel.raster.geotiff" if dataset_type == "raster" else "geointel.vector.geojson"),
|
||||
data_contract_version="1.0.0",
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="not_applicable",
|
||||
quarantine_status="not_quarantined",
|
||||
checksum_sha256=checksum,
|
||||
metadata_json={"fixture_mode": True},
|
||||
source_metadata={"fixture_mode": True, "source_registry_key": source.source_key},
|
||||
provenance_metadata={"fixture_mode": True, "source_snapshot_id": str(snapshot.id)},
|
||||
status="ready",
|
||||
)
|
||||
|
||||
|
||||
def test_map_result_export_request_requires_a_complete_target() -> None:
|
||||
with pytest.raises(ValidationError):
|
||||
MapResultExportRequest(project_id=uuid4(), mode="current", bbox=bbox_payload())
|
||||
@@ -67,13 +134,12 @@ def test_current_vector_map_result_uses_authoritative_selection_export(monkeypat
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
area_id = uuid4()
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
dataset = governed_dataset(
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
name="buildings.geojson",
|
||||
dataset_type="vector",
|
||||
source="fixture",
|
||||
status="ready",
|
||||
source_key="grb",
|
||||
)
|
||||
db = FakeSession({(Dataset, dataset_id): dataset})
|
||||
expected = ExportCreateResponse(
|
||||
@@ -109,14 +175,12 @@ def test_current_vector_map_result_uses_authoritative_selection_export(monkeypat
|
||||
def test_partitioned_vector_map_result_uses_governed_partition_export(monkeypatch) -> None:
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
dataset = governed_dataset(
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
name="vha-municipality.geojson",
|
||||
dataset_type="vector",
|
||||
source="VHA",
|
||||
source_name="vmm_vha_bathymetry_profiles",
|
||||
status="ready",
|
||||
source_key="vmm_vha_bathymetry_profiles",
|
||||
)
|
||||
db = FakeSession({(Dataset, dataset_id): dataset})
|
||||
expected = ExportCreateResponse(
|
||||
@@ -159,14 +223,12 @@ def test_partitioned_vector_map_result_uses_governed_partition_export(monkeypatc
|
||||
def test_raster_map_result_is_recomputed_and_persisted(tmp_path, monkeypatch) -> None:
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
dataset = governed_dataset(
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
name="space-occupation.tif",
|
||||
dataset_type="raster",
|
||||
source="official",
|
||||
source_name="department_omgeving_thematic_raster",
|
||||
status="ready",
|
||||
source_key="department_omgeving_thematic_raster",
|
||||
)
|
||||
db = FakeSession({(Dataset, dataset_id): dataset})
|
||||
export_path = tmp_path / "space-occupation-analysis.json"
|
||||
@@ -209,7 +271,26 @@ def test_evolution_map_result_is_recomputed_and_persisted(tmp_path, monkeypatch)
|
||||
project_id = uuid4()
|
||||
earlier_id = uuid4()
|
||||
later_id = uuid4()
|
||||
db = FakeSession()
|
||||
earlier_dataset = governed_dataset(
|
||||
project_id=project_id,
|
||||
dataset_id=earlier_id,
|
||||
name="forest-earlier.geojson",
|
||||
dataset_type="vector",
|
||||
source_key="inbo_bwk_natura2000",
|
||||
)
|
||||
later_dataset = governed_dataset(
|
||||
project_id=project_id,
|
||||
dataset_id=later_id,
|
||||
name="forest-later.geojson",
|
||||
dataset_type="vector",
|
||||
source_key="inbo_bwk_natura2000",
|
||||
)
|
||||
db = FakeSession(
|
||||
{
|
||||
(Dataset, earlier_id): earlier_dataset,
|
||||
(Dataset, later_id): later_dataset,
|
||||
}
|
||||
)
|
||||
export_path = tmp_path / "forest-evolution.json"
|
||||
captured: dict = {}
|
||||
|
||||
|
||||
@@ -6,6 +6,7 @@ from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from geoalchemy2.shape import to_shape
|
||||
from pyproj import Transformer
|
||||
|
||||
from app.api.routes.qa import compare_candidate_with_reference
|
||||
from app.models import Dataset, Metric, Project, QualityCheck, VectorFeature
|
||||
@@ -117,6 +118,54 @@ def test_vector_feature_service_normalizes_source_z_coordinates_to_canonical_2d(
|
||||
assert to_shape(persisted[0].geometry).has_z is False
|
||||
|
||||
|
||||
def test_vector_feature_service_transforms_declared_source_crs_before_epsg4326_storage() -> None:
|
||||
to_lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
||||
x, y = to_lambert.transform(4.7, 51.1)
|
||||
db = FakeSession()
|
||||
|
||||
persisted = VectorFeatureService.persist_geojson_features(
|
||||
db=db,
|
||||
dataset_id=uuid4(),
|
||||
payload={
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"id": "lambert-point",
|
||||
"properties": {},
|
||||
"geometry": {"type": "Point", "coordinates": [x, y]},
|
||||
}
|
||||
],
|
||||
},
|
||||
source_crs="EPSG:31370",
|
||||
)
|
||||
|
||||
geometry = to_shape(persisted[0].geometry)
|
||||
assert geometry.x == pytest.approx(4.7, abs=0.000001)
|
||||
assert geometry.y == pytest.approx(51.1, abs=0.000001)
|
||||
|
||||
|
||||
def test_vector_feature_service_rejects_invalid_declared_source_crs() -> None:
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
VectorFeatureService.persist_geojson_features(
|
||||
db=FakeSession(),
|
||||
dataset_id=uuid4(),
|
||||
payload={
|
||||
"type": "FeatureCollection",
|
||||
"features": [
|
||||
{
|
||||
"type": "Feature",
|
||||
"properties": {},
|
||||
"geometry": {"type": "Point", "coordinates": [4.7, 51.1]},
|
||||
}
|
||||
],
|
||||
},
|
||||
source_crs="EPSG:not-a-crs",
|
||||
)
|
||||
|
||||
assert getattr(exc_info.value, "code", None) == "INVALID_DATASET_CRS"
|
||||
|
||||
|
||||
def test_dataset_upload_persists_vector_features(monkeypatch, tmp_path) -> None:
|
||||
project_id = uuid4()
|
||||
db = FakeSession(objects={(Project, project_id): Project(id=project_id, name="Geel")})
|
||||
|
||||
@@ -43,7 +43,7 @@ def _project_and_dataset(dataset_type: str = "raster"):
|
||||
project_id=project_id,
|
||||
name="source.tif",
|
||||
dataset_type=dataset_type,
|
||||
source="user_upload",
|
||||
source="test",
|
||||
storage_path="storage/uploads/source.tif",
|
||||
)
|
||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
@@ -10,10 +11,11 @@ import pytest
|
||||
|
||||
from app.core.config import Settings
|
||||
from app.core.errors import AppError
|
||||
from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project
|
||||
from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project, SourceRegistry, SourceSnapshot
|
||||
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.model_registry_service import ModelRegistryService
|
||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||
from app.services.yolo_adapter import YoloDetectionAdapter
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
@@ -79,6 +81,14 @@ class MockYoloAdapter:
|
||||
]
|
||||
|
||||
|
||||
class NeverLoadUnboundModelAdapter(MockYoloAdapter):
|
||||
load_calls = 0
|
||||
|
||||
def load_model(self, model_path: Path):
|
||||
type(self).load_calls += 1
|
||||
raise AssertionError("unbound model provenance must be rejected before adapter.load_model")
|
||||
|
||||
|
||||
class MixedCaseYoloAdapter(MockYoloAdapter):
|
||||
def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict]:
|
||||
return [
|
||||
@@ -141,15 +151,47 @@ class ExplodingPredictModel:
|
||||
def _project_and_dataset(dataset_type: str = "raster"):
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
source_registry_id = uuid4()
|
||||
source_snapshot_id = uuid4()
|
||||
checksum = "a" * 64
|
||||
project = Project(id=project_id, name="Geel")
|
||||
source_registry = SourceRegistry(
|
||||
id=source_registry_id,
|
||||
source_key="test-derived-raster",
|
||||
display_name="Governed test-derived raster",
|
||||
classification="derived",
|
||||
authority_name="GeoIntel test fixture",
|
||||
usage_policy_json={"ground_truth_allowed": False},
|
||||
)
|
||||
source_snapshot = SourceSnapshot(
|
||||
id=source_snapshot_id,
|
||||
source_registry_id=source_registry_id,
|
||||
snapshot_key="test-derived-raster-v1",
|
||||
checksum_sha256=checksum,
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
)
|
||||
dataset = Dataset(
|
||||
id=dataset_id,
|
||||
project_id=project_id,
|
||||
name="source.tif",
|
||||
dataset_type=dataset_type,
|
||||
source="user_upload",
|
||||
source="test-derived-raster",
|
||||
source_name="test-derived-raster",
|
||||
storage_path="storage/uploads/source.tif",
|
||||
checksum_sha256=checksum,
|
||||
source_registry_id=source_registry_id,
|
||||
source_snapshot_id=source_snapshot_id,
|
||||
data_contract_key="geointel.raster.geotiff",
|
||||
data_contract_version="1.0.0",
|
||||
validation_status="passed",
|
||||
provenance_status="complete",
|
||||
lineage_status="not_applicable",
|
||||
quarantine_status="not_quarantined",
|
||||
status="ready",
|
||||
)
|
||||
dataset.source_registry = source_registry
|
||||
dataset.source_snapshot = source_snapshot
|
||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||
return db, project_id, dataset_id
|
||||
|
||||
@@ -165,6 +207,74 @@ def _settings(tmp_path: Path, **overrides) -> Settings:
|
||||
return Settings(**values)
|
||||
|
||||
|
||||
def _write_model_sidecar(
|
||||
model_path: Path,
|
||||
settings: Settings,
|
||||
*,
|
||||
db: FakeSession | None = None,
|
||||
) -> None:
|
||||
"""Create explicit test-only evidence; production never self-generates it."""
|
||||
|
||||
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
|
||||
source_registry_id = uuid4()
|
||||
source_snapshot_id = uuid4()
|
||||
source_version = settings.yolo_model_version or "test-v1"
|
||||
if db is not None:
|
||||
source_registry = SourceRegistry(
|
||||
id=source_registry_id,
|
||||
source_key="model",
|
||||
display_name="Governed test model artifact",
|
||||
classification="experimental",
|
||||
authority_name="GeoIntel test fixture",
|
||||
freshness_status="current",
|
||||
ingest_status="configured",
|
||||
)
|
||||
source_snapshot = SourceSnapshot(
|
||||
id=source_snapshot_id,
|
||||
source_registry_id=source_registry_id,
|
||||
snapshot_key=f"model-{source_version}",
|
||||
source_version=source_version,
|
||||
checksum_sha256=model_sha256,
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
)
|
||||
db.objects[(SourceRegistry, source_registry_id)] = source_registry
|
||||
db.objects[(SourceSnapshot, source_snapshot_id)] = source_snapshot
|
||||
payload = {
|
||||
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
|
||||
"data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"},
|
||||
"model": {
|
||||
"model_id": settings.yolo_model_id,
|
||||
"task_type": "object_detection",
|
||||
"sha256": model_sha256,
|
||||
"model_format": "pytorch",
|
||||
"framework": "ultralytics/pytorch",
|
||||
"class_mapping": {"0": "building"},
|
||||
"source_version": source_version,
|
||||
},
|
||||
"source": {
|
||||
"source_registry_id": str(source_registry_id),
|
||||
"source_snapshot_id": str(source_snapshot_id),
|
||||
"source_registry_key": "model",
|
||||
"source_snapshot_checksum_sha256": model_sha256,
|
||||
},
|
||||
"lineage": {
|
||||
"upstream_asset_ids": ["test-training-corpus"],
|
||||
"upstream_checksums_sha256": ["a" * 64],
|
||||
"transformations": [
|
||||
{"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64}
|
||||
],
|
||||
},
|
||||
"metadata": {"training_manifest_sha256": "c" * 64},
|
||||
"imported_at": "2026-08-01T10:00:00+00:00",
|
||||
}
|
||||
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
|
||||
RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text(
|
||||
json.dumps(payload, sort_keys=True),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def _manifest(tmp_path: Path, tile_count: int = 1) -> Path:
|
||||
tiles = []
|
||||
for index in range(tile_count):
|
||||
@@ -223,10 +333,28 @@ def test_yolo_configured_model_reports_dependency_unavailable(tmp_path: Path) ->
|
||||
assert model.status == "dependency_unavailable"
|
||||
|
||||
|
||||
def test_yolo_configured_model_requires_a_runtime_provenance_sidecar(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"unmanifested local weights")
|
||||
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
||||
|
||||
model = ModelRegistryService.get_model_capability(
|
||||
"yolo-configured",
|
||||
settings=settings,
|
||||
yolo_adapter_class=AvailableAdapter,
|
||||
)
|
||||
|
||||
assert model is not None
|
||||
assert model.configured is False
|
||||
assert model.status == "contract_incomplete"
|
||||
assert "sidecar" in model.limitation_message
|
||||
|
||||
|
||||
def test_yolo_configured_model_reports_configured_with_local_model_and_dependencies(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"local weights")
|
||||
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
||||
_write_model_sidecar(model_path, settings)
|
||||
|
||||
model = ModelRegistryService.get_model_capability("yolo-configured", settings=settings, yolo_adapter_class=AvailableAdapter)
|
||||
|
||||
@@ -306,11 +434,37 @@ def test_yolo_run_requires_tile_manifest_path(tmp_path: Path) -> None:
|
||||
assert getattr(exc_info.value, "code", None) == "DETECTION_TILE_MANIFEST_REQUIRED"
|
||||
|
||||
|
||||
def test_yolo_run_fails_closed_before_adapter_load_without_sidecar(tmp_path: Path) -> None:
|
||||
db, project_id, dataset_id = _project_and_dataset()
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"unmanifested local weights")
|
||||
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
||||
|
||||
# AvailableAdapter intentionally has no load_model method. If runtime
|
||||
# provenance were checked after adapter loading, this would raise instead
|
||||
# of returning the explicit unavailable capability state.
|
||||
result = DetectionService.run_detection(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
model_id="yolo-configured",
|
||||
confidence_threshold=0.5,
|
||||
tile_manifest_path=str(_manifest(tmp_path)),
|
||||
settings=settings,
|
||||
yolo_adapter_class=AvailableAdapter,
|
||||
)
|
||||
|
||||
assert result.status == "failed"
|
||||
assert result.error_code == "DETECTION_MODEL_UNAVAILABLE"
|
||||
assert "sidecar" in result.message
|
||||
|
||||
|
||||
def test_yolo_run_rejects_manifest_over_tile_limit(tmp_path: Path) -> None:
|
||||
db, project_id, dataset_id = _project_and_dataset()
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"local weights")
|
||||
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_max_tiles=1)
|
||||
_write_model_sidecar(model_path, settings, db=db)
|
||||
manifest_path = _manifest(tmp_path, tile_count=2)
|
||||
|
||||
result = DetectionService.run_detection(
|
||||
@@ -333,6 +487,7 @@ def test_yolo_run_rejects_missing_tile_manifest_file(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"local weights")
|
||||
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
||||
_write_model_sidecar(model_path, settings, db=db)
|
||||
|
||||
result = DetectionService.run_detection(
|
||||
db=db,
|
||||
@@ -354,6 +509,7 @@ def test_yolo_run_rejects_invalid_tile_manifest_json(tmp_path: Path) -> None:
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"local weights")
|
||||
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
||||
_write_model_sidecar(model_path, settings, db=db)
|
||||
manifest_path = tmp_path / "manifest.json"
|
||||
manifest_path.write_text("{not-json", encoding="utf-8")
|
||||
|
||||
@@ -372,6 +528,32 @@ def test_yolo_run_rejects_invalid_tile_manifest_json(tmp_path: Path) -> None:
|
||||
assert result.error_code == "DETECTION_TILE_MANIFEST_INVALID"
|
||||
|
||||
|
||||
def test_yolo_run_rejects_unbound_model_snapshot_before_adapter_load(tmp_path: Path) -> None:
|
||||
db, project_id, dataset_id = _project_and_dataset()
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"structurally valid but unbound model")
|
||||
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
||||
# A catalog/preflight sidecar alone is deliberately insufficient for a
|
||||
# production call. Do not register the declared source IDs in ``db``.
|
||||
_write_model_sidecar(model_path, settings)
|
||||
NeverLoadUnboundModelAdapter.load_calls = 0
|
||||
|
||||
result = DetectionService.run_detection(
|
||||
db=db,
|
||||
project_id=project_id,
|
||||
dataset_id=dataset_id,
|
||||
model_id="yolo-configured",
|
||||
confidence_threshold=0.5,
|
||||
tile_manifest_path=str(_manifest(tmp_path)),
|
||||
settings=settings,
|
||||
yolo_adapter_class=NeverLoadUnboundModelAdapter,
|
||||
)
|
||||
|
||||
assert result.status == "failed"
|
||||
assert result.error_code == "MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND"
|
||||
assert NeverLoadUnboundModelAdapter.load_calls == 0
|
||||
|
||||
|
||||
def test_pixel_bbox_to_epsg4326_polygon_from_gdal_transform() -> None:
|
||||
polygon = pixel_bbox_to_epsg4326_polygon(
|
||||
bbox=[10, 20, 30, 40],
|
||||
@@ -390,6 +572,7 @@ def test_yolo_run_persists_mocked_georeferenced_detections(tmp_path: Path) -> No
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"local weights")
|
||||
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_model_version="local-test")
|
||||
_write_model_sidecar(model_path, settings, db=db)
|
||||
manifest_path = _manifest(tmp_path, tile_count=1)
|
||||
|
||||
result = DetectionService.run_detection(
|
||||
@@ -416,7 +599,10 @@ def test_yolo_run_persists_mocked_georeferenced_detections(tmp_path: Path) -> No
|
||||
assert detections[0].confidence == 0.91
|
||||
assert detections[0].source_tile_path.endswith("tile_0000.tif")
|
||||
assert detections[0].bbox_json == {"x_min": 10.0, "y_min": 20.0, "x_max": 30.0, "y_max": 40.0}
|
||||
assert detections[0].properties_json == {"adapter": "mock", "tile_index": 0}
|
||||
assert detections[0].properties_json["adapter"] == "mock"
|
||||
assert detections[0].properties_json["tile_index"] == 0
|
||||
assert detections[0].properties_json["runtime_model_provenance"]["model_sha256"] == sha256(model_path.read_bytes()).hexdigest()
|
||||
assert runs[0].parameters_json["runtime_model_provenance"]["data_contract_key"] == "geointel.model.pytorch"
|
||||
assert runs[0].status == "success"
|
||||
assert jobs[0].status == "success"
|
||||
|
||||
@@ -426,6 +612,7 @@ def test_yolo_class_filter_is_case_insensitive_and_persists_canonical_class(tmp_
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"local weights")
|
||||
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
||||
_write_model_sidecar(model_path, settings, db=db)
|
||||
manifest_path = _manifest(tmp_path, tile_count=1)
|
||||
|
||||
result = DetectionService.run_detection(
|
||||
@@ -453,6 +640,7 @@ def test_yolo_run_suppresses_cross_tile_duplicate_detections(tmp_path: Path) ->
|
||||
model_path = tmp_path / "model.pt"
|
||||
model_path.write_bytes(b"local weights")
|
||||
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_duplicate_iou_threshold=0.5)
|
||||
_write_model_sidecar(model_path, settings, db=db)
|
||||
manifest_path = _manifest(tmp_path, tile_count=2)
|
||||
|
||||
result = DetectionService.run_detection(
|
||||
|
||||
@@ -12,7 +12,7 @@ from shapely.geometry import Polygon, box
|
||||
from app.core.errors import AppError
|
||||
from app.main import app
|
||||
from app.db.session import get_db
|
||||
from app.models import AnalysisRun, Dataset, Detection, Metric, QualityCheck, VectorFeature
|
||||
from app.models import AnalysisRun, Dataset, Detection, Metric, QualityCheck, SourceRegistry, SourceSnapshot, VectorFeature
|
||||
from app.services.detection_service import DetectionService
|
||||
|
||||
|
||||
@@ -96,11 +96,52 @@ def _source_dataset(project_id, dataset_id):
|
||||
project_id=project_id,
|
||||
name="source.tif",
|
||||
dataset_type="raster",
|
||||
source="manual",
|
||||
source_name="manual",
|
||||
source="test",
|
||||
source_name="test",
|
||||
)
|
||||
|
||||
|
||||
def _authoritative_reference(dataset: Dataset) -> Dataset:
|
||||
"""Model the reference as a fully governed GRB fixture, never test data."""
|
||||
|
||||
source_id = uuid4()
|
||||
snapshot_id = uuid4()
|
||||
checksum = "a" * 64
|
||||
source = SourceRegistry(
|
||||
id=source_id,
|
||||
source_key="grb",
|
||||
display_name="GRB QA fixture",
|
||||
classification="authoritative",
|
||||
authority_name="Digitaal Vlaanderen",
|
||||
authority_scope_json={"zone": "Flanders"},
|
||||
usage_policy_json={"ground_truth_allowed": True, "validation_authority": {"building_validation": "primary"}},
|
||||
)
|
||||
snapshot = SourceSnapshot(
|
||||
id=snapshot_id,
|
||||
source_registry_id=source_id,
|
||||
snapshot_key=f"detection-qa-{dataset.id}",
|
||||
checksum_sha256=checksum,
|
||||
ingest_status="ingested",
|
||||
freshness_status="current",
|
||||
)
|
||||
dataset.source = "grb"
|
||||
dataset.source_name = "grb"
|
||||
dataset.dataset_role = "reference"
|
||||
dataset.status = "ready"
|
||||
dataset.checksum_sha256 = checksum
|
||||
dataset.source_registry_id = source_id
|
||||
dataset.source_snapshot_id = snapshot_id
|
||||
dataset.data_contract_key = "geointel.vector.geojson"
|
||||
dataset.data_contract_version = "1.0.0"
|
||||
dataset.validation_status = "passed"
|
||||
dataset.provenance_status = "complete"
|
||||
dataset.lineage_status = "complete"
|
||||
dataset.quarantine_status = "not_quarantined"
|
||||
dataset.source_registry = source
|
||||
dataset.source_snapshot = snapshot
|
||||
return dataset
|
||||
|
||||
|
||||
def test_detection_geojson_feature_collection_shape() -> None:
|
||||
project_id = uuid4()
|
||||
dataset_id = uuid4()
|
||||
@@ -201,14 +242,14 @@ def test_detection_qa_persists_quality_check_and_metrics() -> None:
|
||||
reference_dataset_id = uuid4()
|
||||
analysis_run_id = uuid4()
|
||||
detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1))
|
||||
reference_dataset = Dataset(
|
||||
reference_dataset = _authoritative_reference(Dataset(
|
||||
id=reference_dataset_id,
|
||||
project_id=project_id,
|
||||
name="reference.geojson",
|
||||
dataset_type="vector",
|
||||
source="manual",
|
||||
source="test",
|
||||
dataset_role="reference",
|
||||
)
|
||||
))
|
||||
reference_feature = VectorFeature(
|
||||
id=uuid4(),
|
||||
dataset_id=reference_dataset_id,
|
||||
@@ -262,7 +303,7 @@ def test_detection_qa_rejects_non_overlapping_historical_reference_editions() ->
|
||||
source_dataset.source_metadata = {"product_key": "2020", "supports_detection": False}
|
||||
source_dataset.valid_from = datetime(2020, 1, 1, tzinfo=UTC)
|
||||
source_dataset.valid_to = datetime(2020, 12, 31, 23, 59, 59, tzinfo=UTC)
|
||||
reference_dataset = Dataset(
|
||||
reference_dataset = _authoritative_reference(Dataset(
|
||||
id=reference_dataset_id,
|
||||
project_id=project_id,
|
||||
name="current-grb.geojson",
|
||||
@@ -272,7 +313,7 @@ def test_detection_qa_rejects_non_overlapping_historical_reference_editions() ->
|
||||
dataset_role="reference",
|
||||
valid_from=datetime(2026, 7, 1, tzinfo=UTC),
|
||||
valid_to=datetime(2026, 7, 31, 23, 59, 59, tzinfo=UTC),
|
||||
)
|
||||
))
|
||||
db = FakeSession(
|
||||
objects={
|
||||
(AnalysisRun, analysis_run_id): AnalysisRun(
|
||||
@@ -305,14 +346,14 @@ def test_detection_qa_no_match_case_persists_zero_scores() -> None:
|
||||
reference_dataset_id = uuid4()
|
||||
analysis_run_id = uuid4()
|
||||
detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1))
|
||||
reference_dataset = Dataset(
|
||||
reference_dataset = _authoritative_reference(Dataset(
|
||||
id=reference_dataset_id,
|
||||
project_id=project_id,
|
||||
name="reference.geojson",
|
||||
dataset_type="vector",
|
||||
source="manual",
|
||||
source="test",
|
||||
dataset_role="reference",
|
||||
)
|
||||
))
|
||||
reference_feature = VectorFeature(
|
||||
id=uuid4(),
|
||||
dataset_id=reference_dataset_id,
|
||||
@@ -349,14 +390,14 @@ def test_configured_yolo_qa_requires_persisted_tile_manifest_provenance() -> Non
|
||||
reference_dataset_id = uuid4()
|
||||
analysis_run_id = uuid4()
|
||||
detection = _detection(project_id, dataset_id, analysis_run_id)
|
||||
reference_dataset = Dataset(
|
||||
reference_dataset = _authoritative_reference(Dataset(
|
||||
id=reference_dataset_id,
|
||||
project_id=project_id,
|
||||
name="reference.geojson",
|
||||
dataset_type="vector",
|
||||
source="manual",
|
||||
source="test",
|
||||
dataset_role="reference",
|
||||
)
|
||||
))
|
||||
reference_feature = VectorFeature(
|
||||
id=uuid4(),
|
||||
dataset_id=reference_dataset_id,
|
||||
@@ -421,14 +462,14 @@ def test_detection_qa_excludes_references_outside_persisted_tile_coverage(tmp_pa
|
||||
analysis_run_id = uuid4()
|
||||
manifest_path = _coverage_manifest(tmp_path, dataset_id, bounds=(0.0, 0.0, 1.0, 1.0))
|
||||
detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0.1, 0.1, 0.9, 0.9))
|
||||
reference_dataset = Dataset(
|
||||
reference_dataset = _authoritative_reference(Dataset(
|
||||
id=reference_dataset_id,
|
||||
project_id=project_id,
|
||||
name="reference.geojson",
|
||||
dataset_type="vector",
|
||||
source="manual",
|
||||
source="test",
|
||||
dataset_role="reference",
|
||||
)
|
||||
))
|
||||
inside_reference = VectorFeature(
|
||||
id=uuid4(),
|
||||
dataset_id=reference_dataset_id,
|
||||
@@ -489,14 +530,14 @@ def test_detection_qa_reports_box_to_footprint_diagnostic_without_changing_stric
|
||||
l_shaped_footprint = Polygon(
|
||||
[(0.0, 0.0), (2.0, 0.0), (2.0, 0.4), (0.4, 0.4), (0.4, 2.0), (0.0, 2.0), (0.0, 0.0)]
|
||||
)
|
||||
reference_dataset = Dataset(
|
||||
reference_dataset = _authoritative_reference(Dataset(
|
||||
id=reference_dataset_id,
|
||||
project_id=project_id,
|
||||
name="reference.geojson",
|
||||
dataset_type="vector",
|
||||
source="manual",
|
||||
source="test",
|
||||
dataset_role="reference",
|
||||
)
|
||||
))
|
||||
reference_feature = VectorFeature(
|
||||
id=uuid4(),
|
||||
dataset_id=reference_dataset_id,
|
||||
|
||||
@@ -10,7 +10,18 @@ from shapely.geometry import MultiPolygon, box, mapping
|
||||
|
||||
from app.db.session import get_db
|
||||
from app.main import app
|
||||
from app.models import AnalysisRun, Dataset, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature
|
||||
from app.models import (
|
||||
AnalysisRun,
|
||||
Dataset,
|
||||
Job,
|
||||
Metric,
|
||||
Project,
|
||||
QualityCheck,
|
||||
Segmentation,
|
||||
SourceRegistry,
|
||||
SourceSnapshot,
|
||||
VectorFeature,
|
||||
)
|
||||
from app.services.model_registry_service import ModelRegistryService
|
||||
from app.services.segmentation_service import SegmentationService
|
||||
|
||||
@@ -75,7 +86,7 @@ def _project_and_dataset(dataset_type: str = "raster"):
|
||||
project_id=project_id,
|
||||
name="source.tif",
|
||||
dataset_type=dataset_type,
|
||||
source="user_upload",
|
||||
source="test",
|
||||
storage_path="storage/uploads/source.tif",
|
||||
)
|
||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||
@@ -105,6 +116,47 @@ def _segmentation(project_id, dataset_id, analysis_run_id, class_name="vegetatio
|
||||
)
|
||||
|
||||
|
||||
def _authoritative_reference(dataset: Dataset) -> Dataset:
|
||||
"""Model segmentation QA references as governed authoritative GRB evidence."""
|
||||
|
||||
source_id = uuid4()
|
||||
snapshot_id = uuid4()
|
||||
checksum = "a" * 64
|
||||
source = SourceRegistry(
|
||||
id=source_id,
|
||||
source_key="grb",
|
||||
display_name="GRB segmentation QA fixture",
|
||||
classification="authoritative",
|
||||
authority_name="Digitaal Vlaanderen",
|
||||
authority_scope_json={"zone": "Flanders"},
|
||||
usage_policy_json={"ground_truth_allowed": True, "validation_authority": {"building_validation": "primary"}},
|
||||
)
|
||||
snapshot = SourceSnapshot(
|
||||
id=snapshot_id,
|
||||
source_registry_id=source_id,
|
||||
snapshot_key=f"segmentation-qa-{dataset.id}",
|
||||
checksum_sha256=checksum,
|
||||
ingest_status="ingested",
|
||||
freshness_status="current",
|
||||
)
|
||||
dataset.source = "grb"
|
||||
dataset.source_name = "grb"
|
||||
dataset.dataset_role = "reference"
|
||||
dataset.status = "ready"
|
||||
dataset.checksum_sha256 = checksum
|
||||
dataset.source_registry_id = source_id
|
||||
dataset.source_snapshot_id = snapshot_id
|
||||
dataset.data_contract_key = "geointel.vector.geojson"
|
||||
dataset.data_contract_version = "1.0.0"
|
||||
dataset.validation_status = "passed"
|
||||
dataset.provenance_status = "complete"
|
||||
dataset.lineage_status = "complete"
|
||||
dataset.quarantine_status = "not_quarantined"
|
||||
dataset.source_registry = source
|
||||
dataset.source_snapshot = snapshot
|
||||
return dataset
|
||||
|
||||
|
||||
def test_model_registry_returns_segmentation_states() -> None:
|
||||
models = {model.model_id: model for model in ModelRegistryService.list_model_capabilities(task_type="segmentation")}
|
||||
|
||||
@@ -265,14 +317,14 @@ def test_segmentation_qa_persists_quality_check_and_metrics() -> None:
|
||||
reference_dataset_id = uuid4()
|
||||
analysis_run_id = uuid4()
|
||||
segmentation = _segmentation(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1))
|
||||
reference_dataset = Dataset(
|
||||
reference_dataset = _authoritative_reference(Dataset(
|
||||
id=reference_dataset_id,
|
||||
project_id=project_id,
|
||||
name="reference.geojson",
|
||||
dataset_type="vector",
|
||||
source="manual",
|
||||
source="test",
|
||||
dataset_role="reference",
|
||||
)
|
||||
))
|
||||
reference_feature = VectorFeature(
|
||||
id=uuid4(),
|
||||
dataset_id=reference_dataset_id,
|
||||
@@ -282,6 +334,7 @@ def test_segmentation_qa_persists_quality_check_and_metrics() -> None:
|
||||
db = FakeSession(
|
||||
objects={
|
||||
(AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="segmentation", status="success", parameters_json={}),
|
||||
(Dataset, dataset_id): Dataset(id=dataset_id, project_id=project_id, name="fixture.tif", dataset_type="raster", source="test"),
|
||||
(Dataset, reference_dataset_id): reference_dataset,
|
||||
},
|
||||
query_rows={Segmentation: [segmentation], VectorFeature: [reference_feature]},
|
||||
@@ -318,14 +371,14 @@ def test_segmentation_qa_no_match_case_persists_zero_scores() -> None:
|
||||
reference_dataset_id = uuid4()
|
||||
analysis_run_id = uuid4()
|
||||
segmentation = _segmentation(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1))
|
||||
reference_dataset = Dataset(
|
||||
reference_dataset = _authoritative_reference(Dataset(
|
||||
id=reference_dataset_id,
|
||||
project_id=project_id,
|
||||
name="reference.geojson",
|
||||
dataset_type="vector",
|
||||
source="manual",
|
||||
source="test",
|
||||
dataset_role="reference",
|
||||
)
|
||||
))
|
||||
reference_feature = VectorFeature(
|
||||
id=uuid4(),
|
||||
dataset_id=reference_dataset_id,
|
||||
@@ -335,6 +388,7 @@ def test_segmentation_qa_no_match_case_persists_zero_scores() -> None:
|
||||
db = FakeSession(
|
||||
objects={
|
||||
(AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="segmentation", status="success", parameters_json={}),
|
||||
(Dataset, dataset_id): Dataset(id=dataset_id, project_id=project_id, name="fixture.tif", dataset_type="raster", source="test"),
|
||||
(Dataset, reference_dataset_id): reference_dataset,
|
||||
},
|
||||
query_rows={Segmentation: [segmentation], VectorFeature: [reference_feature]},
|
||||
|
||||
@@ -0,0 +1,410 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPT = ROOT / "scripts" / "training_dataset_eligibility.py"
|
||||
SPEC = importlib.util.spec_from_file_location("training_dataset_eligibility", SCRIPT)
|
||||
assert SPEC and SPEC.loader
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
CHECKSUM = "a" * 64
|
||||
UNSET = object()
|
||||
|
||||
|
||||
def source_registry(
|
||||
*,
|
||||
classification: str = "authoritative",
|
||||
training_allowed: bool = True,
|
||||
ground_truth_allowed: bool = True,
|
||||
allowed_tasks: list[str] | None = None,
|
||||
building_validation_authority: str = "primary",
|
||||
) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
id="source-registry-1",
|
||||
source_key="governed-source",
|
||||
classification=classification,
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
usage_policy_json={
|
||||
"training_allowed": training_allowed,
|
||||
"ground_truth_allowed": ground_truth_allowed,
|
||||
"allowed_tasks": allowed_tasks or ["building_validation", "building_labels"],
|
||||
"validation_authority": {
|
||||
"building_validation": building_validation_authority,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def source_snapshot(*, checksum: str = CHECKSUM) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
id="source-snapshot-1",
|
||||
snapshot_key="2026-08-01",
|
||||
checksum_sha256=checksum,
|
||||
freshness_status="current",
|
||||
ingest_status="ingested",
|
||||
)
|
||||
|
||||
|
||||
def governed_dataset(
|
||||
*,
|
||||
role: str,
|
||||
registry: SimpleNamespace | None | object = UNSET,
|
||||
snapshot: SimpleNamespace | None | object = UNSET,
|
||||
**overrides: object,
|
||||
) -> SimpleNamespace:
|
||||
dataset_type = "raster" if role == "raster" else "vector"
|
||||
values: dict[str, object] = {
|
||||
"id": f"dataset-{role}",
|
||||
"dataset_type": dataset_type,
|
||||
"dataset_role": "source" if role == "raster" else "reference",
|
||||
"source": "governed_import",
|
||||
"source_name": "governed-source",
|
||||
"checksum_sha256": CHECKSUM,
|
||||
"data_contract_key": f"{role}-contract",
|
||||
"data_contract_version": "1.0.0",
|
||||
"validation_status": "passed",
|
||||
"provenance_status": "complete",
|
||||
"lineage_status": "not_applicable",
|
||||
"quarantine_status": "not_quarantined",
|
||||
"status": "ready",
|
||||
"metadata_json": {},
|
||||
"provenance_metadata": {},
|
||||
"source_registry": source_registry() if registry is UNSET else registry,
|
||||
"source_snapshot": source_snapshot() if snapshot is UNSET else snapshot,
|
||||
}
|
||||
values.update(overrides)
|
||||
return SimpleNamespace(**values)
|
||||
|
||||
|
||||
def test_governed_authoritative_pair_is_eligible_for_operational_training() -> None:
|
||||
raster = governed_dataset(
|
||||
role="raster",
|
||||
registry=source_registry(ground_truth_allowed=False),
|
||||
)
|
||||
reference = governed_dataset(role="reference")
|
||||
|
||||
decision = MODULE.training_pair_evidence(raster=raster, reference=reference)
|
||||
|
||||
assert decision["eligible"] is True
|
||||
assert decision["raster"]["reasons"] == []
|
||||
assert decision["reference"]["evidence"]["source_ground_truth_allowed"] is True
|
||||
|
||||
|
||||
def test_operational_training_rejects_invalid_quarantined_incomplete_and_untrusted_inputs() -> None:
|
||||
dataset = governed_dataset(
|
||||
role="reference",
|
||||
validation_status="failed",
|
||||
provenance_status="incomplete",
|
||||
lineage_status="incomplete",
|
||||
quarantine_status="quarantined",
|
||||
source_registry=source_registry(
|
||||
classification="contextual",
|
||||
training_allowed=False,
|
||||
ground_truth_allowed=False,
|
||||
),
|
||||
)
|
||||
|
||||
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
||||
|
||||
assert decision.eligible is False
|
||||
assert set(decision.reasons) >= {
|
||||
"validation_failed",
|
||||
"dataset_quarantined",
|
||||
"provenance_not_complete",
|
||||
"lineage_not_complete",
|
||||
"source_not_allowed_for_training",
|
||||
"reference_source_not_authoritative",
|
||||
"reference_source_not_ground_truth_allowed",
|
||||
}
|
||||
|
||||
|
||||
def test_operational_training_rejects_a_due_source_snapshot() -> None:
|
||||
snapshot = source_snapshot()
|
||||
snapshot.freshness_status = "due"
|
||||
dataset = governed_dataset(role="reference", snapshot=snapshot)
|
||||
|
||||
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
||||
|
||||
assert decision.eligible is False
|
||||
assert "source_snapshot_freshness_not_approved" in decision.reasons
|
||||
|
||||
|
||||
def test_osm_like_context_is_never_accepted_as_building_ground_truth() -> None:
|
||||
dataset = governed_dataset(
|
||||
role="reference",
|
||||
source_name="osm",
|
||||
source_registry=source_registry(
|
||||
classification="contextual",
|
||||
training_allowed=False,
|
||||
ground_truth_allowed=False,
|
||||
),
|
||||
)
|
||||
|
||||
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
||||
|
||||
assert decision.eligible is False
|
||||
assert "source_not_allowed_for_training" in decision.reasons
|
||||
assert "reference_source_not_authoritative" in decision.reasons
|
||||
|
||||
|
||||
def test_regional_building_sources_pending_primary_authority_cannot_enter_training_labels() -> None:
|
||||
for source_key in ("spw_picc", "urbis"):
|
||||
dataset = governed_dataset(
|
||||
role="reference",
|
||||
source_name=source_key,
|
||||
source_registry=source_registry(
|
||||
allowed_tasks=["building_validation", "building_labels"],
|
||||
building_validation_authority="regional_primary_pending_contract",
|
||||
),
|
||||
)
|
||||
|
||||
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
||||
|
||||
assert decision.eligible is False
|
||||
assert "reference_building_validation_not_primary" in decision.reasons
|
||||
|
||||
|
||||
def test_authoritative_source_without_building_validation_task_cannot_be_used_as_a_label_reference() -> None:
|
||||
dataset = governed_dataset(
|
||||
role="reference",
|
||||
source_registry=source_registry(
|
||||
allowed_tasks=["elevation_validation"],
|
||||
building_validation_authority="corroborative",
|
||||
),
|
||||
)
|
||||
|
||||
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
||||
|
||||
assert decision.eligible is False
|
||||
assert set(decision.reasons) >= {
|
||||
"reference_source_not_approved_for_building_validation",
|
||||
"reference_building_validation_not_primary",
|
||||
}
|
||||
|
||||
|
||||
def test_dataset_and_snapshot_registry_bindings_cannot_be_forged() -> None:
|
||||
snapshot = source_snapshot()
|
||||
snapshot.source_registry_id = "different-registry"
|
||||
dataset = governed_dataset(
|
||||
role="reference",
|
||||
registry=source_registry(),
|
||||
snapshot=snapshot,
|
||||
source_registry_id="different-registry",
|
||||
source_snapshot_id="different-snapshot",
|
||||
)
|
||||
|
||||
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
||||
|
||||
assert decision.eligible is False
|
||||
assert set(decision.reasons) >= {
|
||||
"dataset_source_registry_binding_mismatch",
|
||||
"dataset_source_snapshot_binding_mismatch",
|
||||
"source_snapshot_registry_mismatch",
|
||||
}
|
||||
|
||||
|
||||
def test_fixture_mode_only_relaxes_legacy_provenance_for_explicit_fixtures() -> None:
|
||||
fixture = governed_dataset(
|
||||
role="reference",
|
||||
source="fixture",
|
||||
source_name="fixture",
|
||||
validation_status=None,
|
||||
provenance_status="incomplete",
|
||||
lineage_status="incomplete",
|
||||
data_contract_key=None,
|
||||
data_contract_version=None,
|
||||
checksum_sha256=None,
|
||||
source_registry=None,
|
||||
source_snapshot=None,
|
||||
metadata_json={"fixture": True},
|
||||
)
|
||||
unmarked = governed_dataset(
|
||||
role="reference",
|
||||
validation_status=None,
|
||||
provenance_status="incomplete",
|
||||
lineage_status="incomplete",
|
||||
source_registry=None,
|
||||
source_snapshot=None,
|
||||
)
|
||||
|
||||
assert MODULE.evaluate_dataset_training_eligibility(
|
||||
fixture,
|
||||
role="reference",
|
||||
fixture_mode=True,
|
||||
).eligible is True
|
||||
rejected = MODULE.evaluate_dataset_training_eligibility(
|
||||
unmarked,
|
||||
role="reference",
|
||||
fixture_mode=True,
|
||||
)
|
||||
assert rejected.eligible is False
|
||||
assert "fixture_mode_requires_explicit_fixture" in rejected.reasons
|
||||
|
||||
|
||||
def test_fixture_mode_never_allows_failed_validation_or_quarantine() -> None:
|
||||
fixture = governed_dataset(
|
||||
role="raster",
|
||||
source="fixture",
|
||||
source_name="fixture",
|
||||
validation_status="failed",
|
||||
quarantine_status="quarantined",
|
||||
source_registry=None,
|
||||
source_snapshot=None,
|
||||
)
|
||||
|
||||
decision = MODULE.evaluate_dataset_training_eligibility(fixture, role="raster", fixture_mode=True)
|
||||
|
||||
assert decision.eligible is False
|
||||
assert set(decision.reasons) >= {"validation_failed", "dataset_quarantined"}
|
||||
|
||||
|
||||
def test_manifest_gate_rejects_missing_or_tampered_pair_decisions() -> None:
|
||||
raster = governed_dataset(
|
||||
role="raster",
|
||||
registry=source_registry(ground_truth_allowed=False),
|
||||
)
|
||||
reference = governed_dataset(role="reference")
|
||||
pair = MODULE.training_pair_evidence(raster=raster, reference=reference)
|
||||
manifest = {
|
||||
"samples": [{"sample_slug": "governed", "training_eligibility": pair}],
|
||||
"training_eligibility": {
|
||||
"policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||
"status": "eligible",
|
||||
"fixture_mode": False,
|
||||
},
|
||||
}
|
||||
|
||||
assert MODULE.manifest_training_eligibility_failures(manifest) == []
|
||||
tampered = {
|
||||
**manifest,
|
||||
"samples": [{"sample_slug": "governed", "training_eligibility": {**pair, "eligible": False}}],
|
||||
}
|
||||
failures = MODULE.manifest_training_eligibility_failures(tampered)
|
||||
assert "governed:training_pair_not_eligible" in failures
|
||||
assert MODULE.manifest_training_eligibility_failures({"samples": []}) == [
|
||||
"manifest_training_eligibility_missing"
|
||||
]
|
||||
|
||||
|
||||
def test_frozen_manifest_gate_detects_checksum_tampering(tmp_path: Path) -> None:
|
||||
raster = governed_dataset(
|
||||
role="raster",
|
||||
registry=source_registry(ground_truth_allowed=False),
|
||||
)
|
||||
reference = governed_dataset(role="reference")
|
||||
pair = MODULE.training_pair_evidence(raster=raster, reference=reference)
|
||||
manifest_path = tmp_path / "operator_samples_manifest.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"immutable": True,
|
||||
"training_eligibility": {
|
||||
"policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||
"status": "eligible",
|
||||
"fixture_mode": False,
|
||||
},
|
||||
"samples": [{"sample_slug": "governed", "training_eligibility": pair}],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(tmp_path / "corpus-freeze.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": 2,
|
||||
"manifest_sha256": hashlib.sha256(manifest_path.read_bytes()).hexdigest(),
|
||||
"immutable": True,
|
||||
"training_eligibility_policy": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||
"fixture_mode": False,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
assert MODULE.frozen_manifest_training_eligibility_failures(manifest_path) == []
|
||||
manifest_path.write_text(manifest_path.read_text(encoding="utf-8") + "\n", encoding="utf-8")
|
||||
assert "corpus_manifest_checksum_mismatch" in MODULE.frozen_manifest_training_eligibility_failures(
|
||||
manifest_path
|
||||
)
|
||||
|
||||
|
||||
def test_live_manifest_gate_revokes_a_frozen_pair_when_an_upstream_dataset_is_quarantined() -> None:
|
||||
raster_id = uuid4()
|
||||
reference_id = uuid4()
|
||||
raster_registry = source_registry(ground_truth_allowed=False)
|
||||
reference_registry = source_registry()
|
||||
raster_snapshot = source_snapshot()
|
||||
reference_snapshot = source_snapshot()
|
||||
raster_snapshot.source_registry_id = raster_registry.id
|
||||
reference_snapshot.source_registry_id = reference_registry.id
|
||||
raster = governed_dataset(
|
||||
role="raster",
|
||||
id=raster_id,
|
||||
source_registry=raster_registry,
|
||||
source_snapshot=raster_snapshot,
|
||||
source_registry_id=raster_registry.id,
|
||||
source_snapshot_id=raster_snapshot.id,
|
||||
)
|
||||
reference = governed_dataset(
|
||||
role="reference",
|
||||
id=reference_id,
|
||||
source_registry=reference_registry,
|
||||
source_snapshot=reference_snapshot,
|
||||
source_registry_id=reference_registry.id,
|
||||
source_snapshot_id=reference_snapshot.id,
|
||||
)
|
||||
pair = MODULE.training_pair_evidence(raster=raster, reference=reference)
|
||||
manifest = {
|
||||
"training_eligibility": {
|
||||
"policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||
"status": "eligible",
|
||||
"fixture_mode": False,
|
||||
},
|
||||
"samples": [
|
||||
{
|
||||
"sample_slug": "governed-aoi",
|
||||
"raster_dataset_id": str(raster_id),
|
||||
"reference_dataset_id": str(reference_id),
|
||||
"training_eligibility": pair,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
class DatasetModel:
|
||||
pass
|
||||
|
||||
class Session:
|
||||
def __init__(self) -> None:
|
||||
self.closed = False
|
||||
|
||||
@staticmethod
|
||||
def get(model, item_id):
|
||||
assert model is DatasetModel
|
||||
return {raster_id: raster, reference_id: reference}.get(item_id)
|
||||
|
||||
def close(self) -> None:
|
||||
self.closed = True
|
||||
|
||||
assert MODULE.live_manifest_training_eligibility_failures(
|
||||
manifest,
|
||||
session_factory=Session,
|
||||
dataset_model=DatasetModel,
|
||||
) == []
|
||||
|
||||
raster.quarantine_status = "quarantined"
|
||||
failures = MODULE.live_manifest_training_eligibility_failures(
|
||||
manifest,
|
||||
session_factory=Session,
|
||||
dataset_model=DatasetModel,
|
||||
)
|
||||
|
||||
assert "governed-aoi:raster_live_revoked:dataset_quarantined" in failures
|
||||
@@ -0,0 +1,414 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
SCRIPTS = ROOT / "scripts"
|
||||
if str(SCRIPTS) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPTS))
|
||||
SCRIPT = SCRIPTS / "training_release_manifest.py"
|
||||
SPEC = importlib.util.spec_from_file_location("training_release_manifest", SCRIPT)
|
||||
assert SPEC and SPEC.loader
|
||||
MODULE = importlib.util.module_from_spec(SPEC)
|
||||
sys.modules[SPEC.name] = MODULE
|
||||
SPEC.loader.exec_module(MODULE)
|
||||
|
||||
|
||||
def sha256(path: Path) -> str:
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _synthetic_release_uses_a_static_live_registry_spy(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Filesystem unit fixtures cannot resolve a real database, but must ask for it."""
|
||||
|
||||
original_assert = MODULE.assert_frozen_manifest_training_eligible
|
||||
original_failures = MODULE.frozen_manifest_training_eligibility_failures
|
||||
|
||||
def static_assert(*args, **kwargs):
|
||||
assert kwargs.get("verify_live") is True
|
||||
kwargs["verify_live"] = False
|
||||
return original_assert(*args, **kwargs)
|
||||
|
||||
def static_failures(*args, **kwargs):
|
||||
assert kwargs.get("verify_live") is True
|
||||
kwargs["verify_live"] = False
|
||||
return original_failures(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(MODULE, "assert_frozen_manifest_training_eligible", static_assert)
|
||||
monkeypatch.setattr(MODULE, "frozen_manifest_training_eligibility_failures", static_failures)
|
||||
|
||||
|
||||
def write_corpus_manifest(tmp_path: Path, *, fixture_mode: bool = False) -> Path:
|
||||
policy = "geointel-training-source-eligibility/v1"
|
||||
def pair(sample_slug: str) -> dict:
|
||||
raster_id = f"dataset:raster:{sample_slug}"
|
||||
reference_id = f"dataset:reference:{sample_slug}"
|
||||
return {
|
||||
"policy_version": policy,
|
||||
"eligible": True,
|
||||
"fixture_mode": fixture_mode,
|
||||
"raster": {
|
||||
"eligible": True,
|
||||
"reasons": [],
|
||||
"evidence": {
|
||||
"dataset_id": raster_id,
|
||||
"checksum_sha256": "a" * 64,
|
||||
"source_registry_id": "registry:orthophoto",
|
||||
"source_snapshot_id": "snapshot:orthophoto",
|
||||
},
|
||||
},
|
||||
"reference": {
|
||||
"eligible": True,
|
||||
"reasons": [],
|
||||
"evidence": {
|
||||
"dataset_id": reference_id,
|
||||
"checksum_sha256": "b" * 64,
|
||||
"source_registry_id": "registry:grb",
|
||||
"source_snapshot_id": "snapshot:grb",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
samples = []
|
||||
for sample_slug, split in (("fixture-train", "train"), ("fixture-val", "val")):
|
||||
samples.append(
|
||||
{
|
||||
"sample_slug": sample_slug,
|
||||
"split": split,
|
||||
"raster_dataset_id": f"dataset:raster:{sample_slug}",
|
||||
"reference_dataset_id": f"dataset:reference:{sample_slug}",
|
||||
"training_eligibility": pair(sample_slug),
|
||||
}
|
||||
)
|
||||
manifest = tmp_path / "operator_samples_manifest.json"
|
||||
manifest.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"immutable": True,
|
||||
"training_eligibility": {
|
||||
"policy_version": policy,
|
||||
"status": "eligible",
|
||||
"fixture_mode": fixture_mode,
|
||||
},
|
||||
"samples": samples,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(tmp_path / "corpus-freeze.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": 2,
|
||||
"immutable": True,
|
||||
"fixture_mode": fixture_mode,
|
||||
"training_eligibility_policy": policy,
|
||||
"manifest_sha256": sha256(manifest),
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return manifest
|
||||
|
||||
|
||||
def write_yolo_dataset(tmp_path: Path, *, empty_train_label: bool = False) -> Path:
|
||||
dataset_root = tmp_path / "dataset"
|
||||
for split, sample_slug in (("train", "fixture-train"), ("val", "fixture-val")):
|
||||
image = dataset_root / "images" / split / f"{sample_slug}.png"
|
||||
label = dataset_root / "labels" / split / f"{sample_slug}.txt"
|
||||
image.parent.mkdir(parents=True, exist_ok=True)
|
||||
label.parent.mkdir(parents=True, exist_ok=True)
|
||||
image.write_bytes(f"{split}-image".encode("utf-8"))
|
||||
label.write_text("" if split == "train" and empty_train_label else "0 0.5 0.5 0.2 0.2\n", encoding="utf-8")
|
||||
yaml_path = dataset_root / "dataset.yaml"
|
||||
yaml_path.write_text(
|
||||
f"path: {dataset_root}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
return yaml_path
|
||||
|
||||
|
||||
def write_accepted_review_audit(tmp_path: Path, corpus_manifest: Path) -> Path:
|
||||
artifacts = {}
|
||||
for sample_slug in ("fixture-train", "fixture-val"):
|
||||
artifact = tmp_path / f"{sample_slug}-contact-sheet.png"
|
||||
artifact.write_bytes(f"reviewed {sample_slug}".encode("utf-8"))
|
||||
artifacts[sample_slug] = artifact
|
||||
decisions = tmp_path / "review-decisions.json"
|
||||
decisions.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"decisions": [
|
||||
{
|
||||
"sample_slug": sample_slug,
|
||||
"decision": "accepted",
|
||||
"reviewer": "reviewer@example.test",
|
||||
"reviewed_at": "2026-08-01T12:00:00+00:00",
|
||||
"reviewed_artifact_path": str(artifact.resolve()),
|
||||
"reviewed_artifact_sha256": sha256(artifact),
|
||||
}
|
||||
for sample_slug, artifact in artifacts.items()
|
||||
]
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
evidence = {
|
||||
"review_decisions_path": str(decisions.resolve()),
|
||||
"review_decisions_sha256": sha256(decisions),
|
||||
"required_sample_count": 2,
|
||||
"accepted_sample_count": 2,
|
||||
"accepted_sample_slugs": ["fixture-train", "fixture-val"],
|
||||
"reviewer_ids": ["reviewer@example.test"],
|
||||
"reviewed_at_by_sample": {
|
||||
"fixture-train": "2026-08-01T12:00:00+00:00",
|
||||
"fixture-val": "2026-08-01T12:00:00+00:00",
|
||||
},
|
||||
"reviewed_artifact_path_by_sample": {
|
||||
sample_slug: str(artifact.resolve()) for sample_slug, artifact in artifacts.items()
|
||||
},
|
||||
"reviewed_artifact_sha256_by_sample": {
|
||||
sample_slug: sha256(artifact) for sample_slug, artifact in artifacts.items()
|
||||
},
|
||||
}
|
||||
audit = tmp_path / "belgium-building-corpus-audit.json"
|
||||
audit.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "ok",
|
||||
"manifest_immutable": True,
|
||||
"spatial_leakage_status": "ok",
|
||||
"review_complete": True,
|
||||
"corpus_manifest_path": str(corpus_manifest.resolve()),
|
||||
"corpus_manifest_sha256": sha256(corpus_manifest),
|
||||
"human_review_evidence": evidence,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return audit
|
||||
|
||||
|
||||
def test_valid_operational_release_binds_yaml_assets_corpus_and_human_review(tmp_path: Path) -> None:
|
||||
corpus_manifest = write_corpus_manifest(tmp_path)
|
||||
yaml_path = write_yolo_dataset(tmp_path)
|
||||
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
|
||||
|
||||
paths = MODULE.create_training_release_manifest(
|
||||
train_yaml=yaml_path,
|
||||
corpus_manifest=corpus_manifest,
|
||||
review_audit_path=review_audit,
|
||||
)
|
||||
|
||||
assert all(path.is_file() for path in paths.values())
|
||||
assert MODULE.training_release_failures(
|
||||
train_yaml=yaml_path,
|
||||
corpus_manifest=corpus_manifest,
|
||||
) == []
|
||||
|
||||
|
||||
def test_tile_summary_must_be_an_exact_view_of_the_live_verified_release(tmp_path: Path) -> None:
|
||||
corpus_manifest = write_corpus_manifest(tmp_path)
|
||||
yaml_path = write_yolo_dataset(tmp_path)
|
||||
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
|
||||
paths = MODULE.create_training_release_manifest(
|
||||
train_yaml=yaml_path,
|
||||
corpus_manifest=corpus_manifest,
|
||||
review_audit_path=review_audit,
|
||||
)
|
||||
assets = json.loads(paths["asset_manifest"].read_text(encoding="utf-8"))
|
||||
summary = yaml_path.parent / "yolo_tile_dataset_summary.json"
|
||||
summary.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"dataset_yaml": str(yaml_path.resolve()),
|
||||
"training_release_manifest": str(paths["release_manifest"].resolve()),
|
||||
"training_release_manifest_sha256": sha256(paths["release_manifest"]),
|
||||
"training_asset_manifest": str(paths["asset_manifest"].resolve()),
|
||||
"source_manifest_sha256": sha256(corpus_manifest),
|
||||
"tiles": [
|
||||
{
|
||||
"split": entry["split"],
|
||||
"image_path": entry["image_path"],
|
||||
"label_path": entry["label_path"],
|
||||
}
|
||||
for entry in assets["entries"]
|
||||
],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
MODULE.assert_yolo_summary_bound_to_training_release(
|
||||
summary_path=summary,
|
||||
train_yaml=yaml_path,
|
||||
corpus_manifest=corpus_manifest,
|
||||
)
|
||||
payload = json.loads(summary.read_text(encoding="utf-8"))
|
||||
payload["tiles"] = payload["tiles"][:1]
|
||||
summary.write_text(json.dumps(payload), encoding="utf-8")
|
||||
|
||||
with pytest.raises(MODULE.TrainingReleaseError, match="complete immutable view"):
|
||||
MODULE.assert_yolo_summary_bound_to_training_release(
|
||||
summary_path=summary,
|
||||
train_yaml=yaml_path,
|
||||
corpus_manifest=corpus_manifest,
|
||||
)
|
||||
|
||||
|
||||
def test_unbound_or_changed_yaml_is_rejected_before_training(tmp_path: Path) -> None:
|
||||
corpus_manifest = write_corpus_manifest(tmp_path)
|
||||
yaml_path = write_yolo_dataset(tmp_path)
|
||||
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
|
||||
|
||||
assert MODULE.training_release_failures(train_yaml=yaml_path) == ["training_release_manifest_missing"]
|
||||
MODULE.create_training_release_manifest(
|
||||
train_yaml=yaml_path,
|
||||
corpus_manifest=corpus_manifest,
|
||||
review_audit_path=review_audit,
|
||||
)
|
||||
yaml_path.write_text(yaml_path.read_text(encoding="utf-8") + "# tampered\n", encoding="utf-8")
|
||||
|
||||
assert "training_release_yaml_checksum_mismatch" in MODULE.training_release_failures(
|
||||
train_yaml=yaml_path,
|
||||
corpus_manifest=corpus_manifest,
|
||||
)
|
||||
|
||||
|
||||
def test_changed_label_asset_is_rejected_even_when_yaml_bytes_are_unchanged(tmp_path: Path) -> None:
|
||||
corpus_manifest = write_corpus_manifest(tmp_path)
|
||||
yaml_path = write_yolo_dataset(tmp_path)
|
||||
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
|
||||
MODULE.create_training_release_manifest(
|
||||
train_yaml=yaml_path,
|
||||
corpus_manifest=corpus_manifest,
|
||||
review_audit_path=review_audit,
|
||||
)
|
||||
label = yaml_path.parent / "labels" / "train" / "fixture-train.txt"
|
||||
label.write_text("0 0.4 0.4 0.2 0.2\n", encoding="utf-8")
|
||||
|
||||
failures = MODULE.training_release_failures(
|
||||
train_yaml=yaml_path,
|
||||
corpus_manifest=corpus_manifest,
|
||||
)
|
||||
assert "training_release_asset_manifest_content_mismatch" in failures
|
||||
|
||||
|
||||
def test_operational_release_requires_complete_accepted_human_review(tmp_path: Path) -> None:
|
||||
corpus_manifest = write_corpus_manifest(tmp_path)
|
||||
yaml_path = write_yolo_dataset(tmp_path)
|
||||
incomplete_audit = tmp_path / "audit.json"
|
||||
incomplete_audit.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"status": "needs_human_review",
|
||||
"manifest_immutable": True,
|
||||
"spatial_leakage_status": "ok",
|
||||
"review_complete": False,
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
try:
|
||||
MODULE.create_training_release_manifest(
|
||||
train_yaml=yaml_path,
|
||||
corpus_manifest=corpus_manifest,
|
||||
review_audit_path=incomplete_audit,
|
||||
)
|
||||
except MODULE.TrainingReleaseError as exc:
|
||||
assert "review_complete_not_true" in str(exc)
|
||||
assert "accepted_human_review_evidence_missing" in str(exc)
|
||||
else:
|
||||
raise AssertionError("operational release accepted an incomplete human review")
|
||||
|
||||
|
||||
def test_operational_release_rejects_tampered_accepted_review_artifact(tmp_path: Path) -> None:
|
||||
corpus_manifest = write_corpus_manifest(tmp_path)
|
||||
yaml_path = write_yolo_dataset(tmp_path)
|
||||
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
|
||||
artifact = tmp_path / "fixture-train-contact-sheet.png"
|
||||
artifact.write_bytes(b"changed after review")
|
||||
|
||||
try:
|
||||
MODULE.create_training_release_manifest(
|
||||
train_yaml=yaml_path,
|
||||
corpus_manifest=corpus_manifest,
|
||||
review_audit_path=review_audit,
|
||||
)
|
||||
except MODULE.TrainingReleaseError as exc:
|
||||
assert "reviewed_artifact_checksum_mismatch" in str(exc)
|
||||
else:
|
||||
raise AssertionError("tampered human-review artifact was accepted")
|
||||
|
||||
|
||||
def test_fixture_relaxation_requires_explicit_fixture_corpus_and_is_not_operational(tmp_path: Path) -> None:
|
||||
fixture_manifest = write_corpus_manifest(tmp_path, fixture_mode=True)
|
||||
yaml_path = write_yolo_dataset(tmp_path)
|
||||
MODULE.create_training_release_manifest(
|
||||
train_yaml=yaml_path,
|
||||
corpus_manifest=fixture_manifest,
|
||||
fixture_mode=True,
|
||||
)
|
||||
|
||||
assert MODULE.training_release_failures(
|
||||
train_yaml=yaml_path,
|
||||
corpus_manifest=fixture_manifest,
|
||||
fixture_mode=True,
|
||||
) == []
|
||||
assert "training_release_fixture_mode_mismatch" in MODULE.training_release_failures(
|
||||
train_yaml=yaml_path,
|
||||
corpus_manifest=fixture_manifest,
|
||||
fixture_mode=False,
|
||||
)
|
||||
|
||||
|
||||
def test_release_contracts_a_reviewed_empty_label_as_explicit_pure_background(tmp_path: Path) -> None:
|
||||
corpus_manifest = write_corpus_manifest(tmp_path)
|
||||
yaml_path = write_yolo_dataset(tmp_path, empty_train_label=True)
|
||||
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
|
||||
|
||||
paths = MODULE.create_training_release_manifest(
|
||||
train_yaml=yaml_path,
|
||||
corpus_manifest=corpus_manifest,
|
||||
review_audit_path=review_audit,
|
||||
)
|
||||
|
||||
labels = json.loads(paths["label_contract_manifest"].read_text(encoding="utf-8"))
|
||||
assert labels["counts"]["pure_background"] == 1
|
||||
assert MODULE.training_release_failures(train_yaml=yaml_path, corpus_manifest=corpus_manifest) == []
|
||||
|
||||
|
||||
def test_empty_label_without_accepted_sample_review_cannot_become_a_background_negative(tmp_path: Path) -> None:
|
||||
corpus_manifest = write_corpus_manifest(tmp_path)
|
||||
yaml_path = write_yolo_dataset(tmp_path, empty_train_label=True)
|
||||
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
|
||||
audit_payload = json.loads(review_audit.read_text(encoding="utf-8"))
|
||||
evidence = audit_payload["human_review_evidence"]
|
||||
evidence["accepted_sample_slugs"] = ["fixture-val"]
|
||||
review_audit.write_text(json.dumps(audit_payload), encoding="utf-8")
|
||||
review = {
|
||||
"status": "accepted",
|
||||
"fixture_only": False,
|
||||
"review_complete": True,
|
||||
"evidence": evidence,
|
||||
}
|
||||
|
||||
try:
|
||||
MODULE.build_training_label_contract_manifest(
|
||||
corpus_manifest_path=corpus_manifest,
|
||||
asset_manifest=MODULE.build_yolo_asset_manifest(yaml_path),
|
||||
review=review,
|
||||
fixture_mode=False,
|
||||
)
|
||||
except MODULE.TrainingReleaseError as exc:
|
||||
assert "Pure-background label sample was not accepted by review" in str(exc)
|
||||
else:
|
||||
raise AssertionError("unreviewed empty label was accepted as a background negative")
|
||||
@@ -1,7 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from uuid import UUID, uuid4
|
||||
|
||||
from app.core.errors import AppError
|
||||
|
||||
Reference in New Issue
Block a user