feat(provenance): govern source snapshots and data inputs

This commit is contained in:
Jens
2026-08-01 23:46:17 +02:00
parent cebeb5f3b4
commit 5b3c17b494
96 changed files with 20156 additions and 351 deletions
+2
View File
@@ -25,6 +25,8 @@ build/
/artifacts/evidence/accuracy/*
!/artifacts/evidence/accuracy/P1/
!/artifacts/evidence/accuracy/P1/**
!/artifacts/evidence/accuracy/P2/
!/artifacts/evidence/accuracy/P2/**
/.cache/
/datasets/raw/*
/datasets/processed/*
File diff suppressed because it is too large Load Diff
+15 -1
View File
@@ -1 +1,15 @@
__all__ = ["analysis", "areas", "assistant", "auth", "datasets", "health", "projects", "exports", "jobs", "external", "qa", "temporal"]
__all__ = [
"analysis",
"areas",
"assistant",
"auth",
"datasets",
"exports",
"external",
"health",
"jobs",
"projects",
"qa",
"source_registry",
"temporal",
]
+119
View File
@@ -0,0 +1,119 @@
from __future__ import annotations
from uuid import UUID
from fastapi import APIRouter, Depends
from sqlalchemy import func, or_
from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.db.session import get_db
from app.models import Dataset, DatasetLineageEdge, DatasetQuarantine, Project, SourceRegistry, SourceSnapshot
from app.schemas import (
DatasetLineageEdgeRead,
DatasetProvenanceRead,
DatasetQuarantineRead,
Envelope,
ItemList,
SourceRegistryDetailRead,
SourceRegistryRead,
SourceSnapshotRead,
)
from app.utils.response import envelope
router = APIRouter(tags=["source-registry"])
def _source_read(source: SourceRegistry, *, snapshot_count: int = 0) -> SourceRegistryRead:
return SourceRegistryRead.model_validate(source).model_copy(update={"snapshot_count": int(snapshot_count)})
@router.get("/source-registry", response_model=Envelope[ItemList[SourceRegistryRead]])
def list_source_registry(
classification: str | None = None,
db: Session = Depends(get_db),
) -> dict:
query = (
db.query(SourceRegistry, func.count(SourceSnapshot.id).label("snapshot_count"))
.outerjoin(SourceSnapshot, SourceSnapshot.source_registry_id == SourceRegistry.id)
)
if classification:
query = query.filter(SourceRegistry.classification == classification.strip().lower())
rows = (
query.group_by(SourceRegistry.id)
.order_by(SourceRegistry.classification.asc(), SourceRegistry.display_name.asc())
.all()
)
items = [_source_read(source, snapshot_count=count) for source, count in rows]
return envelope({"items": items, "total": len(items)})
@router.get("/source-registry/{source_key}", response_model=Envelope[SourceRegistryDetailRead])
def get_source_registry_entry(source_key: str, db: Session = Depends(get_db)) -> dict:
normalized_key = source_key.strip().lower()
source = db.query(SourceRegistry).filter(SourceRegistry.source_key == normalized_key).one_or_none()
if source is None:
raise AppError(code="SOURCE_REGISTRY_ENTRY_NOT_FOUND", message="Source registry entry was not found", status_code=404)
snapshots = (
db.query(SourceSnapshot)
.filter(SourceSnapshot.source_registry_id == source.id)
.order_by(SourceSnapshot.fetched_at.desc(), SourceSnapshot.created_at.desc())
.all()
)
detail = SourceRegistryDetailRead(
source=_source_read(source, snapshot_count=len(snapshots)),
snapshots=[SourceSnapshotRead.model_validate(snapshot) for snapshot in snapshots],
)
return envelope(detail)
@router.get(
"/projects/{project_id}/datasets/{dataset_id}/provenance",
response_model=Envelope[DatasetProvenanceRead],
)
def get_dataset_provenance(
project_id: UUID,
dataset_id: UUID,
db: Session = Depends(get_db),
) -> dict:
if db.get(Project, project_id) is None:
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
dataset = db.get(Dataset, dataset_id)
if dataset is None or dataset.project_id != project_id:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
source = db.get(SourceRegistry, dataset.source_registry_id) if dataset.source_registry_id else None
snapshot = db.get(SourceSnapshot, dataset.source_snapshot_id) if dataset.source_snapshot_id else None
lineage = (
db.query(DatasetLineageEdge)
.filter(
or_(
DatasetLineageEdge.parent_dataset_id == dataset.id,
DatasetLineageEdge.child_dataset_id == dataset.id,
)
)
.order_by(DatasetLineageEdge.created_at.asc(), DatasetLineageEdge.id.asc())
.all()
)
quarantines = (
db.query(DatasetQuarantine)
.filter(DatasetQuarantine.dataset_id == dataset.id)
.order_by(DatasetQuarantine.created_at.desc(), DatasetQuarantine.id.desc())
.all()
)
result = DatasetProvenanceRead(
dataset_id=dataset.id,
source=_source_read(source) if source else None,
snapshot=SourceSnapshotRead.model_validate(snapshot) if snapshot else None,
data_contract_key=dataset.data_contract_key,
data_contract_version=dataset.data_contract_version,
validation_status=dataset.validation_status,
validation_report_json=dataset.validation_report_json,
provenance_status=dataset.provenance_status,
lineage_status=dataset.lineage_status,
quarantine_status=dataset.quarantine_status,
lineage=[DatasetLineageEdgeRead.model_validate(item) for item in lineage],
quarantines=[DatasetQuarantineRead.model_validate(item) for item in quarantines],
)
return envelope(result)
+2 -1
View File
@@ -12,7 +12,7 @@ from fastapi.exceptions import RequestValidationError
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from app.api.routes import analysis, aoi_operations, areas, assistant, auth, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation, selection_partitions, temporal
from app.api.routes import analysis, aoi_operations, areas, assistant, auth, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation, selection_partitions, source_registry, temporal
from app.core.config import get_settings
from app.core.errors import AppError
from app.core.logging import configure_logging
@@ -103,6 +103,7 @@ def create_app() -> FastAPI:
app.include_router(quality_checks.router, prefix=settings.api_prefix)
app.include_router(exports.router, prefix=settings.api_prefix)
app.include_router(external.router, prefix=settings.api_prefix)
app.include_router(source_registry.router, prefix=settings.api_prefix)
app.include_router(demo.router, prefix=settings.api_prefix)
app.include_router(qa.router, prefix=settings.api_prefix)
app.include_router(detection.router, prefix=settings.api_prefix)
+25 -1
View File
@@ -1,4 +1,24 @@
from .entities import AoiOperation, AoiOperationPartition, AnalysisRun, Area, Dataset, DatasetVersion, Detection, DetectionReview, Export, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature
from .entities import (
AoiOperation,
AoiOperationPartition,
AnalysisRun,
Area,
Dataset,
DatasetLineageEdge,
DatasetQuarantine,
DatasetVersion,
Detection,
DetectionReview,
Export,
Job,
Metric,
Project,
QualityCheck,
Segmentation,
SourceRegistry,
SourceSnapshot,
VectorFeature,
)
__all__ = [
"AnalysisRun",
@@ -6,6 +26,8 @@ __all__ = [
"AoiOperationPartition",
"Area",
"Dataset",
"DatasetLineageEdge",
"DatasetQuarantine",
"DatasetVersion",
"Detection",
"DetectionReview",
@@ -15,5 +37,7 @@ __all__ = [
"Project",
"QualityCheck",
"Segmentation",
"SourceRegistry",
"SourceSnapshot",
"VectorFeature",
]
+365
View File
@@ -12,6 +12,37 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship
from app.db.base import Base
SOURCE_CLASSIFICATIONS = (
"authoritative",
"corroborative",
"contextual",
"derived",
"experimental",
)
SOURCE_FRESHNESS_STATUSES = (
"unknown",
"current",
"due",
"stale",
"not_applicable",
"review_required",
)
SOURCE_INGEST_STATUSES = (
"registered",
"configured",
"not_configured",
"available",
"ingested",
"failed",
"quarantined",
"legacy_unverified",
)
PROVENANCE_STATUSES = ("complete", "incomplete", "not_applicable")
LINEAGE_STATUSES = ("complete", "incomplete", "not_applicable")
VALIDATION_STATUSES = ("not_validated", "passed", "failed")
QUARANTINE_STATUSES = ("not_quarantined", "quarantined")
class Project(Base):
__tablename__ = "projects"
@@ -42,6 +73,123 @@ class Area(Base):
project: Mapped[Project] = relationship("Project", back_populates="areas")
class SourceRegistry(Base):
"""Server-owned source identity and authority contract.
Dataset metadata remains descriptive until a governed importer binds a
dataset to both this registry entry and an immutable SourceSnapshot.
"""
__tablename__ = "source_registry"
__table_args__ = (
UniqueConstraint("source_key", name="uq_source_registry_source_key"),
CheckConstraint(
"classification IN ('authoritative', 'corroborative', 'contextual', 'derived', 'experimental')",
name="ck_source_registry_classification",
),
CheckConstraint(
"freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')",
name="ck_source_registry_freshness_status",
),
CheckConstraint(
"ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', "
"'failed', 'quarantined', 'legacy_unverified')",
name="ck_source_registry_ingest_status",
),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
source_key: Mapped[str] = mapped_column(String(120), nullable=False)
display_name: Mapped[str] = mapped_column(String(255), nullable=False)
classification: Mapped[str] = mapped_column(String(32), nullable=False)
authority_name: Mapped[str] = mapped_column(String(255), nullable=False, default="unknown", server_default="unknown")
authority_scope_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
provider_adapter_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
license_name: Mapped[str] = mapped_column(String(255), nullable=False, default="unknown", server_default="unknown")
license_url: Mapped[str | None] = mapped_column(Text, nullable=True)
usage_restrictions: Mapped[str] = mapped_column(Text, nullable=False, default="unknown", server_default="unknown")
default_crs: Mapped[str] = mapped_column(String(64), nullable=False, default="unknown", server_default="unknown")
default_units: Mapped[str] = mapped_column(String(120), nullable=False, default="unknown", server_default="unknown")
spatial_resolution_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
temporal_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
geographic_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
expected_geometry_types_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
expected_attributes_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
usage_policy_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
freshness_status: Mapped[str] = mapped_column(
String(32), nullable=False, default="unknown", server_default="unknown"
)
ingest_status: Mapped[str] = mapped_column(
String(32), nullable=False, default="registered", server_default="registered"
)
known_limitations_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
registry_metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
snapshots: Mapped[list["SourceSnapshot"]] = relationship(
"SourceSnapshot", back_populates="source_registry", cascade="all, delete-orphan"
)
datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="source_registry")
dataset_versions: Mapped[list["DatasetVersion"]] = relationship("DatasetVersion", back_populates="source_registry")
class SourceSnapshot(Base):
"""Immutable source-version evidence recorded by governed ingestion."""
__tablename__ = "source_snapshots"
__table_args__ = (
UniqueConstraint("source_registry_id", "snapshot_key", name="uq_source_snapshots_registry_key"),
CheckConstraint(
"freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')",
name="ck_source_snapshots_freshness_status",
),
CheckConstraint(
"ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', "
"'failed', 'quarantined', 'legacy_unverified')",
name="ck_source_snapshots_ingest_status",
),
CheckConstraint(
"checksum_sha256 = lower(checksum_sha256) AND checksum_sha256 ~ '^[0-9a-f]{64}$'",
name="ck_source_snapshots_checksum_sha256",
),
Index("ix_source_snapshots_registry_fetched", "source_registry_id", "fetched_at"),
Index("ix_source_snapshots_checksum", "checksum_sha256"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
source_registry_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="CASCADE"), nullable=False
)
snapshot_key: Mapped[str] = mapped_column(String(255), nullable=False)
source_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
snapshot_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
crs: Mapped[str | None] = mapped_column(String(64), nullable=True)
units: Mapped[str | None] = mapped_column(String(120), nullable=True)
spatial_resolution_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
temporal_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
geographic_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
observed_schema_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
freshness_status: Mapped[str] = mapped_column(
String(32), nullable=False, default="unknown", server_default="unknown"
)
ingest_status: Mapped[str] = mapped_column(
String(32), nullable=False, default="registered", server_default="registered"
)
known_limitations_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
snapshot_metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
source_registry: Mapped[SourceRegistry] = relationship("SourceRegistry", back_populates="snapshots")
datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="source_snapshot")
dataset_versions: Mapped[list["DatasetVersion"]] = relationship("DatasetVersion", back_populates="source_snapshot")
quarantines: Mapped[list["DatasetQuarantine"]] = relationship("DatasetQuarantine", back_populates="source_snapshot")
class Dataset(Base):
__tablename__ = "datasets"
__table_args__ = (
@@ -49,6 +197,27 @@ class Dataset(Base):
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
name="ck_datasets_temporal_valid_range",
),
CheckConstraint(
"validation_status IN ('not_validated', 'passed', 'failed')",
name="ck_datasets_validation_status",
),
CheckConstraint(
"provenance_status IN ('complete', 'incomplete', 'not_applicable')",
name="ck_datasets_provenance_status",
),
CheckConstraint(
"lineage_status IN ('complete', 'incomplete', 'not_applicable')",
name="ck_datasets_lineage_status",
),
CheckConstraint(
"quarantine_status IN ('not_quarantined', 'quarantined')",
name="ck_datasets_quarantine_status",
),
CheckConstraint(
"ingest_key IS NULL OR btrim(ingest_key) <> ''",
name="ck_datasets_ingest_key_not_blank",
),
UniqueConstraint("project_id", "ingest_key", name="uq_datasets_project_ingest_key"),
Index(
"ix_datasets_project_temporal_series_observed",
"project_id",
@@ -69,6 +238,7 @@ class Dataset(Base):
content_type: Mapped[str | None] = mapped_column(String(120), nullable=True)
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
ingest_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
derived_from_dataset_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True),
ForeignKey("datasets.id", ondelete="SET NULL"),
@@ -84,6 +254,43 @@ class Dataset(Base):
reference_layer_name: Mapped[str | None] = mapped_column(String(120), nullable=True)
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
source_registry_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="SET NULL"), nullable=True
)
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
)
data_contract_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
data_contract_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
validation_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="not_validated",
server_default="not_validated",
comment="not_validated | passed | failed",
)
validation_report_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="incomplete",
server_default="incomplete",
comment="complete | incomplete | not_applicable",
)
lineage_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="incomplete",
server_default="incomplete",
comment="complete | incomplete | not_applicable",
)
quarantine_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="not_quarantined",
server_default="not_quarantined",
comment="not_quarantined | quarantined",
)
imported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
temporal_series_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
@@ -106,6 +313,23 @@ class Dataset(Base):
back_populates="dataset",
cascade="all, delete-orphan",
)
source_registry: Mapped[SourceRegistry | None] = relationship("SourceRegistry", back_populates="datasets")
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="datasets")
parent_lineage_edges: Mapped[list["DatasetLineageEdge"]] = relationship(
"DatasetLineageEdge",
foreign_keys="DatasetLineageEdge.parent_dataset_id",
back_populates="parent_dataset",
cascade="all, delete-orphan",
)
child_lineage_edges: Mapped[list["DatasetLineageEdge"]] = relationship(
"DatasetLineageEdge",
foreign_keys="DatasetLineageEdge.child_dataset_id",
back_populates="child_dataset",
cascade="all, delete-orphan",
)
quarantines: Mapped[list["DatasetQuarantine"]] = relationship(
"DatasetQuarantine", back_populates="dataset", cascade="all, delete-orphan"
)
class DatasetVersion(Base):
@@ -115,7 +339,24 @@ class DatasetVersion(Base):
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
name="ck_dataset_versions_temporal_valid_range",
),
CheckConstraint(
"validation_status IN ('not_validated', 'passed', 'failed')",
name="ck_dataset_versions_validation_status",
),
CheckConstraint(
"provenance_status IN ('complete', 'incomplete', 'not_applicable')",
name="ck_dataset_versions_provenance_status",
),
CheckConstraint(
"lineage_status IN ('complete', 'incomplete', 'not_applicable')",
name="ck_dataset_versions_lineage_status",
),
CheckConstraint(
"ingest_key IS NULL OR btrim(ingest_key) <> ''",
name="ck_dataset_versions_ingest_key_not_blank",
),
Index("ix_dataset_versions_dataset_version", "dataset_id", "version", unique=True),
UniqueConstraint("dataset_id", "ingest_key", name="uq_dataset_versions_dataset_ingest_key"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
@@ -127,11 +368,135 @@ class DatasetVersion(Base):
valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
ingest_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
source_registry_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="SET NULL"), nullable=True
)
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
)
data_contract_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
data_contract_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
validation_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="not_validated",
server_default="not_validated",
comment="not_validated | passed | failed",
)
validation_report_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
provenance_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="incomplete",
server_default="incomplete",
comment="complete | incomplete | not_applicable",
)
lineage_status: Mapped[str] = mapped_column(
String(32),
nullable=False,
default="incomplete",
server_default="incomplete",
comment="complete | incomplete | not_applicable",
)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
dataset: Mapped[Dataset] = relationship("Dataset", back_populates="versions")
source_registry: Mapped[SourceRegistry | None] = relationship("SourceRegistry", back_populates="dataset_versions")
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="dataset_versions")
quarantines: Mapped[list["DatasetQuarantine"]] = relationship("DatasetQuarantine", back_populates="dataset_version")
class DatasetLineageEdge(Base):
"""Immutable relationship between input/output datasets and transforms."""
__tablename__ = "dataset_lineage_edges"
__table_args__ = (
CheckConstraint("parent_dataset_id <> child_dataset_id", name="ck_dataset_lineage_edges_distinct_datasets"),
UniqueConstraint(
"parent_dataset_id",
"child_dataset_id",
"relation_type",
"transformation_name",
name="uq_dataset_lineage_edges_relation",
),
Index("ix_dataset_lineage_edges_parent", "parent_dataset_id"),
Index("ix_dataset_lineage_edges_child", "child_dataset_id"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
parent_dataset_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False
)
child_dataset_id: Mapped[uuid.UUID] = mapped_column(
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False
)
parent_dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
)
child_dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
)
relation_type: Mapped[str] = mapped_column(String(64), nullable=False)
transformation_name: Mapped[str] = mapped_column(String(255), nullable=False)
transformation_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
parameters_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
input_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
output_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
parent_dataset: Mapped[Dataset] = relationship(
"Dataset", foreign_keys=[parent_dataset_id], back_populates="parent_lineage_edges"
)
child_dataset: Mapped[Dataset] = relationship(
"Dataset", foreign_keys=[child_dataset_id], back_populates="child_lineage_edges"
)
class DatasetQuarantine(Base):
"""Durable fail-closed record for rejected or doubtful source artifacts."""
__tablename__ = "dataset_quarantines"
__table_args__ = (
CheckConstraint(
"dataset_id IS NOT NULL OR dataset_version_id IS NOT NULL OR source_snapshot_id IS NOT NULL",
name="ck_dataset_quarantines_target_present",
),
CheckConstraint(
"status IN ('quarantined', 'released', 'rejected')",
name="ck_dataset_quarantines_status",
),
Index("ix_dataset_quarantines_dataset_status", "dataset_id", "status"),
Index("ix_dataset_quarantines_snapshot_status", "source_snapshot_id", "status"),
)
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
dataset_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True
)
dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
)
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
)
stage: Mapped[str] = mapped_column(String(64), nullable=False)
reason_code: Mapped[str] = mapped_column(String(120), nullable=False)
details_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
artifact_path: Mapped[str | None] = mapped_column(Text, nullable=True)
artifact_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
status: Mapped[str] = mapped_column(
String(32), nullable=False, default="quarantined", server_default="quarantined"
)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
resolved_by: Mapped[str | None] = mapped_column(String(120), nullable=True)
dataset: Mapped[Dataset | None] = relationship("Dataset", back_populates="quarantines")
dataset_version: Mapped[DatasetVersion | None] = relationship("DatasetVersion", back_populates="quarantines")
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="quarantines")
class VectorFeature(Base):
+18
View File
@@ -32,6 +32,14 @@ from .source_catalog import (
SourceCatalogProbeReport,
SourceCatalogProbeSummary,
)
from .source_registry import (
DatasetProvenanceRead,
DatasetLineageEdgeRead,
DatasetQuarantineRead,
SourceRegistryDetailRead,
SourceRegistryRead,
SourceSnapshotRead,
)
from .grb_refresh import GrbRefreshLayerPlan, GrbRefreshPlan, GrbRefreshPlanSummary
from .grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead
from .official_vector import (
@@ -207,6 +215,12 @@ __all__ = [
"SourceCatalogProbeItem",
"SourceCatalogProbeReport",
"SourceCatalogProbeSummary",
"SourceRegistryRead",
"SourceRegistryDetailRead",
"SourceSnapshotRead",
"DatasetLineageEdgeRead",
"DatasetQuarantineRead",
"DatasetProvenanceRead",
"GrbRefreshLayerPlan",
"GrbRefreshPlan",
"GrbRefreshPlanSummary",
@@ -271,6 +285,10 @@ __all__ = [
"FloodHazardSelectionSummary",
"BathymetryProfileAcquireRequest",
"BathymetryProfileAcquisitionResult",
"BathymetryRasterMetric",
"BathymetryRasterSelectionRequest",
"BathymetryRasterSelectionResponse",
"BathymetryRasterSelectionSummary",
"BathymetryPartitionFinalizeRequest",
"BathymetryPartitionFinalizationResult",
"BathymetrySourceProbeRead",
+19
View File
@@ -35,6 +35,16 @@ class DatasetCreateResponse(BaseModel):
reference_layer_name: str | None = None
source_metadata: dict | None = None
provenance_metadata: dict | None = None
ingest_key: str | None = None
source_registry_id: UUID | None = None
source_snapshot_id: UUID | None = None
data_contract_key: str | None = None
data_contract_version: str | None = None
validation_status: str | None = None
validation_report_json: dict | None = None
provenance_status: str | None = None
lineage_status: str | None = None
quarantine_status: str | None = None
imported_at: datetime | None = None
temporal_series_key: str | None = None
observed_at: datetime | None = None
@@ -97,6 +107,15 @@ class DatasetVersionRead(BaseModel):
checksum_sha256: str | None = None
source_metadata: dict | None = None
provenance_metadata: dict | None = None
ingest_key: str | None = None
source_registry_id: UUID | None = None
source_snapshot_id: UUID | None = None
data_contract_key: str | None = None
data_contract_version: str | None = None
validation_status: str | None = None
validation_report_json: dict | None = None
provenance_status: str | None = None
lineage_status: str | None = None
created_at: datetime | None = None
model_config = {"from_attributes": True}
+2
View File
@@ -144,6 +144,8 @@ class YoloPreflightChecks(BaseModel):
accelerator_ready: bool | None = None
model_path_set: bool | None = None
model_file_exists: bool | None = None
model_provenance_manifest_path: str | None = None
model_provenance_valid: bool | None = None
model_load_requested: bool
model_load_ok: bool | None = None
manifest_path_set: bool | None = None
+120
View File
@@ -0,0 +1,120 @@
from __future__ import annotations
from datetime import datetime
from uuid import UUID
from pydantic import BaseModel, Field
class SourceRegistryRead(BaseModel):
"""Read-only, server-owned source-authority definition."""
id: UUID
source_key: str
display_name: str
classification: str
authority_name: str
authority_scope_json: dict
provider_adapter_key: str | None = None
source_url: str | None = None
license_name: str
license_url: str | None = None
usage_restrictions: str
default_crs: str
default_units: str
spatial_resolution_json: dict
temporal_coverage_json: dict
geographic_coverage_json: dict
expected_geometry_types_json: list
expected_attributes_json: dict
usage_policy_json: dict
freshness_status: str
ingest_status: str
known_limitations_json: list
registry_metadata_json: dict
created_at: datetime | None = None
updated_at: datetime | None = None
snapshot_count: int = 0
model_config = {"from_attributes": True}
class SourceSnapshotRead(BaseModel):
"""Immutable version/snapshot evidence attached to an imported dataset."""
id: UUID
source_registry_id: UUID
snapshot_key: str
source_version: str | None = None
snapshot_at: datetime | None = None
fetched_at: datetime | None = None
source_url: str | None = None
checksum_sha256: str | None = None
crs: str | None = None
units: str | None = None
spatial_resolution_json: dict
temporal_coverage_json: dict
geographic_coverage_json: dict
observed_schema_json: dict
freshness_status: str
ingest_status: str
known_limitations_json: list
snapshot_metadata_json: dict
created_at: datetime | None = None
model_config = {"from_attributes": True}
class SourceRegistryDetailRead(BaseModel):
source: SourceRegistryRead
snapshots: list[SourceSnapshotRead]
class DatasetLineageEdgeRead(BaseModel):
id: UUID
parent_dataset_id: UUID
child_dataset_id: UUID
parent_dataset_version_id: UUID | None = None
child_dataset_version_id: UUID | None = None
relation_type: str
transformation_name: str
transformation_version: str | None = None
parameters_json: dict | None = None
input_checksum_sha256: str | None = None
output_checksum_sha256: str | None = None
created_at: datetime | None = None
model_config = {"from_attributes": True}
class DatasetQuarantineRead(BaseModel):
id: UUID
dataset_id: UUID | None = None
dataset_version_id: UUID | None = None
source_snapshot_id: UUID | None = None
stage: str
reason_code: str
details_json: dict | None = None
artifact_path: str | None = None
artifact_checksum_sha256: str | None = None
status: str
created_at: datetime | None = None
resolved_at: datetime | None = None
resolved_by: str | None = None
model_config = {"from_attributes": True}
class DatasetProvenanceRead(BaseModel):
dataset_id: UUID
source: SourceRegistryRead | None = None
snapshot: SourceSnapshotRead | None = None
data_contract_key: str | None = None
data_contract_version: str | None = None
validation_status: str | None = None
validation_report_json: dict | None = None
provenance_status: str | None = None
lineage_status: str | None = None
quarantine_status: str | None = None
lineage: list[DatasetLineageEdgeRead] = Field(default_factory=list)
quarantines: list[DatasetQuarantineRead] = Field(default_factory=list)
@@ -12,6 +12,7 @@ from sqlalchemy.orm import Session
from app.core.errors import AppError
from app.models import Area, Dataset, Project
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
from app.schemas.coverage import (
CoverageBBox,
CoverageCatalogResponse,
@@ -463,6 +464,10 @@ class CoverageRegistryService:
for dataset in datasets:
if dataset.status != "ready" or dataset.source_name not in definition.materialized_source_names:
continue
# A source-name claim alone must not cause an unsafe artifact to
# appear as operational authoritative coverage.
if not DatasetConsumptionGate.eligible_for_authoritative_coverage(dataset):
continue
layer_names = definition.materialized_layer_names
if definition.contract.source_name == "digitaal_vlaanderen":
theme_sources = FLANDERS_THEME_DATASETS.get(theme, {})
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,160 @@
"""Fail-closed quarantine decisions for validated data assets.
Persistence is intentionally delegated to the caller's transaction. This
module derives stable decisions from immutable validation reports and blocks a
quarantined or failed asset from training, production inference and derived
processing until an explicit, separately persisted release action exists.
"""
from __future__ import annotations
from dataclasses import dataclass
from enum import StrEnum
from hashlib import sha256
from typing import Any
import json
from app.core.errors import AppError
from app.services.data_contract_validation import QuarantineStatus, ValidationReport, ValidationStatus
class AssetUse(StrEnum):
TRAINING = "training"
PRODUCTION_INFERENCE = "production_inference"
DERIVED_PROCESSING = "derived_processing"
EXPORT = "export"
@dataclass(frozen=True)
class QuarantineDecision:
"""A deterministic, auditable quarantine decision for one report."""
asset_id: str
data_contract_key: str
data_contract_version: str
validation_report_sha256: str
validation_status: ValidationStatus
quarantine_status: QuarantineStatus
reason_codes: tuple[str, ...]
idempotency_key: str
requires_explicit_release: bool = False
@property
def eligible_for_use(self) -> bool:
return self.validation_status == ValidationStatus.PASSED and self.quarantine_status == QuarantineStatus.NOT_QUARANTINED
def to_dict(self) -> dict[str, Any]:
return {
"asset_id": self.asset_id,
"data_contract_key": self.data_contract_key,
"data_contract_version": self.data_contract_version,
"validation_report_sha256": self.validation_report_sha256,
"validation_status": self.validation_status.value,
"quarantine_status": self.quarantine_status.value,
"reason_codes": list(self.reason_codes),
"idempotency_key": self.idempotency_key,
"requires_explicit_release": self.requires_explicit_release,
"eligible_for_use": self.eligible_for_use,
}
class DataQuarantineService:
"""Derive and enforce safe use decisions from validation results."""
@staticmethod
def decide(
report: ValidationReport,
*,
previous: QuarantineDecision | None = None,
) -> QuarantineDecision:
"""Create a stable decision without silently releasing old quarantines.
A fresh passing validation report can be persisted as a new validated
version by the import transaction. It cannot automatically release an
existing quarantined record: the caller must explicitly record that
reviewed state transition against the new report/version.
"""
is_quarantined = report.validation_status == ValidationStatus.FAILED or report.quarantine_status == QuarantineStatus.QUARANTINED
failure_codes = tuple(
sorted(
{
issue.code
for issue in report.issues
if issue.severity.value == "error" or is_quarantined
}
)
)
requires_explicit_release = False
reason_codes = failure_codes
status = QuarantineStatus.QUARANTINED if is_quarantined else QuarantineStatus.NOT_QUARANTINED
if previous is not None and previous.quarantine_status == QuarantineStatus.QUARANTINED and not is_quarantined:
status = QuarantineStatus.QUARANTINED
requires_explicit_release = True
reason_codes = ("QUARANTINE_RELEASE_REQUIRES_EXPLICIT_PERSISTENCE",)
idempotency_key = _decision_key(
asset_id=report.asset_id,
contract_key=report.data_contract_key,
contract_version=report.data_contract_version,
report_sha256=report.report_sha256,
quarantine_status=status,
reason_codes=reason_codes,
requires_explicit_release=requires_explicit_release,
)
return QuarantineDecision(
asset_id=report.asset_id,
data_contract_key=report.data_contract_key,
data_contract_version=report.data_contract_version,
validation_report_sha256=report.report_sha256,
validation_status=report.validation_status,
quarantine_status=status,
reason_codes=reason_codes,
idempotency_key=idempotency_key,
requires_explicit_release=requires_explicit_release,
)
@staticmethod
def require_eligible(decision: QuarantineDecision, *, use: AssetUse) -> None:
"""Raise a typed error before a non-eligible artifact reaches a pipeline."""
if decision.eligible_for_use:
return
raise AppError(
code="DATASET_QUARANTINED",
message="Dataset is quarantined or failed validation and cannot enter this pipeline.",
status_code=409,
details={
"asset_id": decision.asset_id,
"use": use.value,
"quarantine_status": decision.quarantine_status.value,
"validation_status": decision.validation_status.value,
"reason_codes": list(decision.reason_codes),
"validation_report_sha256": decision.validation_report_sha256,
"idempotency_key": decision.idempotency_key,
"requires_explicit_release": decision.requires_explicit_release,
},
)
def _decision_key(
*,
asset_id: str,
contract_key: str,
contract_version: str,
report_sha256: str,
quarantine_status: QuarantineStatus,
reason_codes: tuple[str, ...],
requires_explicit_release: bool,
) -> str:
payload = {
"asset_id": asset_id,
"contract_key": contract_key,
"contract_version": contract_version,
"report_sha256": report_sha256,
"quarantine_status": quarantine_status.value,
"reason_codes": list(reason_codes),
"requires_explicit_release": requires_explicit_release,
}
return sha256(json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")).hexdigest()
@@ -0,0 +1,440 @@
"""Fail-closed provenance gates at data-consumption boundaries.
Import validation protects newly staged assets, but a persisted record can
subsequently become quarantined or have its provenance marked incomplete. The
callers of this service therefore re-check the durable Dataset state directly
before production inference, QA, derived processing, export, or authoritative
coverage reporting.
The only legacy relaxation is deliberately narrow: an *explicitly tagged*
fixture with no Phase-2 state can be used for fixture QA. A caller-provided
``fixture_mode`` flag alone never creates that trust claim. Fixture data can
never become a production inference, derived-processing, export or
authoritative-coverage input, and it never relaxes a recorded failed,
incomplete, or quarantined state.
"""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
import re
from typing import Any, Literal
from sqlalchemy import inspect as sa_inspect
from app.core.errors import AppError
from app.models import Dataset
DatasetConsumptionPurpose = Literal[
"production_inference",
"quality_assessment",
"reference_validation",
"derived_processing",
"authoritative_coverage",
"export",
]
_FIXTURE_SOURCE_KEYS = {"fixture", "test", "test_fixture", "test-fixture", "unit-test-fixture"}
_UNTRUSTED_SOURCE_KEYS = {"manual", "fixture", "experimental", "legacy_unknown"}
_CHECKSUM_SHA256 = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE)
_CONSUMABLE_SNAPSHOT_FRESHNESS = {"current", "not_applicable"}
_VALID_PURPOSES = {
"production_inference",
"quality_assessment",
"reference_validation",
"derived_processing",
"authoritative_coverage",
"export",
}
@dataclass(frozen=True)
class DatasetConsumptionDecision:
"""Auditable decision returned by the consumption gate."""
eligible: bool
purpose: DatasetConsumptionPurpose
fixture_legacy_exception: bool
reasons: tuple[str, ...]
evidence: dict[str, Any]
class DatasetConsumptionGate:
"""Evaluate durable provenance before a Dataset is consumed downstream."""
@staticmethod
def _value(dataset: Any, field: str, default: Any = None) -> Any:
if isinstance(dataset, Mapping):
return dataset.get(field, default)
return getattr(dataset, field, default)
@staticmethod
def _mapping(value: Any) -> dict[str, Any]:
return dict(value) if isinstance(value, Mapping) else {}
@classmethod
def _normalise(cls, value: Any) -> str:
return str(value or "").strip().lower()
@classmethod
def is_explicit_fixture(cls, dataset: Any) -> bool:
source = cls._normalise(cls._value(dataset, "source"))
source_name = cls._normalise(cls._value(dataset, "source_name"))
metadata = cls._mapping(cls._value(dataset, "metadata_json"))
source_metadata = cls._mapping(cls._value(dataset, "source_metadata"))
provenance = cls._mapping(cls._value(dataset, "provenance_metadata"))
return bool(
source in _FIXTURE_SOURCE_KEYS
or source_name in _FIXTURE_SOURCE_KEYS
or metadata.get("fixture") is True
or metadata.get("fixture_mode") is True
or source_metadata.get("fixture") is True
or source_metadata.get("fixture_mode") is True
or provenance.get("fixture") is True
or provenance.get("fixture_mode") is True
)
@classmethod
def _phase2_state_is_absent(cls, dataset: Any) -> bool:
fields = (
"data_contract_key",
"data_contract_version",
"validation_status",
"provenance_status",
"lineage_status",
"quarantine_status",
"source_registry_id",
"source_snapshot_id",
)
return all(cls._value(dataset, field) in {None, ""} for field in fields)
@staticmethod
def _is_transient_orm_dataset(dataset: Any) -> bool:
"""Recognize only unpersisted ORM fixtures, never database rows."""
if not isinstance(dataset, Dataset):
return False
try:
return bool(sa_inspect(dataset).transient)
except Exception: # pragma: no cover - defensive for unusual test doubles
return False
@classmethod
def _evidence(
cls,
dataset: Any,
purpose: DatasetConsumptionPurpose,
reference_task: str | None = None,
) -> dict[str, Any]:
source_registry = cls._value(dataset, "source_registry")
source_snapshot = cls._value(dataset, "source_snapshot")
source_policy = cls._mapping(cls._value(source_registry, "usage_policy_json"))
validation_authority = cls._mapping(source_policy.get("validation_authority"))
reference_approvals = cls._mapping(source_policy.get("reference_validation_approvals"))
return {
"dataset_id": str(cls._value(dataset, "id") or ""),
"purpose": purpose,
"dataset_status": cls._normalise(cls._value(dataset, "status")),
"source": cls._normalise(cls._value(dataset, "source")),
"source_name": cls._normalise(cls._value(dataset, "source_name")),
"data_contract_key": cls._value(dataset, "data_contract_key"),
"data_contract_version": cls._value(dataset, "data_contract_version"),
"validation_status": cls._normalise(cls._value(dataset, "validation_status")),
"provenance_status": cls._normalise(cls._value(dataset, "provenance_status")),
"lineage_status": cls._normalise(cls._value(dataset, "lineage_status")),
"quarantine_status": cls._normalise(cls._value(dataset, "quarantine_status")),
"checksum_sha256": cls._value(dataset, "checksum_sha256"),
"source_registry_id": str(cls._value(dataset, "source_registry_id") or ""),
"source_snapshot_id": str(cls._value(dataset, "source_snapshot_id") or ""),
"source_classification": cls._normalise(cls._value(source_registry, "classification")),
"source_key": cls._normalise(cls._value(source_registry, "source_key")),
"source_ground_truth_allowed": source_policy.get("ground_truth_allowed") is True,
"source_validation_authority": validation_authority,
"source_reference_validation_approvals": reference_approvals,
"source_authority_scope": cls._mapping(cls._value(source_registry, "authority_scope_json")),
"reference_task": cls._normalise(reference_task),
"snapshot_source_registry_id": str(cls._value(source_snapshot, "source_registry_id") or ""),
"snapshot_ingest_status": cls._normalise(cls._value(source_snapshot, "ingest_status")),
"snapshot_freshness_status": cls._normalise(cls._value(source_snapshot, "freshness_status")),
"snapshot_checksum_sha256": cls._value(source_snapshot, "checksum_sha256"),
}
@classmethod
def evaluate(
cls,
dataset: Any,
*,
purpose: DatasetConsumptionPurpose,
fixture_mode: bool = False,
reference_task: str | None = None,
) -> DatasetConsumptionDecision:
"""Return a stable decision without mutating the Dataset.
``fixture_mode`` can be supplied only by an explicitly fixture-only
caller. It is evidence for a QA fixture path, never a relaxation for
a production boundary or an explicit unsafe state.
"""
if purpose not in _VALID_PURPOSES:
raise ValueError(f"Unsupported dataset-consumption purpose: {purpose}")
evidence = cls._evidence(dataset, purpose, reference_task)
reasons: list[str] = []
explicit_fixture = cls.is_explicit_fixture(dataset)
phase2_absent = cls._phase2_state_is_absent(dataset)
# These are irrevocable safety states. They are checked before a
# fixture exception, so fixture rows cannot hide a bad recorded state.
if evidence["dataset_status"] in {"failed", "quarantined"}:
reasons.append("dataset_status_unsafe")
if evidence["quarantine_status"] == "quarantined":
reasons.append("dataset_quarantined")
if evidence["validation_status"] == "failed":
reasons.append("validation_failed")
if evidence["provenance_status"] == "incomplete":
reasons.append("provenance_incomplete")
if evidence["lineage_status"] == "incomplete":
reasons.append("lineage_incomplete")
if evidence["snapshot_ingest_status"] in {"failed", "quarantined"}:
reasons.append("source_snapshot_unsafe")
# A source family can be authoritative while an individual snapshot
# remains too old or insufficiently described to trust. Historical
# data that is intentionally valid needs an explicit
# ``not_applicable`` contract policy; an omitted, due or stale status
# cannot silently enter a production boundary.
if evidence["source_snapshot_id"] and evidence["snapshot_freshness_status"] not in _CONSUMABLE_SNAPSHOT_FRESHNESS:
reasons.append("source_snapshot_freshness_not_eligible")
if reasons:
return DatasetConsumptionDecision(
eligible=False,
purpose=purpose,
fixture_legacy_exception=False,
reasons=tuple(sorted(set(reasons))),
evidence=evidence,
)
# Fixtures are evidence for tests and QA only. They cannot become
# production inference, derived processing or export inputs merely by
# presenting a fixture flag at a public service boundary.
if phase2_absent and explicit_fixture:
if purpose == "quality_assessment":
return DatasetConsumptionDecision(
eligible=True,
purpose=purpose,
fixture_legacy_exception=True,
reasons=(),
evidence=evidence,
)
if purpose == "authoritative_coverage":
reasons.append("fixture_not_authoritative_coverage")
else:
reasons.append("fixture_qa_only")
elif phase2_absent and fixture_mode:
# `fixture_mode` is a caller flag, not a source trust claim. A
# manual/unknown production dataset must never self-designate as a
# fixture merely by supplying this parameter.
reasons.append("fixture_source_required")
# Existing service tests construct transient SQLAlchemy Dataset objects
# directly rather than retrieving a persisted row. A production
# `db.get()` result is persistent and never enters this branch. This
# compatibility path is intentionally unavailable to coverage, where
# a fixture must never appear authoritative.
if (
phase2_absent
and not reasons
and cls._is_transient_orm_dataset(dataset)
and purpose == "quality_assessment"
):
return DatasetConsumptionDecision(
eligible=True,
purpose=purpose,
fixture_legacy_exception=True,
reasons=(),
evidence=evidence,
)
# QA unit tests intentionally use projection objects
# instead of persisted ORM Datasets. Those projections cannot enter an
# application API boundary; keep the exception isolated to read-only
# candidate verification. Inference, reference validation, derived
# processing, export and coverage never accept a projection.
if phase2_absent and not reasons and not isinstance(dataset, Dataset) and purpose == "quality_assessment":
return DatasetConsumptionDecision(
eligible=True,
purpose=purpose,
fixture_legacy_exception=True,
reasons=(),
evidence=evidence,
)
if phase2_absent:
reasons.append("phase2_provenance_missing")
if evidence["dataset_status"] != "ready":
reasons.append("dataset_not_ready")
if evidence["validation_status"] != "passed":
reasons.append("validation_not_passed")
if evidence["provenance_status"] != "complete":
reasons.append("provenance_not_complete")
if evidence["lineage_status"] not in {"complete", "not_applicable"}:
reasons.append("lineage_not_complete")
if evidence["quarantine_status"] != "not_quarantined":
reasons.append("quarantine_status_not_clear")
if not evidence["data_contract_key"] or not evidence["data_contract_version"]:
reasons.append("data_contract_not_versioned")
if not _CHECKSUM_SHA256.fullmatch(str(evidence["checksum_sha256"] or "")):
reasons.append("dataset_checksum_invalid")
if not evidence["source_registry_id"]:
reasons.append("source_registry_missing")
if not evidence["source_snapshot_id"]:
reasons.append("source_snapshot_missing")
if evidence["source_registry_id"] and not evidence["source_classification"]:
reasons.append("source_registry_unresolved")
if evidence["source_snapshot_id"] and evidence["snapshot_ingest_status"] != "ingested":
reasons.append("source_snapshot_not_ingested")
if evidence["source_snapshot_id"] and not _CHECKSUM_SHA256.fullmatch(
str(evidence["snapshot_checksum_sha256"] or "")
):
reasons.append("source_snapshot_checksum_invalid")
if (
evidence["source_registry_id"]
and evidence["source_snapshot_id"]
and evidence["snapshot_source_registry_id"]
and evidence["source_registry_id"] != evidence["snapshot_source_registry_id"]
):
reasons.append("source_snapshot_registry_mismatch")
if (
_CHECKSUM_SHA256.fullmatch(str(evidence["checksum_sha256"] or ""))
and _CHECKSUM_SHA256.fullmatch(str(evidence["snapshot_checksum_sha256"] or ""))
and str(evidence["checksum_sha256"]).lower() != str(evidence["snapshot_checksum_sha256"]).lower()
):
reasons.append("source_snapshot_checksum_mismatch")
if evidence["source_key"] and evidence["source_name"] and evidence["source_key"] != evidence["source_name"]:
reasons.append("source_registry_identity_mismatch")
# Contextual, corroborative, authoritative and properly derived
# sources can serve their declared non-ground-truth roles once the
# full governed contract passes. Experimental/manual sources cannot
# cross a production boundary; only an explicitly marked fixture may
# participate in candidate QA.
experimental_source = (
evidence["source_classification"] == "experimental"
or evidence["source_key"] in _UNTRUSTED_SOURCE_KEYS
)
if experimental_source:
if purpose == "quality_assessment" and explicit_fixture:
pass
elif purpose == "quality_assessment":
reasons.append("experimental_source_requires_fixture_qa")
else:
reasons.append("experimental_source_not_allowed_for_purpose")
if purpose == "authoritative_coverage":
if evidence["source_classification"] != "authoritative":
reasons.append("coverage_source_not_authoritative")
if evidence["source_key"] and evidence["source_name"] and evidence["source_key"] != evidence["source_name"]:
reasons.append("coverage_source_identity_mismatch")
if purpose == "reference_validation":
if cls._normalise(cls._value(dataset, "dataset_role")) != "reference":
reasons.append("reference_dataset_role_required")
if evidence["source_classification"] != "authoritative":
reasons.append("reference_source_not_authoritative")
if evidence["source_ground_truth_allowed"] is not True:
reasons.append("reference_source_not_ground_truth_allowed")
if not evidence["reference_task"]:
reasons.append("reference_validation_task_required")
elif not cls._reference_task_is_approved(evidence):
reasons.append("reference_task_authority_not_approved")
return DatasetConsumptionDecision(
eligible=not reasons,
purpose=purpose,
fixture_legacy_exception=False,
reasons=tuple(sorted(set(reasons))),
evidence=evidence,
)
@classmethod
def assert_eligible(
cls,
dataset: Any,
*,
purpose: DatasetConsumptionPurpose,
fixture_mode: bool = False,
reference_task: str | None = None,
) -> DatasetConsumptionDecision:
decision = cls.evaluate(
dataset,
purpose=purpose,
fixture_mode=fixture_mode,
reference_task=reference_task,
)
if decision.eligible:
return decision
code = "DATASET_QUARANTINED" if any(
reason in {"dataset_status_unsafe", "dataset_quarantined", "validation_failed", "source_snapshot_unsafe"}
for reason in decision.reasons
) else "DATASET_PROVENANCE_INCOMPLETE"
raise AppError(
code=code,
message="Dataset cannot be consumed until its provenance and validation gates are satisfied.",
status_code=409,
details={
"dataset_id": decision.evidence["dataset_id"],
"purpose": purpose,
"reasons": list(decision.reasons),
"fixture_legacy_exception": decision.fixture_legacy_exception,
},
)
@classmethod
def eligible_for_authoritative_coverage(cls, dataset: Any) -> bool:
"""Return false instead of raising so coverage can report a gap safely."""
return cls.evaluate(dataset, purpose="authoritative_coverage").eligible
@staticmethod
def _reference_task_is_approved(evidence: Mapping[str, Any]) -> bool:
"""Require task-specific primary authority or an explicit zone approval.
``classification=authoritative`` is deliberately not a blanket
permission to serve as building truth. A source may be authoritative
for an address lifecycle or elevation product while remaining only
corroborative for footprint QA. A regional product that is marked
``*_pending_contract`` similarly remains blocked until an operator
records a narrow product-and-zone approval in its server-owned policy.
"""
task = str(evidence.get("reference_task") or "").strip().lower()
authority = DatasetConsumptionGate._normalise(
DatasetConsumptionGate._mapping(evidence.get("source_validation_authority")).get(task)
)
if authority == "primary":
return True
if authority not in {"approved", "approved_product_zone"}:
return False
approvals = DatasetConsumptionGate._mapping(evidence.get("source_reference_validation_approvals"))
approval = DatasetConsumptionGate._mapping(approvals.get(task))
if approval.get("approved") is not True:
return False
source_key = str(evidence.get("source_key") or "").strip().lower()
source_scope = DatasetConsumptionGate._mapping(evidence.get("source_authority_scope"))
source_zone = str(source_scope.get("zone") or source_scope.get("scope") or "").strip()
approved_keys = approval.get("source_keys")
approved_zones = approval.get("zones")
if not isinstance(approved_keys, list) or source_key not in {
str(value).strip().lower() for value in approved_keys
}:
return False
if not isinstance(approved_zones, list) or source_zone not in {
str(value).strip() for value in approved_zones
}:
return False
return True
File diff suppressed because it is too large Load Diff
+41 -21
View File
@@ -12,6 +12,7 @@ from sqlalchemy.orm import Session
from app.models import Area, Dataset, DatasetVersion, Metric, Project, QualityCheck
from app.schemas.demo import DemoWorkflowResponse
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
from app.services.geojson_service import parse_geojson_payload
from app.services.qa_service import QaService
from app.services.quality_service import QualityService
@@ -30,17 +31,17 @@ class DemoWorkflowService:
EXPECTED_METRICS_FILENAME = "expected_qa_metrics.json"
@staticmethod
def _add_initial_version(db: Session, dataset: Dataset) -> None:
db.add(
DatasetVersion(
dataset_id=dataset.id,
version=1,
storage_path=dataset.storage_path,
checksum_sha256=dataset.checksum_sha256,
source_metadata=dataset.source_metadata,
provenance_metadata=dataset.provenance_metadata,
)
def _add_initial_version(db: Session, dataset: Dataset) -> DatasetVersion:
version = DatasetVersion(
dataset_id=dataset.id,
version=1,
storage_path=dataset.storage_path,
checksum_sha256=dataset.checksum_sha256,
source_metadata=dataset.source_metadata,
provenance_metadata=dataset.provenance_metadata,
)
db.add(version)
return version
@staticmethod
def _repo_root() -> Path:
@@ -233,24 +234,34 @@ class DemoWorkflowService:
crs=metadata.get("crs"),
bounds_json=metadata.get("bounds_json"),
metadata_json=metadata,
status="ready",
status="validating",
)
db.add(dataset)
DemoWorkflowService._add_initial_version(db, dataset)
version = DemoWorkflowService._add_initial_version(db, dataset)
is_ready = DerivedDatasetGovernanceService.govern_vector(
db,
dataset=dataset,
dataset_version=version,
feature_collection=payload,
source_key="fixture",
operation="demo.fixture_vector",
operation_parameters={"fixture_name": filename, "role": role},
)
if is_ready:
VectorFeatureService.persist_geojson_features(
db=db,
dataset_id=dataset.id,
payload=payload,
feature_class=reference_layer_name or "building",
commit=False,
)
db.commit()
db.refresh(dataset)
VectorFeatureService.persist_geojson_features(
db=db,
dataset_id=dataset.id,
payload=payload,
feature_class=reference_layer_name or "building",
)
return dataset
@staticmethod
def _create_demo_raster_bytes() -> bytes:
numpy = importlib.import_module("numpy")
rasterio = importlib.import_module("rasterio")
rasterio_io = importlib.import_module("rasterio.io")
rasterio_transform = importlib.import_module("rasterio.transform")
@@ -318,10 +329,19 @@ class DemoWorkflowService:
crs=metadata.get("crs"),
bounds_json=bounds_json,
metadata_json=metadata,
status="ready",
status="validating",
)
db.add(dataset)
DemoWorkflowService._add_initial_version(db, dataset)
version = DemoWorkflowService._add_initial_version(db, dataset)
DerivedDatasetGovernanceService.govern_raster(
db,
dataset=dataset,
dataset_version=version,
raster_metadata=metadata,
source_key="fixture",
operation="demo.fixture_raster",
operation_parameters={"fixture_name": DemoWorkflowService.RASTER_FILENAME},
)
db.commit()
db.refresh(dataset)
return dataset
@@ -0,0 +1,516 @@
"""Governance for persisted derived and fixture datasets.
Dataset importers own raw-source ingestion. This small service owns the
other persistence boundary: artifacts produced inside the workbench (vector
and raster operations) and the explicitly local demo fixtures. It is kept
separate from :mod:`dataset_service` so an operation can never create a ready
dataset without a source registry binding, immutable snapshot, validation
report and, for derived results, a durable lineage edge.
The ``Session.query`` capability check deliberately preserves lightweight
unit-test doubles used by pre-Phase-2 tests. Real SQLAlchemy sessions always
take the governed branch; the compatibility branch is not reachable in the
application runtime.
"""
from __future__ import annotations
from datetime import datetime, timezone
from hashlib import sha256
import json
import re
from typing import Any, Mapping
from sqlalchemy.orm import Session
from app.models import Dataset, DatasetVersion
from app.services.data_contract_validation import (
IssueSeverity,
LineageStatus,
LineageEvidence,
ProvenanceStatus,
QuarantineStatus,
TransformationEvidence,
ValidationIssue,
ValidationReport,
ValidationStatus,
build_raster_ingest_input,
build_vector_ingest_input,
validate_registered_asset,
)
from app.services.data_quarantine_service import DataQuarantineService
from app.services.dataset_consumption_gate_service import DatasetConsumptionDecision, DatasetConsumptionGate
from app.services.source_registry_service import SourceRegistryService
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
_CANONICAL_VECTOR_CRS = "EPSG:4326"
class DerivedDatasetGovernanceService:
"""Apply Phase-2 provenance rules to non-importer Dataset creation.
Callers add the dataset and first immutable version, then call one of the
``govern_*`` methods before committing. A failed contract deliberately
leaves the artifact and its durable quarantine record in the transaction;
it is never silently promoted to ``ready``.
"""
@staticmethod
def persistence_available(db: Session) -> bool:
"""Return whether this is a real ORM persistence session.
Historical unit tests use minimal fakes with ``add``/``commit`` only.
Keeping that explicitly isolated avoids pretending a fake test store
has source-registry guarantees while production remains fail-closed.
"""
return callable(getattr(db, "query", None)) and callable(getattr(db, "flush", None))
@classmethod
def govern_vector(
cls,
db: Session,
*,
dataset: Dataset,
dataset_version: DatasetVersion,
feature_collection: Mapping[str, Any],
source_key: str,
operation: str,
parent_dataset: Dataset | None = None,
operation_parameters: Mapping[str, Any] | None = None,
) -> bool:
"""Validate and bind a vector result, returning ``True`` when ready."""
if not cls.persistence_available(db):
# Explicit compatibility for historical minimal test fixtures.
# Production sessions always have query/flush and never take this
# branch.
dataset.status = "ready"
return True
parent_gate = cls._parent_derived_processing_gate(parent_dataset)
metadata = dict(dataset.metadata_json or {})
output_crs = str(metadata.get("crs") or dataset.crs or _CANONICAL_VECTOR_CRS)
source = SourceRegistryService.ensure_server_owned_source(db, source_key)
snapshot = cls._record_snapshot(
db,
source_key=source_key,
dataset=dataset,
operation=operation,
source_crs=output_crs,
spatial_resolution=metadata.get("resolution_json") or {"status": "not_applicable"},
geographic_coverage={"bounds": metadata.get("bounds_json") or dataset.bounds_json},
observed_schema={
"dataset_type": "vector",
"geometry_types": metadata.get("geometry_types") or [],
"feature_count": metadata.get("feature_count"),
},
)
lineage = cls._lineage_evidence(parent_dataset, operation, operation_parameters)
report = validate_registered_asset(
build_vector_ingest_input(
asset_id=str(dataset.id),
source_crs=output_crs,
storage_crs=output_crs,
feature_collection=feature_collection,
checksum_sha256=dataset.checksum_sha256,
computed_checksum_sha256=dataset.checksum_sha256,
source_registry_id=str(source.id),
source_snapshot_id=str(snapshot.id),
imported_at=dataset.imported_at or datetime.now(timezone.utc),
metadata=cls._contract_metadata(metadata, source.license_name),
observed_at=dataset.observed_at,
valid_from=dataset.valid_from,
valid_to=dataset.valid_to,
temporal_unknown_reason=cls._temporal_unknown_reason(dataset),
source_version=dataset.source_version,
source_version_unknown_reason=cls._source_version_unknown_reason(dataset),
lineage=lineage,
)
)
if parent_gate is not None and not parent_gate.eligible:
report = cls._with_parent_gate_failure(report, parent_gate)
return cls._apply(
db,
dataset=dataset,
dataset_version=dataset_version,
source=source,
snapshot=snapshot,
report=report,
stage="derived_vector_validation",
parent_dataset=parent_dataset,
operation=operation,
operation_parameters=operation_parameters,
)
@classmethod
def govern_raster(
cls,
db: Session,
*,
dataset: Dataset,
dataset_version: DatasetVersion,
raster_metadata: Mapping[str, Any],
source_key: str,
operation: str,
parent_dataset: Dataset | None = None,
operation_parameters: Mapping[str, Any] | None = None,
) -> bool:
"""Validate and bind a raster result, returning ``True`` when ready."""
if not cls.persistence_available(db):
dataset.status = "ready"
return True
parent_gate = cls._parent_derived_processing_gate(parent_dataset)
metadata = dict(raster_metadata or {})
output_crs = str(metadata.get("crs") or dataset.crs or "") or None
source = SourceRegistryService.ensure_server_owned_source(db, source_key)
snapshot = cls._record_snapshot(
db,
source_key=source_key,
dataset=dataset,
operation=operation,
source_crs=output_crs,
spatial_resolution=cls._raster_resolution(metadata, output_crs),
geographic_coverage={"bounds": metadata.get("bounds") or dataset.bounds_json},
observed_schema={
"dataset_type": "raster",
"width": metadata.get("width"),
"height": metadata.get("height"),
"band_count": metadata.get("band_count"),
"dtype": metadata.get("dtype"),
},
)
lineage = cls._lineage_evidence(parent_dataset, operation, operation_parameters)
report = validate_registered_asset(
build_raster_ingest_input(
asset_id=str(dataset.id),
source_crs=output_crs,
storage_crs=output_crs,
raster_profile=metadata,
bounds=metadata.get("bounds") or dataset.bounds_json,
resolution=cls._raster_resolution(metadata, output_crs),
checksum_sha256=dataset.checksum_sha256,
computed_checksum_sha256=dataset.checksum_sha256,
source_registry_id=str(source.id),
source_snapshot_id=str(snapshot.id),
imported_at=dataset.imported_at or datetime.now(timezone.utc),
metadata=cls._contract_metadata(metadata, source.license_name),
observed_at=dataset.observed_at,
valid_from=dataset.valid_from,
valid_to=dataset.valid_to,
temporal_unknown_reason=cls._temporal_unknown_reason(dataset),
source_version=dataset.source_version,
source_version_unknown_reason=cls._source_version_unknown_reason(dataset),
lineage=lineage,
)
)
if parent_gate is not None and not parent_gate.eligible:
report = cls._with_parent_gate_failure(report, parent_gate)
return cls._apply(
db,
dataset=dataset,
dataset_version=dataset_version,
source=source,
snapshot=snapshot,
report=report,
stage="derived_raster_validation",
parent_dataset=parent_dataset,
operation=operation,
operation_parameters=operation_parameters,
)
@classmethod
def _record_snapshot(
cls,
db: Session,
*,
source_key: str,
dataset: Dataset,
operation: str,
source_crs: str | None,
spatial_resolution: Mapping[str, Any] | None,
geographic_coverage: Mapping[str, Any] | None,
observed_schema: Mapping[str, Any] | None,
):
checksum = cls._checksum_or_placeholder(dataset.checksum_sha256)
# A derived artifact has a distinct creation event even when its bytes
# equal a prior output. Include the immutable dataset id so source
# snapshots never collide on a different fetched_at timestamp.
snapshot_key = f"{source_key}:{operation}:{dataset.id}:{checksum}"
return SourceRegistryService.record_snapshot(
db,
source_key=source_key,
snapshot_key=snapshot_key,
checksum_sha256=checksum,
source_version=dataset.source_version or f"{operation}:1.0.0",
snapshot_at=dataset.observed_at,
fetched_at=dataset.imported_at or datetime.now(timezone.utc),
crs=source_crs,
units=cls._units_for_crs(source_crs),
spatial_resolution=dict(spatial_resolution or {"status": "unknown"}),
temporal_coverage={
"observed_at": cls._datetime_value(dataset.observed_at),
"valid_from": cls._datetime_value(dataset.valid_from),
"valid_to": cls._datetime_value(dataset.valid_to),
},
geographic_coverage=dict(geographic_coverage or {"status": "unknown"}),
observed_schema=dict(observed_schema or {"status": "unknown"}),
# A transform cannot establish source freshness. With no source
# observation it is not applicable; with one it still needs an
# explicit policy review rather than a fabricated "current" flag.
freshness_status="not_applicable" if dataset.observed_at is None else "review_required",
ingest_status="ingested",
known_limitations=[
"Derived and fixture artifacts inherit no automatic source authority beyond their explicit registry entry and lineage.",
],
snapshot_metadata={
"operation": operation,
"dataset_id": str(dataset.id),
"storage_path": dataset.storage_path,
"artifact_checksum_sha256": dataset.checksum_sha256,
},
)
@classmethod
def _apply(
cls,
db: Session,
*,
dataset: Dataset,
dataset_version: DatasetVersion,
source: Any,
snapshot: Any,
report: ValidationReport,
stage: str,
parent_dataset: Dataset | None,
operation: str,
operation_parameters: Mapping[str, Any] | None,
) -> bool:
fields = report.persistence_fields()
dataset.validation_report_json = fields["validation_report_json"]
dataset_version.validation_report_json = fields["validation_report_json"]
SourceRegistryService.bind_dataset_provenance(
dataset,
source=source,
snapshot=snapshot,
data_contract_key=fields["data_contract_key"],
data_contract_version=fields["data_contract_version"],
validation_status=fields["validation_status"],
provenance_status=fields["provenance_status"],
lineage_status=fields["lineage_status"],
)
SourceRegistryService.bind_dataset_version_provenance(
dataset_version,
source=source,
snapshot=snapshot,
data_contract_key=fields["data_contract_key"],
data_contract_version=fields["data_contract_version"],
validation_status=fields["validation_status"],
provenance_status=fields["provenance_status"],
lineage_status=fields["lineage_status"],
)
# IDs for DatasetVersion defaults exist only after the caller adds and
# flushes both rows. The operation services call this before commit.
db.flush()
if parent_dataset is not None:
SourceRegistryService.record_lineage_edge(
db,
parent_dataset_id=parent_dataset.id,
child_dataset_id=dataset.id,
parent_dataset_version_id=cls._latest_parent_version_id(db, parent_dataset),
child_dataset_version_id=dataset_version.id,
relation_type="derived_from",
transformation_name=operation,
transformation_version="1.0.0",
parameters=dict(operation_parameters or {}),
input_checksum_sha256=cls._valid_checksum(parent_dataset.checksum_sha256),
output_checksum_sha256=cls._valid_checksum(dataset.checksum_sha256),
)
decision = DataQuarantineService.decide(report)
if decision.eligible_for_use:
dataset.status = "ready"
dataset.quarantine_status = "not_quarantined"
return True
SourceRegistryService.quarantine_dataset(
db,
dataset=dataset,
dataset_version=dataset_version,
source_snapshot=snapshot,
stage=stage,
reason_code=(decision.reason_codes[0] if decision.reason_codes else "DATA_CONTRACT_FAILED"),
details={"validation_report": report.to_dict(), "quarantine_decision": decision.to_dict()},
artifact_path=dataset.storage_path,
artifact_checksum_sha256=cls._valid_checksum(dataset.checksum_sha256),
)
return False
@staticmethod
def _contract_metadata(metadata: Mapping[str, Any], license_name: str) -> dict[str, Any]:
values = dict(metadata)
values.setdefault("license", license_name)
return values
@classmethod
def _lineage_evidence(
cls,
parent_dataset: Dataset | None,
operation: str,
operation_parameters: Mapping[str, Any] | None,
) -> LineageEvidence:
upstream_ids: tuple[str, ...] = ()
upstream_checksums: tuple[str, ...] = ()
if parent_dataset is not None:
upstream_ids = (str(parent_dataset.id),)
# An invalid/missing parent checksum intentionally fails the
# derived contract rather than inventing traceability. The same
# applies to a legacy/unvalidated parent: it may remain visible
# as evidence, but cannot create a new ready derived asset.
parent_is_governed = (
parent_dataset.status == "ready"
and parent_dataset.quarantine_status == "not_quarantined"
and parent_dataset.validation_status == "passed"
and parent_dataset.provenance_status == "complete"
and parent_dataset.source_registry_id is not None
and parent_dataset.source_snapshot_id is not None
)
upstream_checksums = (
parent_dataset.checksum_sha256 if parent_is_governed else "parent_dataset_not_governed",
)
transform_checksum = cls._stable_hash(
{"operation": operation, "version": "1.0.0", "parameters": dict(operation_parameters or {})}
)
return LineageEvidence(
upstream_asset_ids=upstream_ids,
upstream_checksums_sha256=upstream_checksums,
transformations=(
TransformationEvidence(
name=operation,
version="1.0.0",
checksum_sha256=transform_checksum,
),
),
)
@staticmethod
def _parent_derived_processing_gate(parent_dataset: Dataset | None) -> DatasetConsumptionDecision | None:
"""Evaluate the durable parent boundary before creating a ready child.
We deliberately turn a rejected parent into a child validation failure
(instead of simply raising): the output is then persisted with its
source snapshot, validation evidence and durable quarantine record.
This makes an attempted derivation from a manual or experimental
dataset observable and prevents a caller from bypassing the boundary
by invoking the governance service directly.
"""
if parent_dataset is None:
return None
return DatasetConsumptionGate.evaluate(parent_dataset, purpose="derived_processing")
@staticmethod
def _with_parent_gate_failure(
report: ValidationReport,
decision: DatasetConsumptionDecision,
) -> ValidationReport:
"""Attach an auditable, fail-closed lineage failure to a report."""
issue = ValidationIssue(
code="PARENT_DATASET_NOT_ELIGIBLE_FOR_DERIVED_PROCESSING",
category="lineage",
field="lineage.parent_dataset",
message="Parent dataset failed the governed derived-processing consumption gate.",
severity=IssueSeverity.ERROR,
expected="eligible governed parent dataset",
observed={
"dataset_id": decision.evidence.get("dataset_id"),
"reasons": list(decision.reasons),
"source_key": decision.evidence.get("source_key"),
"source_classification": decision.evidence.get("source_classification"),
},
)
return ValidationReport(
asset_id=report.asset_id,
data_contract_key=report.data_contract_key,
data_contract_version=report.data_contract_version,
contract_fingerprint_sha256=report.contract_fingerprint_sha256,
validation_status=ValidationStatus.FAILED,
provenance_status=(
ProvenanceStatus.INCOMPLETE
if report.provenance_status == ProvenanceStatus.COMPLETE
else report.provenance_status
),
lineage_status=LineageStatus.INCOMPLETE,
quarantine_status=QuarantineStatus.QUARANTINED,
validation_scope=report.validation_scope,
checked_at=report.checked_at,
issues=(*report.issues, issue),
)
@staticmethod
def _latest_parent_version_id(db: Session, dataset: Dataset):
version = (
db.query(DatasetVersion)
.filter(DatasetVersion.dataset_id == dataset.id)
.order_by(DatasetVersion.version.desc())
.first()
)
return version.id if version is not None else None
@staticmethod
def _raster_resolution(metadata: Mapping[str, Any], crs: str | None) -> dict[str, Any] | None:
values = metadata.get("resolution")
if not isinstance(values, (list, tuple)) or len(values) < 2:
return None
try:
return {"x": abs(float(values[0])), "y": abs(float(values[1])), "unit": DerivedDatasetGovernanceService._resolution_unit(crs)}
except (TypeError, ValueError):
return None
@staticmethod
def _resolution_unit(crs: str | None) -> str:
return "degree" if str(crs or "").upper() == _CANONICAL_VECTOR_CRS else "m"
@staticmethod
def _units_for_crs(crs: str | None) -> str:
return "degrees" if str(crs or "").upper() == _CANONICAL_VECTOR_CRS else "metres"
@staticmethod
def _temporal_unknown_reason(dataset: Dataset) -> str | None:
if dataset.observed_at is not None:
return None
return "Derived or fixture artifact inherits no precise observation timestamp from its input."
@staticmethod
def _source_version_unknown_reason(dataset: Dataset) -> str | None:
if dataset.source_version:
return None
return "Derived or fixture artifact has no source edition; the transform version is recorded separately."
@staticmethod
def _datetime_value(value: datetime | None) -> str | None:
return value.astimezone(timezone.utc).isoformat() if value is not None else None
@staticmethod
def _stable_hash(value: Mapping[str, Any]) -> str:
return sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")).hexdigest()
@staticmethod
def _valid_checksum(value: str | None) -> str | None:
normalized = str(value or "").strip().lower()
return normalized if _SHA256.fullmatch(normalized) else None
@classmethod
def _checksum_or_placeholder(cls, value: str | None) -> str:
checksum = cls._valid_checksum(value)
if checksum is not None:
return checksum
# The contract receives the original invalid/missing checksum and
# quarantines it. A deterministic placeholder only permits storing
# the rejected snapshot without fabricating a valid artifact hash.
return cls._stable_hash({"invalid_dataset_checksum": value or "missing"})
+80 -10
View File
@@ -19,10 +19,12 @@ from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project, Vect
from app.schemas.detection import DetectionListResponse, DetectionRead, DetectionRunListResponse, DetectionRunRead, DetectionRunResponse
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
from app.services.detection_qa_service import DetectionQaService
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
from app.services.model_asset_catalog_service import ModelAssetCatalogService
from app.services.model_registry_service import ModelRegistryService
from app.services.qa_service import QaService
from app.services.quality_service import QualityService
from app.services.runtime_model_provenance_service import RuntimeModelProvenance, RuntimeModelProvenanceService
from app.services.temporal_compatibility_service import TemporalCompatibilityService
from app.services.yolo_adapter import YoloDetectionAdapter
@@ -97,6 +99,18 @@ class DetectionService:
if model.model_id == resolved_settings.yolo_model_id and resolved_settings.yolo_enforce_validation_scope:
DetectionService._validate_model_area_scope(db, dataset, resolved_settings)
# Never enter a production inference path with a persisted dataset
# that has failed validation, incomplete provenance, or an active
# quarantine. Fixture detection is a separate QA/test-only path.
if model.model_id == "manual-fixture-detector":
DatasetConsumptionGate.assert_eligible(
dataset,
purpose="quality_assessment",
fixture_mode=True,
)
elif model.model_id == resolved_settings.yolo_model_id and model.configured:
DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
run_parameters = {
"model_id": model.model_id,
"model_asset_id": selected_model_asset.model_asset_id if selected_model_asset else None,
@@ -352,15 +366,6 @@ class DetectionService:
reference_dataset,
)
detections = DetectionService._query_detection_rows(
db,
analysis_run_id=analysis_run_id,
dataset_id=run.dataset_id,
class_name=class_name,
min_confidence=min_confidence,
)
raw_candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in detections]
candidate_geometries = raw_candidate_geometries
run_parameters = run.parameters_json if isinstance(run.parameters_json, dict) else {}
manifest_path = DetectionQaService.tile_manifest_path(run_parameters)
resolved_settings = get_settings()
@@ -375,6 +380,33 @@ class DetectionService:
status_code=422,
)
fixture_parameters = run_parameters.get("parameters_json")
fixture_mode = bool(
run.model_name == "manual-fixture-detector"
and isinstance(fixture_parameters, dict)
and fixture_parameters.get("fixture_mode") is True
)
DatasetConsumptionGate.assert_eligible(
candidate_dataset,
purpose="quality_assessment",
fixture_mode=fixture_mode,
)
DatasetConsumptionGate.assert_eligible(
reference_dataset,
purpose="reference_validation",
reference_task="building_validation",
)
detections = DetectionService._query_detection_rows(
db,
analysis_run_id=analysis_run_id,
dataset_id=run.dataset_id,
class_name=class_name,
min_confidence=min_confidence,
)
raw_candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in detections]
candidate_geometries = raw_candidate_geometries
coverage = None
if manifest_path:
manifest = DetectionService._load_tile_manifest(manifest_path, resolved_settings.yolo_max_tiles)
@@ -728,6 +760,19 @@ class DetectionService:
) -> tuple[list[Detection], dict[str, Any]]:
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles)
model_path = Path(settings.yolo_model_path or "").expanduser()
runtime_model_provenance = RuntimeModelProvenanceService.validate_for_production_runtime(
db=db,
model_path=model_path,
model_id=model_name,
task_type="object_detection",
expected_model_version=model_version,
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
)
DetectionService._attach_runtime_model_provenance(
analysis_run,
job,
runtime_model_provenance,
)
adapter = yolo_adapter_class(settings)
model = adapter.load_model(model_path)
allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)}
@@ -785,7 +830,10 @@ class DetectionService:
"y_max": float(bbox[3]),
},
source_tile_path=candidate["source_tile_path"],
properties_json=candidate["properties"],
properties_json={
**candidate["properties"],
"runtime_model_provenance": runtime_model_provenance.as_dict(),
},
)
db.add(detection)
persisted.append(detection)
@@ -796,8 +844,30 @@ class DetectionService:
"raw_detection_count": len(candidates),
"suppressed_detection_count": len(candidates) - len(filtered_candidates),
"duplicate_iou_threshold": float(settings.yolo_duplicate_iou_threshold),
"runtime_model_provenance": runtime_model_provenance.as_dict(),
}
@staticmethod
def _attach_runtime_model_provenance(
analysis_run: AnalysisRun,
job: Job,
provenance: RuntimeModelProvenance,
) -> None:
"""Persist byte-bound model evidence with the run before adapter loading.
Individual detections retain the same evidence in ``properties_json``;
this run-level copy is the compact audit root for a complete inference.
Assigning fresh dictionaries matters for SQLAlchemy JSON change tracking.
"""
evidence = provenance.as_dict()
analysis_parameters = dict(analysis_run.parameters_json or {})
analysis_parameters["runtime_model_provenance"] = evidence
analysis_run.parameters_json = analysis_parameters
job_parameters = dict(job.parameters_json or {})
job_parameters["runtime_model_provenance"] = evidence
job.parameters_json = job_parameters
@staticmethod
def _canonical_class_name(value: Any) -> str:
return str(value or "").strip().casefold()
+34
View File
@@ -23,6 +23,7 @@ from app.schemas.flood_hazard import FloodHazardPartitionSelectionRequest, Flood
from app.schemas.temporal import TemporalComparisonRequest
from app.schemas.thematic_raster import ThematicRasterSelectionRequest
from app.services.dataset_service import DatasetService
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
from app.services.detection_service import DetectionService
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
from app.services.segmentation_service import SegmentationService
@@ -34,12 +35,39 @@ from app.services.vector_feature_service import VectorFeatureService
class ExportService:
@staticmethod
def _assert_run_source_dataset_exportable(db: Session, run: AnalysisRun) -> Dataset:
"""Block an output export when its persisted source dataset is unsafe."""
if not run.dataset_id:
raise AppError(
code="DATASET_PROVENANCE_INCOMPLETE",
message="Analysis output cannot be exported without a persisted source dataset.",
status_code=409,
)
dataset = db.get(Dataset, run.dataset_id)
if not dataset or dataset.project_id != run.project_id:
raise AppError(code="DATASET_NOT_FOUND", message="Analysis source dataset not found", status_code=404)
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
return dataset
@staticmethod
def export_map_result(
db: Session,
payload: MapResultExportRequest,
) -> ExportCreateResponse:
if payload.mode == "evolution":
earlier_dataset = db.get(Dataset, payload.earlier_dataset_id)
later_dataset = db.get(Dataset, payload.later_dataset_id)
if (
not earlier_dataset
or not later_dataset
or earlier_dataset.project_id != payload.project_id
or later_dataset.project_id != payload.project_id
):
raise AppError(code="DATASET_NOT_FOUND", message="Temporal export dataset not found", status_code=404)
DatasetConsumptionGate.assert_eligible(earlier_dataset, purpose="export")
DatasetConsumptionGate.assert_eligible(later_dataset, purpose="export")
comparison = TemporalAnalysisService.compare(
db,
project_id=payload.project_id,
@@ -86,6 +114,7 @@ class ExportService:
dataset = db.get(Dataset, payload.dataset_id)
if not dataset or dataset.project_id != payload.project_id:
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
if dataset.dataset_type in DatasetService.VECTOR_TYPES:
if payload.partitioned:
return ExportService.export_partitioned_vector_selection_geojson(
@@ -219,6 +248,7 @@ class ExportService:
limit: int = 1000,
name: str | None = None,
) -> ExportCreateResponse:
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
if dataset.source_name != "vmm_vha_bathymetry_profiles" or partition_scope_key != "flanders":
raise AppError(
code="PARTITIONED_VECTOR_EXPORT_UNSUPPORTED",
@@ -308,6 +338,7 @@ class ExportService:
details={"dataset_type": dataset.dataset_type},
status_code=400,
)
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
selection_kwargs: dict[str, Any] = {
"dataset_id": dataset_id,
@@ -374,6 +405,7 @@ class ExportService:
details={"dataset_type": dataset.dataset_type},
status_code=400,
)
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
feature_collection = DatasetService.get_dataset_geojson(db, dataset_id)
filename = ExportService._filename(name, f"{dataset.id}.geojson", ".geojson")
@@ -401,6 +433,7 @@ class ExportService:
run = db.get(AnalysisRun, analysis_run_id)
if not run or run.analysis_type != "detection":
raise AppError(code="DETECTION_RUN_NOT_FOUND", message="Detection run not found", status_code=404)
ExportService._assert_run_source_dataset_exportable(db, run)
feature_collection = DetectionService.detections_to_geojson(db, analysis_run_id=analysis_run_id)
filename = ExportService._filename(name, f"{run.id}-detections.geojson", ".geojson")
@@ -428,6 +461,7 @@ class ExportService:
run = db.get(AnalysisRun, analysis_run_id)
if not run or run.analysis_type != "segmentation":
raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404)
ExportService._assert_run_source_dataset_exportable(db, run)
feature_collection = SegmentationService.segmentations_to_geojson(db, analysis_run_id=analysis_run_id)
filename = ExportService._filename(name, f"{run.id}-segmentations.geojson", ".geojson")
+55 -9
View File
@@ -9,6 +9,7 @@ from app.services.segmentation_adapter import (
SamSegmentationAdapter,
YoloSegmentationAdapter,
)
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
from app.services.yolo_adapter import YoloDetectionAdapter
from app.core.errors import AppError
@@ -144,9 +145,24 @@ class ModelRegistryService:
elif not model_path.exists() or not model_path.is_file():
limitation = "YOLO_SEG_MODEL_PATH does not point to an existing local model file. GeoIntel will not download segmentation model weights automatically."
else:
configured = True
status = "configured"
limitation = "Configured for local YOLO segmentation inference over an existing raster tile manifest."
try:
RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id=settings.yolo_seg_model_id,
task_type="segmentation",
expected_model_version=settings.yolo_seg_model_version,
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
)
except AppError as exc:
status = "contract_incomplete"
limitation = (
"Configured YOLO segmentation weights are not runnable until their immutable "
f"runtime provenance sidecar validates: {exc.message}"
)
else:
configured = True
status = "configured"
limitation = "Configured for local YOLO segmentation inference over an existing raster tile manifest."
return DetectionModelCapability(
model_id=settings.yolo_seg_model_id,
@@ -186,9 +202,24 @@ class ModelRegistryService:
elif not model_path.exists() or not model_path.is_file():
limitation = "SAM_MODEL_PATH does not point to an existing local model file. GeoIntel will not download segmentation model weights automatically."
else:
configured = True
status = "configured"
limitation = "Configured for local class-agnostic SAM segmentation over an existing raster tile manifest."
try:
RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id=settings.sam_model_id,
task_type="segmentation",
expected_model_version=settings.sam_model_version,
allowed_frameworks=("ultralytics/sam", "sam", "ultralytics", "pytorch"),
)
except AppError as exc:
status = "contract_incomplete"
limitation = (
"Configured SAM weights are not runnable until their immutable runtime provenance "
f"sidecar validates: {exc.message}"
)
else:
configured = True
status = "configured"
limitation = "Configured for local class-agnostic SAM segmentation over an existing raster tile manifest."
return DetectionModelCapability(
model_id=settings.sam_model_id,
@@ -233,9 +264,24 @@ class ModelRegistryService:
status = "accelerator_unavailable"
limitation = exc.message
else:
configured = True
status = "configured"
limitation = "Configured for local YOLO inference over an existing raster tile manifest within its validated area scope."
try:
RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id=settings.yolo_model_id,
task_type="object_detection",
expected_model_version=settings.yolo_model_version,
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
)
except AppError as exc:
status = "contract_incomplete"
limitation = (
"Configured YOLO weights are not runnable until their immutable runtime provenance "
f"sidecar validates: {exc.message}"
)
else:
configured = True
status = "configured"
limitation = "Configured for local YOLO inference over an existing raster tile manifest within its validated area scope."
return DetectionModelCapability(
model_id=settings.yolo_model_id,
+7 -1
View File
@@ -11,10 +11,10 @@ from shapely.geometry.base import BaseGeometry
from shapely.strtree import STRtree
from shapely.ops import unary_union
from shapely.validation import make_valid
from shapely.geometry import shape
from app.core.errors import AppError
from app.models import Area, Dataset
from app.schemas.qa import QaProviderComparisonResult
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
from app.services.vector_operations_service import VectorOperationsService
@@ -264,6 +264,12 @@ class QaService:
reference_dataset_id,
expected_project_id=project_id,
)
DatasetConsumptionGate.assert_eligible(candidate_dataset, purpose="quality_assessment")
DatasetConsumptionGate.assert_eligible(
reference_dataset,
purpose="reference_validation",
reference_task="building_validation",
)
area_geometry = QaService._validate_area(
db,
@@ -13,6 +13,7 @@ from shapely.validation import make_valid
from app.core.errors import AppError
from app.models import Area, Dataset, DatasetVersion
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
from app.services.raster_service import extract_raster_metadata
from app.services.storage_service import StorageService
@@ -334,7 +335,10 @@ class RasterOperationsService:
dataset_type="raster",
source=f"operation:{operation_name}",
dataset_role="derived",
source_name=source_dataset.source_name,
# The source identity of this artifact is the server-owned
# derived-operation registry entry. The parent remains explicit
# in provenance and the lineage edge below.
source_name="derived",
source_metadata=source_dataset.source_metadata,
provenance_metadata=provenance,
imported_at=datetime.now(timezone.utc),
@@ -360,22 +364,31 @@ class RasterOperationsService:
resolution_json=metadata_payload.get("resolution"),
bands_json={"dtype": metadata_payload.get("dtype")} if metadata_payload.get("dtype") is not None else None,
metadata_json=metadata_payload,
status="ready",
status="validating",
)
db.add(derived_dataset)
db.add(
DatasetVersion(
dataset_id=derived_dataset.id,
version=1,
storage_path=derived_dataset.storage_path,
source_version=derived_dataset.source_version,
observed_at=derived_dataset.observed_at,
valid_from=derived_dataset.valid_from,
valid_to=derived_dataset.valid_to,
checksum_sha256=derived_dataset.checksum_sha256,
source_metadata=derived_dataset.source_metadata,
provenance_metadata=derived_dataset.provenance_metadata,
)
dataset_version = DatasetVersion(
dataset_id=derived_dataset.id,
version=1,
storage_path=derived_dataset.storage_path,
source_version=derived_dataset.source_version,
observed_at=derived_dataset.observed_at,
valid_from=derived_dataset.valid_from,
valid_to=derived_dataset.valid_to,
checksum_sha256=derived_dataset.checksum_sha256,
source_metadata=derived_dataset.source_metadata,
provenance_metadata=derived_dataset.provenance_metadata,
)
db.add(dataset_version)
DerivedDatasetGovernanceService.govern_raster(
db,
dataset=derived_dataset,
dataset_version=dataset_version,
raster_metadata=metadata_payload,
source_key="derived",
operation=operation_name,
parent_dataset=source_dataset,
operation_parameters=metadata_payload.get("operation_parameters"),
)
db.commit()
db.refresh(derived_dataset)
@@ -0,0 +1,652 @@
"""Fail-closed provenance validation for configured local model files.
Configured YOLO and SAM weights are intentionally not trusted merely because a
file exists on the server. Before an adapter is allowed to load local weights,
this service verifies a neighbouring immutable sidecar manifest, validates the
model artifact against ``geointel.model.pytorch@1.0.0`` and binds that sidecar
to the server-owned source registry and source snapshot recorded in Postgres.
``validate_for_runtime`` remains a structural sidecar check for catalog and
preflight inspection. Production inference must call
``validate_for_production_runtime`` with a database session; it rejects an
unregistered, mismatched, stale, unsafe or quarantined source snapshot before
an adapter can load model bytes. This keeps focused unit tests able to inspect
sidecars without inventing database rows while keeping the production boundary
strict.
"""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timezone
from hashlib import sha256
import json
from pathlib import Path
import re
from typing import Any, Mapping
from uuid import UUID
from app.core.errors import AppError
from app.models import SourceRegistry, SourceSnapshot
from app.services.data_contract_validation import (
PYTORCH_MODEL_CONTRACT_KEY,
PYTORCH_MODEL_CONTRACT_VERSION,
LineageEvidence,
TransformationEvidence,
build_model_validation_input,
validate_registered_asset,
)
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
_CONSUMABLE_FRESHNESS = {"current", "not_applicable"}
_SAFE_SOURCE_REGISTRY_INGEST_STATUSES = {"configured", "ingested"}
_SAFE_SOURCE_SNAPSHOT_INGEST_STATUS = "ingested"
@dataclass(frozen=True)
class RuntimeModelProvenance:
"""Validated, immutable evidence attached to one local inference run."""
model_id: str
task_type: str
model_path: str
manifest_path: str
model_sha256: str
manifest_sha256: str
runtime_manifest_sha256: str
data_contract_key: str
data_contract_version: str
validation_report_sha256: str
source_registry_id: str
source_snapshot_id: str
source_snapshot_checksum_sha256: str
source_version: str
def as_dict(self) -> dict[str, str]:
return {
"model_id": self.model_id,
"task_type": self.task_type,
"model_path": self.model_path,
"manifest_path": self.manifest_path,
"model_sha256": self.model_sha256,
"manifest_sha256": self.manifest_sha256,
"runtime_manifest_sha256": self.runtime_manifest_sha256,
"data_contract_key": self.data_contract_key,
"data_contract_version": self.data_contract_version,
"validation_report_sha256": self.validation_report_sha256,
"source_registry_id": self.source_registry_id,
"source_snapshot_id": self.source_snapshot_id,
"source_snapshot_checksum_sha256": self.source_snapshot_checksum_sha256,
"source_version": self.source_version,
}
class RuntimeModelProvenanceService:
"""Validate model sidecars and their production database binding.
A sidecar lives next to its model as ``<model-file>.geointel-model.json``.
Its ``metadata.runtime_manifest_sha256`` is the SHA-256 of canonical JSON
after omitting that one self-referential field. Any other mutation of the
manifest therefore invalidates it. Structural validation is deliberately
separate from :meth:`validate_for_production_runtime`: discovery and
preflight have no database session, whereas every production inference
path must prove an active source registry/snapshot binding.
"""
MANIFEST_SUFFIX = ".geointel-model.json"
MANIFEST_SCHEMA_VERSION = "geointel.runtime-model-manifest/v1"
SOURCE_REGISTRY_KEY = "model"
@classmethod
def manifest_path_for_model(cls, model_path: str | Path) -> Path:
path = Path(model_path).expanduser()
return Path(f"{path}{cls.MANIFEST_SUFFIX}")
@staticmethod
def manifest_self_checksum(payload: Mapping[str, Any]) -> str:
"""Hash sidecar semantics without its self-referential checksum field."""
canonical_payload = json.loads(json.dumps(payload, sort_keys=True, ensure_ascii=True))
metadata = canonical_payload.get("metadata")
if isinstance(metadata, dict):
metadata.pop("runtime_manifest_sha256", None)
encoded = json.dumps(
canonical_payload,
sort_keys=True,
separators=(",", ":"),
ensure_ascii=True,
).encode("utf-8")
return sha256(encoded).hexdigest()
@classmethod
def validate_for_runtime(
cls,
*,
model_path: str | Path,
model_id: str,
task_type: str,
expected_model_version: str | None = None,
allowed_frameworks: tuple[str, ...] = (),
) -> RuntimeModelProvenance:
"""Return structural sidecar evidence without asserting database state.
This method is appropriate for read-only catalog/preflight checks and
focused sidecar unit tests. It is insufficient for production
inference; adapters must use :meth:`validate_for_production_runtime`.
"""
path = Path(model_path).expanduser()
if not path.exists() or not path.is_file():
cls._raise(
"MODEL_PROVENANCE_MODEL_FILE_MISSING",
"Configured model file is missing; runtime provenance cannot be verified.",
model_path=str(path),
)
manifest_path = cls.manifest_path_for_model(path)
if not manifest_path.exists() or not manifest_path.is_file():
cls._raise(
"MODEL_PROVENANCE_MANIFEST_MISSING",
"Configured model requires an immutable .geointel-model.json sidecar before inference.",
model_path=str(path),
manifest_path=str(manifest_path),
)
try:
raw_manifest = manifest_path.read_bytes()
payload = json.loads(raw_manifest.decode("utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
"Configured model sidecar must be a readable UTF-8 JSON object.",
manifest_path=str(manifest_path),
error_type=type(exc).__name__,
)
if not isinstance(payload, dict):
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
"Configured model sidecar must contain a JSON object.",
manifest_path=str(manifest_path),
)
cls._require_exact_text(
payload.get("schema_version"),
cls.MANIFEST_SCHEMA_VERSION,
field="schema_version",
manifest_path=manifest_path,
)
contract = cls._require_mapping(payload, "data_contract", manifest_path)
contract_key = cls._require_text(contract, "key", manifest_path)
contract_version = cls._require_text(contract, "version", manifest_path)
# A valid manifest for a different artifact family must never make a
# local PyTorch/SAM weight executable. The structural validator below
# has a registry lookup too, but pinning the identity here keeps this
# runtime gate fail-closed if more model contracts are introduced.
cls._require_exact_text(
contract_key,
PYTORCH_MODEL_CONTRACT_KEY,
field="data_contract.key",
manifest_path=manifest_path,
)
cls._require_exact_text(
contract_version,
PYTORCH_MODEL_CONTRACT_VERSION,
field="data_contract.version",
manifest_path=manifest_path,
)
model = cls._require_mapping(payload, "model", manifest_path)
declared_model_id = cls._require_text(model, "model_id", manifest_path)
declared_task_type = cls._require_text(model, "task_type", manifest_path)
cls._require_exact_text(declared_model_id, model_id, field="model.model_id", manifest_path=manifest_path)
cls._require_exact_text(declared_task_type, task_type, field="model.task_type", manifest_path=manifest_path)
declared_checksum = cls._require_checksum(model.get("sha256"), "model.sha256", manifest_path)
model_checksum = cls._file_sha256(path)
if declared_checksum != model_checksum:
cls._raise(
"MODEL_PROVENANCE_MODEL_CHECKSUM_MISMATCH",
"Model bytes do not match the checksum bound by the runtime sidecar.",
model_path=str(path),
expected=declared_checksum,
observed=model_checksum,
)
model_format = cls._require_text(model, "model_format", manifest_path)
framework = cls._require_text(model, "framework", manifest_path)
class_mapping = model.get("class_mapping")
if not isinstance(class_mapping, (dict, list, tuple)) or not class_mapping:
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
"model.class_mapping must be a non-empty mapping or sequence.",
manifest_path=str(manifest_path),
)
normalized_framework = framework.strip().lower()
if allowed_frameworks and normalized_framework not in {value.strip().lower() for value in allowed_frameworks}:
cls._raise(
"MODEL_PROVENANCE_FRAMEWORK_MISMATCH",
"Model framework does not match the configured runtime adapter.",
manifest_path=str(manifest_path),
expected=sorted({value.strip().lower() for value in allowed_frameworks}),
observed=framework,
)
source_version = cls._require_text(model, "source_version", manifest_path)
if expected_model_version and source_version != expected_model_version:
cls._raise(
"MODEL_PROVENANCE_VERSION_MISMATCH",
"Model sidecar version does not match the configured model version.",
manifest_path=str(manifest_path),
expected=expected_model_version,
observed=source_version,
)
source = cls._require_mapping(payload, "source", manifest_path)
source_registry_id = cls._require_uuid(source.get("source_registry_id"), "source.source_registry_id", manifest_path)
source_snapshot_id = cls._require_uuid(source.get("source_snapshot_id"), "source.source_snapshot_id", manifest_path)
cls._require_exact_text(
cls._require_text(source, "source_registry_key", manifest_path),
cls.SOURCE_REGISTRY_KEY,
field="source.source_registry_key",
manifest_path=manifest_path,
)
snapshot_checksum = cls._require_checksum(
source.get("source_snapshot_checksum_sha256"),
"source.source_snapshot_checksum_sha256",
manifest_path,
)
if snapshot_checksum != model_checksum:
cls._raise(
"MODEL_PROVENANCE_SNAPSHOT_CHECKSUM_MISMATCH",
"Model source snapshot checksum must bind the exact model bytes.",
manifest_path=str(manifest_path),
expected=model_checksum,
observed=snapshot_checksum,
)
metadata = cls._require_mapping(payload, "metadata", manifest_path)
training_manifest_sha256 = cls._require_checksum(
metadata.get("training_manifest_sha256"),
"metadata.training_manifest_sha256",
manifest_path,
)
declared_runtime_manifest_sha256 = cls._require_checksum(
metadata.get("runtime_manifest_sha256"),
"metadata.runtime_manifest_sha256",
manifest_path,
)
computed_runtime_manifest_sha256 = cls.manifest_self_checksum(payload)
if declared_runtime_manifest_sha256 != computed_runtime_manifest_sha256:
cls._raise(
"MODEL_PROVENANCE_MANIFEST_CHECKSUM_MISMATCH",
"Runtime model sidecar integrity checksum does not match its canonical contents.",
manifest_path=str(manifest_path),
expected=declared_runtime_manifest_sha256,
observed=computed_runtime_manifest_sha256,
)
lineage = cls._lineage_evidence(payload, manifest_path)
imported_at = cls._parse_imported_at(payload.get("imported_at"), manifest_path)
report = validate_registered_asset(
build_model_validation_input(
asset_id=f"{model_id}:{model_checksum}",
model_metadata={
"model_format": model_format,
"framework": framework,
"class_mapping": class_mapping,
},
checksum_sha256=declared_checksum,
computed_checksum_sha256=model_checksum,
source_registry_id=source_registry_id,
source_snapshot_id=source_snapshot_id,
imported_at=imported_at,
metadata={
"training_manifest_sha256": training_manifest_sha256,
"runtime_manifest_sha256": declared_runtime_manifest_sha256,
},
source_version=source_version,
lineage=lineage,
data_contract_key=contract_key,
data_contract_version=contract_version,
)
)
if report.failed:
cls._raise(
"MODEL_PROVENANCE_CONTRACT_FAILED",
"Configured model sidecar failed the exact versioned model data contract.",
manifest_path=str(manifest_path),
data_contract=f"{contract_key}@{contract_version}",
issue_codes=[issue.code for issue in report.issues],
validation_report_sha256=report.report_sha256,
)
return RuntimeModelProvenance(
model_id=model_id,
task_type=task_type,
model_path=str(path.resolve()),
manifest_path=str(manifest_path.resolve()),
model_sha256=model_checksum,
manifest_sha256=sha256(raw_manifest).hexdigest(),
runtime_manifest_sha256=declared_runtime_manifest_sha256,
data_contract_key=contract_key,
data_contract_version=contract_version,
validation_report_sha256=report.report_sha256,
source_registry_id=source_registry_id,
source_snapshot_id=source_snapshot_id,
source_snapshot_checksum_sha256=snapshot_checksum,
source_version=source_version,
)
@classmethod
def validate_for_production_runtime(
cls,
*,
db: Any,
model_path: str | Path,
model_id: str,
task_type: str,
expected_model_version: str | None = None,
allowed_frameworks: tuple[str, ...] = (),
) -> RuntimeModelProvenance:
"""Validate byte-bound model evidence against governed database state.
The sidecar is not itself a source of authority. The model bytes may
enter production inference only when the declared source registry and
immutable source snapshot both exist, belong together, are safe to
consume and bind the same SHA-256 and source version as the sidecar.
This check is intentionally invoked immediately before adapter loading.
"""
if db is None or not callable(getattr(db, "get", None)):
cls._raise(
"MODEL_PROVENANCE_DATABASE_REQUIRED",
"Production model inference requires a database session for source snapshot provenance.",
)
evidence = cls.validate_for_runtime(
model_path=model_path,
model_id=model_id,
task_type=task_type,
expected_model_version=expected_model_version,
allowed_frameworks=allowed_frameworks,
)
cls._assert_database_binding(db, evidence)
return evidence
@classmethod
def _assert_database_binding(cls, db: Any, evidence: RuntimeModelProvenance) -> None:
registry_id = UUID(evidence.source_registry_id)
snapshot_id = UUID(evidence.source_snapshot_id)
source_registry = db.get(SourceRegistry, registry_id)
if not isinstance(source_registry, SourceRegistry):
cls._raise(
"MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND",
"Configured model sidecar refers to a source registry record that does not exist.",
source_registry_id=evidence.source_registry_id,
model_id=evidence.model_id,
)
source_snapshot = db.get(SourceSnapshot, snapshot_id)
if not isinstance(source_snapshot, SourceSnapshot):
cls._raise(
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_NOT_FOUND",
"Configured model sidecar refers to a source snapshot record that does not exist.",
source_snapshot_id=evidence.source_snapshot_id,
model_id=evidence.model_id,
)
if str(source_registry.id) != evidence.source_registry_id or source_registry.source_key != cls.SOURCE_REGISTRY_KEY:
cls._raise(
"MODEL_PROVENANCE_SOURCE_REGISTRY_IDENTITY_MISMATCH",
"Database source registry does not match the immutable model sidecar identity.",
expected_source_registry_id=evidence.source_registry_id,
observed_source_registry_id=str(source_registry.id),
expected_source_key=cls.SOURCE_REGISTRY_KEY,
observed_source_key=source_registry.source_key,
)
if str(source_snapshot.id) != evidence.source_snapshot_id or str(source_snapshot.source_registry_id) != evidence.source_registry_id:
cls._raise(
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_REGISTRY_MISMATCH",
"Model source snapshot does not belong to the declared source registry.",
source_registry_id=evidence.source_registry_id,
source_snapshot_id=evidence.source_snapshot_id,
observed_snapshot_registry_id=str(source_snapshot.source_registry_id),
)
registry_ingest_status = cls._normalise_status(source_registry.ingest_status)
registry_freshness_status = cls._normalise_status(source_registry.freshness_status)
if (
registry_ingest_status not in _SAFE_SOURCE_REGISTRY_INGEST_STATUSES
or registry_freshness_status not in _CONSUMABLE_FRESHNESS
):
cls._raise(
"MODEL_PROVENANCE_SOURCE_REGISTRY_UNSAFE",
"Configured model source registry is not in a safe configured/current state.",
source_registry_id=evidence.source_registry_id,
ingest_status=registry_ingest_status,
freshness_status=registry_freshness_status,
)
snapshot_ingest_status = cls._normalise_status(source_snapshot.ingest_status)
snapshot_freshness_status = cls._normalise_status(source_snapshot.freshness_status)
if (
snapshot_ingest_status != _SAFE_SOURCE_SNAPSHOT_INGEST_STATUS
or snapshot_freshness_status not in _CONSUMABLE_FRESHNESS
):
cls._raise(
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_UNSAFE",
"Configured model source snapshot is not an ingested current immutable artifact.",
source_snapshot_id=evidence.source_snapshot_id,
ingest_status=snapshot_ingest_status,
freshness_status=snapshot_freshness_status,
)
snapshot_checksum = str(source_snapshot.checksum_sha256 or "").strip().lower()
if snapshot_checksum != evidence.source_snapshot_checksum_sha256 or snapshot_checksum != evidence.model_sha256:
cls._raise(
"MODEL_PROVENANCE_DATABASE_SNAPSHOT_CHECKSUM_MISMATCH",
"Database model source snapshot checksum does not bind the exact sidecar and model bytes.",
source_snapshot_id=evidence.source_snapshot_id,
expected_model_sha256=evidence.model_sha256,
expected_sidecar_snapshot_sha256=evidence.source_snapshot_checksum_sha256,
observed_snapshot_sha256=snapshot_checksum,
)
snapshot_version = str(source_snapshot.source_version or "").strip()
if snapshot_version != evidence.source_version:
cls._raise(
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_VERSION_MISMATCH",
"Database model source snapshot version does not match the immutable sidecar model version.",
source_snapshot_id=evidence.source_snapshot_id,
expected_source_version=evidence.source_version,
observed_source_version=snapshot_version or None,
)
# The persisted snapshot state is the primary quarantine signal. The
# relationship check is a defensive second line for an active
# quarantine record that predates or bypassed a status transition.
active_quarantines = getattr(source_snapshot, "quarantines", ())
if any(cls._normalise_status(getattr(item, "status", None)) == "quarantined" for item in active_quarantines):
cls._raise(
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_QUARANTINED",
"Configured model source snapshot has an active quarantine record.",
source_snapshot_id=evidence.source_snapshot_id,
)
@staticmethod
def _normalise_status(value: Any) -> str:
return str(value or "").strip().lower()
@classmethod
def _lineage_evidence(cls, payload: Mapping[str, Any], manifest_path: Path) -> LineageEvidence:
lineage = cls._require_mapping(payload, "lineage", manifest_path)
raw_asset_ids = lineage.get("upstream_asset_ids")
raw_checksums = lineage.get("upstream_checksums_sha256")
raw_transformations = lineage.get("transformations")
if not isinstance(raw_asset_ids, list) or not raw_asset_ids or not all(
isinstance(value, str) and value.strip() for value in raw_asset_ids
):
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
"lineage.upstream_asset_ids must be a non-empty string list.",
manifest_path=str(manifest_path),
)
if not isinstance(raw_checksums, list) or len(raw_checksums) != len(raw_asset_ids):
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
"lineage.upstream_checksums_sha256 must match upstream_asset_ids one-for-one.",
manifest_path=str(manifest_path),
)
upstream_checksums = tuple(
cls._require_checksum(value, f"lineage.upstream_checksums_sha256[{index}]", manifest_path)
for index, value in enumerate(raw_checksums)
)
if not isinstance(raw_transformations, list) or not raw_transformations:
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
"lineage.transformations must contain at least one immutable transformation record.",
manifest_path=str(manifest_path),
)
transformations: list[TransformationEvidence] = []
for index, raw in enumerate(raw_transformations):
if not isinstance(raw, Mapping):
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
"Each lineage transformation must be an object.",
manifest_path=str(manifest_path),
index=index,
)
transformations.append(
TransformationEvidence(
name=cls._require_text(raw, "name", manifest_path, prefix=f"lineage.transformations[{index}]."),
version=cls._require_text(raw, "version", manifest_path, prefix=f"lineage.transformations[{index}]."),
checksum_sha256=cls._require_checksum(
raw.get("checksum_sha256"),
f"lineage.transformations[{index}].checksum_sha256",
manifest_path,
),
)
)
return LineageEvidence(
upstream_asset_ids=tuple(raw_asset_ids),
upstream_checksums_sha256=upstream_checksums,
transformations=tuple(transformations),
)
@staticmethod
def _file_sha256(path: Path) -> str:
digest = sha256()
try:
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
except OSError as exc:
RuntimeModelProvenanceService._raise(
"MODEL_PROVENANCE_MODEL_FILE_UNREADABLE",
"Configured model file could not be read for checksum validation.",
model_path=str(path),
error_type=type(exc).__name__,
)
return digest.hexdigest()
@classmethod
def _parse_imported_at(cls, value: Any, manifest_path: Path) -> datetime:
if not isinstance(value, str) or not value.strip():
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
"imported_at must be a timezone-aware ISO-8601 timestamp.",
manifest_path=str(manifest_path),
)
try:
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
except ValueError:
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
"imported_at must be a timezone-aware ISO-8601 timestamp.",
manifest_path=str(manifest_path),
observed=value,
)
if parsed.tzinfo is None:
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
"imported_at must include a timezone offset.",
manifest_path=str(manifest_path),
observed=value,
)
return parsed.astimezone(timezone.utc)
@classmethod
def _require_mapping(cls, payload: Mapping[str, Any], key: str, manifest_path: Path) -> Mapping[str, Any]:
value = payload.get(key)
if not isinstance(value, Mapping):
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
f"{key} must be a JSON object.",
manifest_path=str(manifest_path),
)
return value
@classmethod
def _require_text(
cls,
payload: Mapping[str, Any],
key: str,
manifest_path: Path,
*,
prefix: str = "",
) -> str:
value = payload.get(key)
if not isinstance(value, str) or not value.strip():
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
f"{prefix}{key} must be a non-empty string.",
manifest_path=str(manifest_path),
)
return value.strip()
@classmethod
def _require_checksum(cls, value: Any, field: str, manifest_path: Path) -> str:
normalized = value.strip().lower() if isinstance(value, str) else ""
if not _SHA256.fullmatch(normalized) or value != normalized:
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
f"{field} must be a lowercase SHA-256 digest.",
manifest_path=str(manifest_path),
)
return normalized
@classmethod
def _require_uuid(cls, value: Any, field: str, manifest_path: Path) -> str:
if not isinstance(value, str) or not value.strip():
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
f"{field} must be a UUID string.",
manifest_path=str(manifest_path),
)
try:
return str(UUID(value))
except (AttributeError, ValueError):
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
f"{field} must be a UUID string.",
manifest_path=str(manifest_path),
observed=value,
)
@classmethod
def _require_exact_text(
cls,
observed: Any,
expected: str,
*,
field: str,
manifest_path: Path,
) -> None:
if not isinstance(observed, str) or observed.strip() != expected:
cls._raise(
"MODEL_PROVENANCE_MANIFEST_INVALID",
f"{field} does not match the configured runtime identity.",
manifest_path=str(manifest_path),
expected=expected,
observed=observed,
)
@staticmethod
def _raise(code: str, message: str, **details: Any) -> None:
raise AppError(code=code, message=message, details=details, status_code=422)
+70 -2
View File
@@ -21,9 +21,11 @@ from app.schemas.segmentation import (
)
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
from app.services.detection_service import DetectionService
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
from app.services.model_registry_service import ModelRegistryService
from app.services.qa_service import QaService
from app.services.quality_service import QualityService
from app.services.runtime_model_provenance_service import RuntimeModelProvenance, RuntimeModelProvenanceService
from app.services.segmentation_adapter import (
FixtureSegmentationAdapter,
SamSegmentationAdapter,
@@ -89,6 +91,18 @@ class SegmentationService:
status_code=400,
)
# Production segmentation must consume only a passed, complete and
# non-quarantined dataset. The fixture segmenter is QA/test-only and
# cannot be classified as production inference.
if model.model_id == "fixture-segmenter":
DatasetConsumptionGate.assert_eligible(
dataset,
purpose="quality_assessment",
fixture_mode=True,
)
elif model.model_id in configured_model_ids and model.configured:
DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
run_parameters = {
"model_id": model.model_id,
"confidence_threshold": confidence_threshold,
@@ -326,6 +340,27 @@ class SegmentationService:
if reference_dataset.dataset_type not in {"vector", "geojson"}:
raise AppError(code="INVALID_DATASET_TYPE", message="Reference dataset must be vector data", status_code=400)
candidate_dataset = db.get(Dataset, run.dataset_id)
if not candidate_dataset:
raise AppError(code="DATASET_NOT_FOUND", message="Segmentation source dataset not found", status_code=404)
run_parameters = run.parameters_json if isinstance(run.parameters_json, dict) else {}
fixture_parameters = run_parameters.get("parameters_json")
fixture_mode = bool(
run.model_name == "fixture-segmenter"
and isinstance(fixture_parameters, dict)
and fixture_parameters.get("fixture_mode") is True
)
DatasetConsumptionGate.assert_eligible(
candidate_dataset,
purpose="quality_assessment",
fixture_mode=fixture_mode,
)
DatasetConsumptionGate.assert_eligible(
reference_dataset,
purpose="reference_validation",
reference_task="building_validation",
)
segmentations = SegmentationService._query_segmentation_rows(
db,
analysis_run_id=analysis_run_id,
@@ -510,11 +545,26 @@ class SegmentationService:
) -> tuple[list[Segmentation], dict[str, Any]]:
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles)
if model_name == settings.sam_model_id:
adapter = sam_adapter_class(settings)
model_path = Path(settings.sam_model_path or "").expanduser()
allowed_frameworks = ("ultralytics/sam", "sam", "ultralytics", "pytorch")
adapter = sam_adapter_class(settings)
else:
adapter = yolo_seg_adapter_class(settings)
model_path = Path(settings.yolo_seg_model_path or "").expanduser()
allowed_frameworks = ("ultralytics/pytorch", "ultralytics", "pytorch")
adapter = yolo_seg_adapter_class(settings)
runtime_model_provenance = RuntimeModelProvenanceService.validate_for_production_runtime(
db=db,
model_path=model_path,
model_id=model_name,
task_type="segmentation",
expected_model_version=model_version,
allowed_frameworks=allowed_frameworks,
)
SegmentationService._attach_runtime_model_provenance(
analysis_run,
job,
runtime_model_provenance,
)
model = adapter.load_model(model_path)
allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)}
@@ -591,6 +641,7 @@ class SegmentationService:
"tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()),
"tile_index": candidate["tile_index"],
"device": settings.yolo_device,
"runtime_model_provenance": runtime_model_provenance.as_dict(),
},
)
db.add(segmentation)
@@ -603,8 +654,25 @@ class SegmentationService:
"suppressed_segmentation_count": len(candidates) - len(filtered_candidates),
"duplicate_iou_threshold": float(settings.segmentation_duplicate_iou_threshold),
"tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()),
"runtime_model_provenance": runtime_model_provenance.as_dict(),
}
@staticmethod
def _attach_runtime_model_provenance(
analysis_run: AnalysisRun,
job: Job,
provenance: RuntimeModelProvenance,
) -> None:
"""Record immutable model evidence with a configured segmentation run."""
evidence = provenance.as_dict()
analysis_parameters = dict(analysis_run.parameters_json or {})
analysis_parameters["runtime_model_provenance"] = evidence
analysis_run.parameters_json = analysis_parameters
job_parameters = dict(job.parameters_json or {})
job_parameters["runtime_model_provenance"] = evidence
job.parameters_json = job_parameters
@staticmethod
def _geodesic_area_m2(geometry: MultiPolygon | Polygon) -> float | None:
try:
File diff suppressed because it is too large Load Diff
+130 -5
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import json
import math
from pathlib import Path
from typing import Any, Iterable
from uuid import UUID
@@ -8,6 +9,7 @@ from uuid import UUID
from geoalchemy2.functions import ST_Intersects, ST_MakeEnvelope
from geoalchemy2.shape import from_shape
from geoalchemy2.shape import to_shape
from pyproj import CRS, Transformer
from shapely.geometry import box, mapping, shape
from shapely.ops import transform as transform_geometry
from shapely.validation import make_valid
@@ -268,7 +270,14 @@ class VectorFeatureService:
return ("municipality", municipality) if municipality else None
@staticmethod
def _feature_row(dataset_id: UUID, feature: dict[str, Any], index: int, feature_class: str | None) -> VectorFeature | None:
def _feature_row(
dataset_id: UUID,
feature: dict[str, Any],
index: int,
feature_class: str | None,
*,
source_crs: str = "EPSG:4326",
) -> VectorFeature | None:
geometry_payload = feature.get("geometry")
if geometry_payload is None:
return None
@@ -282,8 +291,7 @@ class VectorFeatureService:
geometry = make_valid(geometry)
if geometry.is_empty or not geometry.is_valid:
raise AppError(code="INVALID_GEOMETRY", message=f"Invalid feature geometry at index {index}", status_code=400)
if geometry.has_z:
geometry = transform_geometry(lambda x, y, z=None: (x, y), geometry)
geometry = VectorFeatureService._canonical_geometry(geometry, source_crs=source_crs, index=index)
properties = feature.get("properties") if isinstance(feature.get("properties"), dict) else {}
source_feature_id = feature.get("id")
@@ -298,6 +306,109 @@ class VectorFeatureService:
geometry=from_shape(geometry, srid=4326),
)
@staticmethod
def _canonical_geometry(geometry: Any, *, source_crs: str, index: int):
"""Transform one source geometry to canonical EPSG:4326 safely."""
if geometry.has_z:
geometry = transform_geometry(lambda x, y, z=None: (x, y), geometry)
# VectorFeature is deliberately canonical WGS84 storage. Treating
# Lambert or another source CRS as EPSG:4326 produces geometries that
# look syntactically valid but are spatially wrong. All governed
# import callers therefore pass the declared source CRS; the default
# only preserves compatibility for legacy, already-WGS84 call sites.
try:
parsed_source_crs = CRS.from_user_input(source_crs)
target_crs = CRS.from_epsg(4326)
except Exception as exc:
raise AppError(
code="INVALID_DATASET_CRS",
message=f"Invalid source CRS for vector feature at index {index}",
details={"source_crs": source_crs},
status_code=400,
) from exc
if not parsed_source_crs.equals(target_crs):
try:
transformer = Transformer.from_crs(parsed_source_crs, target_crs, always_xy=True)
geometry = transform_geometry(transformer.transform, geometry)
except Exception as exc:
raise AppError(
code="VECTOR_CRS_TRANSFORMATION_FAILED",
message=f"Could not transform vector feature at index {index} to EPSG:4326",
details={"source_crs": source_crs},
status_code=400,
) from exc
if geometry.is_empty or not geometry.is_valid:
geometry = make_valid(geometry)
if geometry.is_empty or not geometry.is_valid:
raise AppError(
code="INVALID_GEOMETRY",
message=f"Invalid transformed feature geometry at index {index}",
status_code=400,
)
min_x, min_y, max_x, max_y = geometry.bounds
if (
not all(math.isfinite(value) for value in (min_x, min_y, max_x, max_y))
or min_x < -180
or max_x > 180
or min_y < -90
or max_y > 90
):
raise AppError(
code="VECTOR_GEOMETRY_OUTSIDE_EPSG4326",
message=f"Transformed feature geometry at index {index} is outside EPSG:4326 bounds",
details={"source_crs": source_crs, "bounds": [min_x, min_y, max_x, max_y]},
status_code=400,
)
return geometry
@staticmethod
def canonicalize_geojson_payload(payload: dict[str, Any], *, source_crs: str) -> dict[str, Any]:
"""Return a canonical-WGS84 feature collection without losing source attributes.
Callers use this payload for validation, spatial indexing and
map-safe consumption storage. A non-canonical source file, when
retained, belongs to explicit provenance evidence rather than the
Dataset consumption path; no implicit CRS assumption is recorded.
"""
features = payload.get("features")
if payload.get("type") != "FeatureCollection" or not isinstance(features, list):
raise AppError(code="INVALID_GEOJSON", message="GeoJSON payload must be a FeatureCollection", status_code=400)
canonical_features: list[dict[str, Any]] = []
for index, feature in enumerate(features):
if not isinstance(feature, dict):
raise AppError(code="INVALID_GEOJSON", message=f"Feature {index} must be an object", status_code=400)
canonical_feature = dict(feature)
geometry_payload = feature.get("geometry")
if geometry_payload is not None:
try:
geometry = shape(geometry_payload)
except Exception as exc:
raise AppError(
code="INVALID_GEOJSON",
message=f"Invalid feature geometry at index {index}",
status_code=400,
) from exc
if not geometry.is_empty:
if not geometry.is_valid:
geometry = make_valid(geometry)
if geometry.is_empty or not geometry.is_valid:
raise AppError(
code="INVALID_GEOMETRY",
message=f"Invalid feature geometry at index {index}",
status_code=400,
)
canonical_feature["geometry"] = mapping(
VectorFeatureService._canonical_geometry(geometry, source_crs=source_crs, index=index)
)
canonical_features.append(canonical_feature)
return {
**{key: value for key, value in payload.items() if key not in {"crs", "features"}},
"type": "FeatureCollection",
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
"features": canonical_features,
}
@staticmethod
def _normalize_selection_bbox(bbox: dict[str, Any]) -> dict[str, float | str]:
try:
@@ -882,6 +993,7 @@ class VectorFeatureService:
feature_class: str | None = None,
*,
commit: bool = True,
source_crs: str = "EPSG:4326",
) -> list[VectorFeature]:
features = payload.get("features")
if payload.get("type") != "FeatureCollection" or not isinstance(features, list):
@@ -891,7 +1003,13 @@ class VectorFeatureService:
for index, feature in enumerate(features):
if not isinstance(feature, dict):
raise AppError(code="INVALID_GEOJSON", message=f"Feature {index} must be an object", status_code=400)
row = VectorFeatureService._feature_row(dataset_id, feature, index, feature_class)
row = VectorFeatureService._feature_row(
dataset_id,
feature,
index,
feature_class,
source_crs=source_crs,
)
if row is None:
continue
db.add(row)
@@ -910,6 +1028,7 @@ class VectorFeatureService:
feature_class: str | None = None,
*,
batch_size: int = 1000,
source_crs: str = "EPSG:4326",
) -> int:
if batch_size <= 0:
raise ValueError("batch_size must be positive")
@@ -942,7 +1061,13 @@ class VectorFeatureService:
message=f"Feature {index} in {path.name} must be an object",
status_code=400,
)
row = VectorFeatureService._feature_row(dataset_id, feature, index, feature_class)
row = VectorFeatureService._feature_row(
dataset_id,
feature,
index,
feature_class,
source_crs=source_crs,
)
if row is None:
continue
if row.source_feature_id:
+128 -25
View File
@@ -3,13 +3,17 @@ from __future__ import annotations
import json
import uuid
from datetime import datetime, timezone
from hashlib import sha256
from pathlib import Path
import re
from typing import Any
from geoalchemy2.shape import to_shape
from pyproj import CRS, Transformer
from shapely.geometry import GeometryCollection, MultiPolygon, shape
from shapely.geometry.base import BaseGeometry
from shapely.geometry import mapping
from shapely.ops import transform as shapely_transform
from shapely.ops import unary_union
from shapely.validation import make_valid
from sqlalchemy.orm import Session
@@ -18,12 +22,16 @@ from app.core.errors import AppError
from app.models import Area, Dataset, DatasetVersion
from app.schemas.dataset import DatasetCreateResponse
from app.schemas.operations import VectorOperationResult
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
from app.services.geojson_service import parse_geojson_payload
from app.services.storage_service import StorageService
from app.services.vector_feature_service import VectorFeatureService
class VectorOperationsService:
CANONICAL_VECTOR_CRS = "EPSG:4326"
_CHECKSUM_SHA256 = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE)
@staticmethod
def _require_vector_dataset(dataset: Dataset) -> None:
if dataset.dataset_type not in {"vector", "geojson"}:
@@ -37,7 +45,8 @@ class VectorOperationsService:
if not path.exists():
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
try:
payload = json.loads(path.read_text(encoding="utf-8"))
stored_bytes = path.read_bytes()
payload = json.loads(stored_bytes.decode("utf-8"))
except Exception as exc:
raise AppError(code="INVALID_GEOJSON", message="Stored dataset is not valid JSON", status_code=400) from exc
@@ -47,6 +56,55 @@ class VectorOperationsService:
features = payload.get("features")
if not isinstance(features, list):
raise AppError(code="INVALID_GEOJSON", message="Dataset payload is missing features", status_code=400)
# The normal Dataset storage path is a consumption artifact, not a
# provenance source archive. Refuse projected/original source bytes
# here rather than letting a spatial operation interpret them as
# canonical map coordinates.
raw_crs = payload.get("crs")
if isinstance(raw_crs, dict):
crs_properties = raw_crs.get("properties")
raw_crs = crs_properties.get("name") if isinstance(crs_properties, dict) else None
stored_crs = str(raw_crs or VectorOperationsService.CANONICAL_VECTOR_CRS).strip().upper()
dataset_crs = str(dataset.crs or "").strip().upper()
if stored_crs != VectorOperationsService.CANONICAL_VECTOR_CRS or (
dataset_crs and dataset_crs != VectorOperationsService.CANONICAL_VECTOR_CRS
):
raise AppError(
code="DATASET_STORAGE_CRS_MISMATCH",
message="Vector operations require canonical EPSG:4326 dataset storage.",
details={
"stored_crs": raw_crs or VectorOperationsService.CANONICAL_VECTOR_CRS,
"dataset_crs": dataset.crs,
"expected_crs": VectorOperationsService.CANONICAL_VECTOR_CRS,
},
status_code=409,
)
expected_checksum = str(dataset.checksum_sha256 or "").strip().lower()
actual_checksum = sha256(stored_bytes).hexdigest()
governed_artifact = bool(
getattr(dataset, "data_contract_key", None)
or (
isinstance(getattr(dataset, "metadata_json", None), dict)
and dataset.metadata_json.get("canonical_storage_crs")
)
)
if expected_checksum and VectorOperationsService._CHECKSUM_SHA256.fullmatch(expected_checksum):
if expected_checksum != actual_checksum:
raise AppError(
code="DATASET_STORAGE_CHECKSUM_MISMATCH",
message="Vector dataset storage no longer matches its validated checksum.",
details={"expected_checksum_sha256": expected_checksum, "actual_checksum_sha256": actual_checksum},
status_code=409,
)
elif governed_artifact:
raise AppError(
code="DATASET_STORAGE_CHECKSUM_UNVERIFIABLE",
message="Governed vector storage requires a valid SHA-256 checksum before use.",
details={"checksum_sha256": dataset.checksum_sha256},
status_code=409,
)
return payload, [feature for feature in features if isinstance(feature, dict)]
@staticmethod
@@ -73,6 +131,25 @@ class VectorOperationsService:
raise AppError(code="EMPTY_VECTOR_DATASET", message="Dataset has no parseable geometries", status_code=422)
return geometries
@staticmethod
def _buffer_in_metres(geometry: BaseGeometry, distance_m: float, source_crs: str) -> BaseGeometry:
"""Buffer in a Belgian projected CRS, never in angular degrees."""
try:
input_crs = CRS.from_user_input(source_crs)
metric_crs = CRS.from_epsg(31370)
if input_crs == metric_crs:
return geometry.buffer(distance_m)
forward = Transformer.from_crs(input_crs, metric_crs, always_xy=True)
backward = Transformer.from_crs(metric_crs, input_crs, always_xy=True)
return shapely_transform(backward.transform, shapely_transform(forward.transform, geometry).buffer(distance_m))
except Exception as exc:
raise AppError(
code="INVALID_CRS",
message="A valid explicit CRS is required for metre-based vector buffering.",
status_code=400,
) from exc
@staticmethod
def inspect(db: Session, dataset_id: uuid.UUID) -> VectorOperationResult:
dataset = db.get(Dataset, dataset_id)
@@ -94,7 +171,7 @@ class VectorOperationsService:
feature_count=len(geometries),
geometry_type_summary=geometry_type_summary,
bounds_json={"min_x": float(bounds[0]), "min_y": float(bounds[1]), "max_x": float(bounds[2]), "max_y": float(bounds[3])},
crs=payload.get("crs") if isinstance(payload.get("crs"), str) else dataset.crs,
crs=VectorOperationsService.CANONICAL_VECTOR_CRS,
)
@staticmethod
@@ -184,9 +261,13 @@ class VectorOperationsService:
if distance_m <= 0:
raise AppError(code="INVALID_PARAMETERS", message="distance_m must be greater than 0", status_code=400)
_, features = VectorOperationsService._load_dataset_payload(source_dataset)
payload, features = VectorOperationsService._load_dataset_payload(source_dataset)
source_crs = VectorOperationsService.CANONICAL_VECTOR_CRS
geometries = VectorOperationsService._extract_geometries(features)
buffered_features = [(feature, geometry.buffer(distance_m)) for feature, geometry in geometries]
buffered_features = [
(feature, VectorOperationsService._buffer_in_metres(geometry, distance_m, source_crs))
for feature, geometry in geometries
]
output_features: list[dict[str, Any]] = []
for feature, geometry in buffered_features:
@@ -431,7 +512,14 @@ class VectorOperationsService:
if not output_name_value.strip():
output_name_value = f"{default_name}.geojson"
stored = json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
# All current governed vector storage is EPSG:4326. A legacy source
# with another CRS is not relabelled here: the derived contract will
# quarantine the result instead of placing non-WGS84 coordinates on
# the map as if they were WGS84.
output_crs = VectorOperationsService.CANONICAL_VECTOR_CRS
output_feature_collection = dict(feature_collection)
output_feature_collection["crs"] = output_crs
stored = json.dumps(output_feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
storage_info = StorageService.persist_dataset_file(
project_id=str(source_dataset.project_id),
dataset_id=str(derived_id),
@@ -441,7 +529,7 @@ class VectorOperationsService:
content_type="application/geo+json",
)
metadata = parse_geojson_payload(json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")))
metadata = parse_geojson_payload(json.dumps(output_feature_collection, ensure_ascii=False, separators=(",", ":")))
if metadata_extra:
metadata.update(metadata_extra)
derived_dataset = Dataset(
@@ -452,7 +540,7 @@ class VectorOperationsService:
dataset_type="vector",
source=f"operation:{operation}",
dataset_role=dataset_role,
source_name=source_name,
source_name=source_name or "derived",
source_metadata=source_metadata,
provenance_metadata=provenance_metadata,
imported_at=datetime.now(timezone.utc),
@@ -478,29 +566,44 @@ class VectorOperationsService:
resolution_json=metadata.get("resolution_json"),
bands_json=metadata.get("bands_json"),
metadata_json=metadata,
status="ready",
status="validating",
)
db.add(derived_dataset)
db.add(
DatasetVersion(
dataset_id=derived_dataset.id,
version=1,
storage_path=derived_dataset.storage_path,
source_version=derived_dataset.source_version,
observed_at=derived_dataset.observed_at,
valid_from=derived_dataset.valid_from,
valid_to=derived_dataset.valid_to,
checksum_sha256=derived_dataset.checksum_sha256,
source_metadata=derived_dataset.source_metadata,
provenance_metadata=derived_dataset.provenance_metadata,
)
dataset_version = DatasetVersion(
dataset_id=derived_dataset.id,
version=1,
storage_path=derived_dataset.storage_path,
source_version=derived_dataset.source_version,
observed_at=derived_dataset.observed_at,
valid_from=derived_dataset.valid_from,
valid_to=derived_dataset.valid_to,
checksum_sha256=derived_dataset.checksum_sha256,
source_metadata=derived_dataset.source_metadata,
provenance_metadata=derived_dataset.provenance_metadata,
)
db.commit()
db.refresh(derived_dataset)
if persist_vector_features:
db.add(dataset_version)
derived_source_key = "map_selection" if source_name == "map_selection" else "derived"
is_ready = DerivedDatasetGovernanceService.govern_vector(
db,
dataset=derived_dataset,
dataset_version=dataset_version,
feature_collection=output_feature_collection,
source_key=derived_source_key,
operation=f"vector.{operation}",
parent_dataset=source_dataset,
operation_parameters={
"operation": operation,
"output_name": output_name_value,
**(metadata_extra or {}),
},
)
if persist_vector_features and is_ready:
VectorFeatureService.persist_geojson_features(
db=db,
dataset_id=derived_dataset.id,
payload=feature_collection,
payload=output_feature_collection,
commit=False,
)
db.commit()
db.refresh(derived_dataset)
return derived_id
@@ -9,6 +9,7 @@ from app.core.config import Settings, get_settings
from app.core.errors import AppError
from app.services.detection_service import DetectionService
from app.services.model_asset_catalog_service import ModelAssetCatalogService
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
from app.services.yolo_adapter import YoloDetectionAdapter
@@ -41,6 +42,8 @@ class YoloPreflightService:
"accelerator_ready": None,
"model_path_set": None,
"model_file_exists": None,
"model_provenance_manifest_path": None,
"model_provenance_valid": None,
"model_load_requested": check_model_load,
"model_load_ok": None,
"manifest_path_set": None,
@@ -98,6 +101,29 @@ class YoloPreflightService:
result["message"] = "YOLO_MODEL_PATH does not point to an existing local model file."
return result
result["checks"]["model_provenance_manifest_path"] = str(
RuntimeModelProvenanceService.manifest_path_for_model(model_path)
)
try:
RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id=resolved_settings.yolo_model_id,
task_type="object_detection",
expected_model_version=resolved_settings.yolo_model_version,
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
)
except AppError as exc:
result["checks"]["model_provenance_valid"] = False
result["status"] = "contract_incomplete"
result["message"] = (
"Configured YOLO weights are not runnable until their immutable runtime provenance sidecar validates: "
f"{exc.message}"
)
result["error_code"] = exc.code
result["details"] = exc.details
return result
result["checks"]["model_provenance_valid"] = True
if check_model_load:
try:
yolo_adapter_class(resolved_settings).load_model(model_path)
+19
View File
@@ -0,0 +1,19 @@
"""Canonical backend-test import boundary.
Pytest is intentionally runnable from ``backend/`` because that is the CI
entrypoint. Some contract tests exercise repository-level deterministic
scripts; put the canonical repository root ahead of the legacy
``backend/scripts`` helper directory so those imports resolve to the code that
is actually shipped by the root Docker build.
"""
from __future__ import annotations
import sys
from pathlib import Path
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
repository_root_text = str(REPOSITORY_ROOT)
if repository_root_text not in sys.path:
sys.path.insert(0, repository_root_text)
@@ -0,0 +1,34 @@
from __future__ import annotations
import importlib.util
from pathlib import Path
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "run_accuracy_phase2_foundation_audit.py"
SPEC = importlib.util.spec_from_file_location("accuracy_phase2_foundation_audit", SCRIPT)
assert SPEC is not None and SPEC.loader is not None
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
def test_phase2_foundation_audit_enumerates_exact_source_and_contract_policies() -> None:
payload = MODULE.collect()
assert payload["phase"] == "P2"
assert payload["migration_revision"] == "202608010001"
assert payload["source_registry"]["definition_count"] >= 40
assert payload["source_registry"]["required_building_policy"] == {
"grb_primary_building_validation": "primary",
"buildings_register_classification": "authoritative",
"sentinel_2_classification": "contextual",
"dhmv_classification": "authoritative",
"osm_ground_truth_allowed": False,
}
assert {(item["key"], item["version"]) for item in payload["data_contracts"]} == {
("geointel.vector.geojson", "1.0.0"),
("geointel.raster.geotiff", "1.0.0"),
("geointel.label.yolo", "1.0.0"),
("geointel.label.yolo", "1.1.0"),
("geointel.model.pytorch", "1.0.0"),
}
@@ -0,0 +1,772 @@
from __future__ import annotations
from datetime import timedelta
from importlib.util import module_from_spec, spec_from_file_location
from pathlib import Path
from uuid import uuid4
import pytest
from sqlalchemy import CheckConstraint
from app.core.errors import AppError
from app.models import (
Dataset,
DatasetLineageEdge,
DatasetQuarantine,
DatasetVersion,
SourceRegistry,
SourceSnapshot,
)
from app.services.coverage_registry_service import SOURCE_DEFINITIONS
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
from app.services.dataset_service import DatasetService
from app.services.source_registry_service import (
SERVER_OWNED_SOURCE_DEFINITIONS,
SourceRegistryService,
)
class _Query:
def __init__(self, session: "InMemorySession", model: type) -> None:
self.session = session
self.model = model
self.predicates = []
def filter(self, *predicates):
self.predicates.extend(predicates)
return self
def one_or_none(self):
matches = self._matches()
if len(matches) > 1:
raise AssertionError(
f"Expected one {self.model.__name__}, found {len(matches)}"
)
return matches[0] if matches else None
def all(self):
return self._matches()
def _matches(self):
matches = list(self.session.objects.get(self.model, []))
for predicate in self.predicates:
field_name = predicate.left.key
expected = predicate.right.value
operator_name = getattr(predicate.operator, "__name__", "")
if operator_name == "in_op":
matches = [
item for item in matches if getattr(item, field_name) in expected
]
else:
matches = [
item for item in matches if getattr(item, field_name) == expected
]
return matches
class InMemorySession:
def __init__(self, *objects: object) -> None:
self.objects: dict[type, list[object]] = {}
self.added: list[object] = []
self.flushes = 0
for item in objects:
self._store(item)
def query(self, model: type) -> _Query:
return _Query(self, model)
def add(self, item: object) -> None:
if getattr(item, "id", None) is None:
setattr(item, "id", uuid4())
self._store(item)
self.added.append(item)
def flush(self) -> None:
self.flushes += 1
def _store(self, item: object) -> None:
self.objects.setdefault(type(item), []).append(item)
def _registry(source_key: str) -> SourceRegistry:
definition = SERVER_OWNED_SOURCE_DEFINITIONS[source_key]
return SourceRegistry(id=uuid4(), **definition.as_model_values())
def _dataset() -> Dataset:
return Dataset(
id=uuid4(),
project_id=uuid4(),
name="candidate.tif",
dataset_type="raster",
source="governed",
status="ready",
validation_status="not_validated",
provenance_status="incomplete",
lineage_status="incomplete",
quarantine_status="not_quarantined",
)
def test_server_owned_definitions_encode_building_authority_and_non_ground_truth_sources() -> (
None
):
grb = SERVER_OWNED_SOURCE_DEFINITIONS["grb"]
buildings_register = SERVER_OWNED_SOURCE_DEFINITIONS[
"digitaal_vlaanderen_buildings_addresses_register"
]
sentinel = SERVER_OWNED_SOURCE_DEFINITIONS["sentinel_2"]
dhmv = SERVER_OWNED_SOURCE_DEFINITIONS["digitaal_vlaanderen_dhmv"]
osm = SERVER_OWNED_SOURCE_DEFINITIONS["osm"]
assert grb.classification == "authoritative"
assert grb.usage_policy["ground_truth_allowed"] is True
assert grb.usage_policy["validation_authority"]["building_validation"] == "primary"
assert buildings_register.classification == "authoritative"
assert buildings_register.usage_policy["ground_truth_allowed"] is False
assert (
buildings_register.usage_policy["validation_authority"]["building_validation"]
== "corroborative"
)
assert (
buildings_register.usage_policy["validation_authority"][
"building_register_validation"
]
== "primary"
)
assert sentinel.classification == "contextual"
assert dhmv.classification == "authoritative"
assert dhmv.usage_policy["ground_truth_allowed"] is False
assert (
dhmv.usage_policy["validation_authority"]["building_validation"]
== "corroborative"
)
assert (
dhmv.usage_policy["validation_authority"]["elevation_validation"] == "primary"
)
assert osm.classification == "contextual"
assert osm.usage_policy["ground_truth_allowed"] is False
assert osm.usage_policy["automatic_ground_truth"] is False
assert osm.usage_policy["training_allowed"] is False
assert {
"ngi_adminvector",
"rbins_marine_reporting_units",
"rbins_msp_2026",
"grb",
"digitaal_vlaanderen",
"vrbg",
"digitaal_vlaanderen_buildings_addresses_register",
"digitaal_vlaanderen_orthophoto",
"spw_orthophoto",
"urbis_orthophoto",
"digitaal_vlaanderen_dhmv",
"spw_terrain",
"spw_walous_land_cover",
"spw_geoportail",
"spw_picc",
"urbis",
"vmm_flood_hazard",
"vmm_vha_bathymetry_profiles",
"dov_soil_map",
"statbel",
"waterinfo",
"agentschap_landbouw_zeevisserij_agricultural_parcels",
"sentinel_2",
"osm",
"manual",
"fixture",
"map_selection",
"derived",
"training_label",
"model",
"experimental",
"mdk_bcp_bathymetry",
}.issubset(SERVER_OWNED_SOURCE_DEFINITIONS)
for umbrella_key in ("digitaal_vlaanderen", "spw_geoportail"):
definition = SERVER_OWNED_SOURCE_DEFINITIONS[umbrella_key]
assert definition.classification == "authoritative"
assert definition.usage_policy["ground_truth_allowed"] is False
assert definition.usage_policy["automatic_ground_truth"] is False
assert (
SERVER_OWNED_SOURCE_DEFINITIONS["mdk_bcp_bathymetry"].ingest_status
== "not_configured"
)
def test_coverage_and_direct_adapter_source_keys_are_registry_backed() -> None:
coverage_source_keys = {
definition.contract.source_name for definition in SOURCE_DEFINITIONS
}
coverage_materialization_keys = {
source_key
for definition in SOURCE_DEFINITIONS
for source_key in definition.materialized_source_names
}
direct_adapter_source_keys = {
"digitaal_vlaanderen",
"spw_geoportail",
"mdk_bcp_bathymetry",
}
assert (
coverage_source_keys
| coverage_materialization_keys
| direct_adapter_source_keys
<= set(SERVER_OWNED_SOURCE_DEFINITIONS)
)
def test_new_adapter_source_seed_rows_match_server_owned_registry_semantics() -> None:
migration_path = (
Path(__file__).resolve().parents[1]
/ "alembic"
/ "versions"
/ "202608010001_source_registry_provenance.py"
)
spec = spec_from_file_location("phase2_source_registry_migration", migration_path)
assert spec is not None and spec.loader is not None
migration = module_from_spec(spec)
spec.loader.exec_module(migration)
seed_rows = {row["source_key"]: row for row in migration._seed_rows()}
for source_key in ("digitaal_vlaanderen", "spw_geoportail", "mdk_bcp_bathymetry"):
expected = SERVER_OWNED_SOURCE_DEFINITIONS[source_key].as_model_values()
observed = seed_rows[source_key]
for field in (
"source_key",
"display_name",
"classification",
"authority_name",
"authority_scope_json",
"provider_adapter_key",
"source_url",
"default_crs",
"default_units",
"geographic_coverage_json",
"usage_policy_json",
"freshness_status",
"ingest_status",
"known_limitations_json",
):
assert observed[field] == expected[field]
def test_ensure_source_is_idempotent_and_rejects_caller_owned_unknown_sources() -> None:
grb = _registry("grb")
session = InMemorySession(grb)
assert SourceRegistryService.ensure_server_owned_source(session, "GRB") is grb
assert session.added == []
with pytest.raises(AppError) as exc_info:
SourceRegistryService.ensure_server_owned_source(session, "caller_claimed_grb")
assert exc_info.value.code == "SOURCE_REGISTRY_ENTRY_NOT_FOUND"
def test_snapshot_is_checksum_bound_and_idempotent() -> None:
grb = _registry("grb")
session = InMemorySession(grb)
checksum = "a" * 64
snapshot = SourceRegistryService.record_snapshot(
session,
source_key="grb",
snapshot_key="2026-08-01-gbg",
checksum_sha256=checksum,
source_version="2026-08-01",
crs="EPSG:31370",
units="metres",
)
assert snapshot.source_registry_id == grb.id
assert snapshot.checksum_sha256 == checksum
assert snapshot.ingest_status == "ingested"
assert (
SourceRegistryService.record_snapshot(
session,
source_key="grb",
snapshot_key="2026-08-01-gbg",
checksum_sha256=checksum,
)
is snapshot
)
with pytest.raises(AppError) as exc_info:
SourceRegistryService.record_snapshot(
session,
source_key="grb",
snapshot_key="2026-08-01-gbg",
checksum_sha256="b" * 64,
)
assert exc_info.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT"
with pytest.raises(AppError) as version_conflict:
SourceRegistryService.record_snapshot(
session,
source_key="grb",
snapshot_key="2026-08-01-gbg",
checksum_sha256=checksum,
source_version="2026-08-02",
)
assert version_conflict.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT"
with pytest.raises(AppError) as invalid_checksum:
SourceRegistryService.record_snapshot(
session,
source_key="grb",
snapshot_key="bad-checksum",
checksum_sha256="not-a-checksum",
)
assert invalid_checksum.value.code == "SOURCE_SNAPSHOT_CHECKSUM_INVALID"
def test_governed_import_reuses_an_identical_snapshot_without_rewriting_fetched_at() -> None:
"""A second project may bind the same immutable source snapshot safely."""
grb = _registry("grb")
session = InMemorySession(grb)
checksum = "a" * 64
observed_at = None
metadata = {
"dataset_type": "vector",
"bounds_json": {"min_x": 4.0, "min_y": 50.0, "max_x": 4.1, "max_y": 50.1},
}
source_metadata = {"source_url": "https://example.test/grb", "units": "metres"}
# Dataset ingest keys are project-scoped, while a source snapshot is
# globally keyed by immutable source evidence. This represents the same
# source file arriving through two independently resumable imports.
project_one, project_two = uuid4(), uuid4()
assert (
DatasetService._ingest_key(
project_id=project_one,
source_key="grb",
checksum_sha256=checksum,
dataset_type="vector",
dataset_role="source",
area_id=None,
reference_layer_name=None,
source_version="2026-08-01",
)
!= DatasetService._ingest_key(
project_id=project_two,
source_key="grb",
checksum_sha256=checksum,
dataset_type="vector",
dataset_role="source",
area_id=None,
reference_layer_name=None,
source_version="2026-08-01",
)
)
first_source, first_snapshot = DatasetService._record_snapshot(
session,
source_key="grb",
checksum_sha256=checksum,
source_version="2026-08-01",
observed_at=observed_at,
valid_from=None,
valid_to=None,
source_crs="EPSG:31370",
source_metadata=source_metadata,
metadata=metadata,
)
original_fetched_at = first_snapshot.fetched_at
replay_source, replay_snapshot = DatasetService._record_snapshot(
session,
source_key="grb",
checksum_sha256=checksum,
source_version="2026-08-01",
observed_at=observed_at,
valid_from=None,
valid_to=None,
source_crs="EPSG:31370",
source_metadata=source_metadata,
metadata=metadata,
)
assert replay_source is first_source
assert replay_snapshot is first_snapshot
assert replay_snapshot.fetched_at == original_fetched_at
assert session.objects[SourceSnapshot] == [first_snapshot]
# Outside the governed replay path, a contradictory acquisition timestamp
# remains immutable evidence and is still rejected.
with pytest.raises(AppError) as fetched_at_conflict:
SourceRegistryService.record_snapshot(
session,
source_key="grb",
snapshot_key=first_snapshot.snapshot_key,
checksum_sha256=checksum,
fetched_at=original_fetched_at + timedelta(seconds=1),
)
assert fetched_at_conflict.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT"
# Replay mode is narrow: a changed immutable evidence field still fails.
with pytest.raises(AppError) as conflict:
SourceRegistryService.record_snapshot(
session,
source_key="grb",
snapshot_key=first_snapshot.snapshot_key,
checksum_sha256=checksum,
crs="EPSG:4326",
reuse_existing_snapshot=True,
)
assert conflict.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT"
def test_snapshot_schema_requires_a_canonical_sha256() -> None:
constraints = {
constraint.name: str(constraint.sqltext)
for constraint in SourceSnapshot.__table__.constraints
if isinstance(constraint, CheckConstraint)
}
assert SourceSnapshot.__table__.c.checksum_sha256.nullable is False
assert "ck_source_snapshots_checksum_sha256" in constraints
assert (
"lower(checksum_sha256)" in constraints["ck_source_snapshots_checksum_sha256"]
)
def test_complete_provenance_binding_is_required_before_authoritative_validation() -> (
None
):
grb = _registry("grb")
snapshot = SourceSnapshot(
id=uuid4(),
source_registry_id=grb.id,
snapshot_key="governed-grb",
checksum_sha256="c" * 64,
ingest_status="ingested",
)
dataset = _dataset()
SourceRegistryService.bind_dataset_provenance(
dataset,
source=grb,
snapshot=snapshot,
data_contract_key="vector.grb.buildings",
data_contract_version="1.0.0",
validation_status="passed",
provenance_status="complete",
lineage_status="complete",
)
assert SourceRegistryService.is_dataset_eligible_for_authoritative_validation(
dataset,
source=grb,
snapshot=snapshot,
task="building_validation",
)
osm = _registry("osm")
dataset.source_registry_id = osm.id
snapshot.source_registry_id = osm.id
assert not SourceRegistryService.is_dataset_eligible_for_authoritative_validation(
dataset,
source=osm,
snapshot=snapshot,
task="building_validation",
)
def test_lineage_and_quarantine_are_fail_closed_and_observable() -> None:
session = InMemorySession()
parent_id = uuid4()
child_id = uuid4()
edge = SourceRegistryService.record_lineage_edge(
session,
parent_dataset_id=parent_id,
child_dataset_id=child_id,
relation_type="derived_from",
transformation_name="vector_clip",
input_checksum_sha256="d" * 64,
output_checksum_sha256="e" * 64,
)
assert isinstance(edge, DatasetLineageEdge)
assert (
SourceRegistryService.record_lineage_edge(
session,
parent_dataset_id=parent_id,
child_dataset_id=child_id,
relation_type="derived_from",
transformation_name="vector_clip",
input_checksum_sha256="d" * 64,
output_checksum_sha256="e" * 64,
)
is edge
)
with pytest.raises(AppError) as self_reference:
SourceRegistryService.record_lineage_edge(
session,
parent_dataset_id=parent_id,
child_dataset_id=parent_id,
relation_type="derived_from",
transformation_name="vector_clip",
)
assert self_reference.value.code == "DATASET_LINEAGE_SELF_REFERENCE"
grandchild_id = uuid4()
SourceRegistryService.record_lineage_edge(
session,
parent_dataset_id=child_id,
child_dataset_id=grandchild_id,
relation_type="derived_from",
transformation_name="vector_buffer",
)
with pytest.raises(AppError) as cycle:
SourceRegistryService.record_lineage_edge(
session,
parent_dataset_id=grandchild_id,
child_dataset_id=parent_id,
relation_type="derived_from",
transformation_name="vector_union",
)
assert cycle.value.code == "DATASET_LINEAGE_CYCLE_DETECTED"
dataset = _dataset()
record = SourceRegistryService.quarantine_dataset(
session,
dataset=dataset,
stage="vector_ingest",
reason_code="CRS_UNVERIFIED",
details={"observed_crs": None},
)
assert isinstance(record, DatasetQuarantine)
assert dataset.status == "quarantined"
assert dataset.quarantine_status == "quarantined"
assert dataset.validation_status == "failed"
version_parent = _dataset()
version = DatasetVersion(id=uuid4(), dataset_id=version_parent.id, version=1)
version_session = InMemorySession(version_parent, version)
version_record = SourceRegistryService.quarantine_dataset(
version_session,
dataset_version=version,
stage="dataset_version_validation",
reason_code="CHECKSUM_MISMATCH",
)
assert version_record.dataset_id == version_parent.id
assert version_record.dataset_version_id == version.id
assert version_parent.status == "quarantined"
assert version_parent.quarantine_status == "quarantined"
assert version_parent.validation_status == "failed"
assert version_parent.provenance_status == "incomplete"
assert version_parent.lineage_status == "incomplete"
assert version.validation_status == "failed"
assert version.provenance_status == "incomplete"
snapshot = SourceSnapshot(
id=uuid4(),
source_registry_id=uuid4(),
snapshot_key="quarantined-source",
checksum_sha256="f" * 64,
ingest_status="ingested",
)
SourceRegistryService.quarantine_dataset(
session,
source_snapshot=snapshot,
stage="source_snapshot_validation",
reason_code="CHECKSUM_MISMATCH",
)
assert snapshot.ingest_status == "quarantined"
def test_quarantine_propagates_transitively_to_descendant_dataset_and_version_consumption_gates() -> (
None
):
"""A -> B -> C must fail closed when the governing A artifact is rejected."""
source = _registry("grb")
snapshot = SourceSnapshot(
id=uuid4(),
source_registry_id=source.id,
snapshot_key="transitive-quarantine-source",
checksum_sha256="a" * 64,
freshness_status="current",
ingest_status="ingested",
)
def governed_dataset(name: str) -> Dataset:
dataset = _dataset()
dataset.name = name
dataset.source = "grb"
dataset.source_name = "grb"
dataset.dataset_role = "source"
dataset.checksum_sha256 = snapshot.checksum_sha256
dataset.source_registry_id = source.id
dataset.source_snapshot_id = snapshot.id
dataset.data_contract_key = "geointel.raster.geotiff"
dataset.data_contract_version = "1.0.0"
dataset.validation_status = "passed"
dataset.provenance_status = "complete"
dataset.lineage_status = "complete"
dataset.quarantine_status = "not_quarantined"
dataset.status = "ready"
dataset.source_registry = source
dataset.source_snapshot = snapshot
return dataset
parent = governed_dataset("parent.tif")
child = governed_dataset("child.tif")
grandchild = governed_dataset("grandchild.tif")
parent_version = DatasetVersion(
id=uuid4(),
dataset_id=parent.id,
version=1,
validation_status="passed",
provenance_status="complete",
lineage_status="complete",
)
child_version = DatasetVersion(
id=uuid4(),
dataset_id=child.id,
version=1,
validation_status="passed",
provenance_status="complete",
lineage_status="complete",
)
grandchild_version = DatasetVersion(
id=uuid4(),
dataset_id=grandchild.id,
version=1,
validation_status="passed",
provenance_status="complete",
lineage_status="complete",
)
session = InMemorySession(
parent,
child,
grandchild,
parent_version,
child_version,
grandchild_version,
)
SourceRegistryService.record_lineage_edge(
session,
parent_dataset_id=parent.id,
child_dataset_id=child.id,
relation_type="derived_from",
transformation_name="clip",
)
SourceRegistryService.record_lineage_edge(
session,
parent_dataset_id=child.id,
child_dataset_id=grandchild.id,
relation_type="derived_from",
transformation_name="buffer",
)
assert (
DatasetConsumptionGate.evaluate(child, purpose="production_inference").eligible
is True
)
assert (
DatasetConsumptionGate.evaluate(
grandchild, purpose="production_inference"
).eligible
is True
)
SourceRegistryService.quarantine_dataset(
session,
dataset=parent,
stage="contract_validation",
reason_code="CHECKSUM_MISMATCH",
)
for dataset in (parent, child, grandchild):
decision = DatasetConsumptionGate.evaluate(
dataset, purpose="production_inference"
)
assert dataset.status == "quarantined"
assert dataset.quarantine_status == "quarantined"
assert dataset.validation_status == "failed"
assert dataset.provenance_status == "incomplete"
assert dataset.lineage_status == "incomplete"
assert decision.eligible is False
assert "dataset_quarantined" in decision.reasons
for dataset_version in (parent_version, child_version, grandchild_version):
assert dataset_version.validation_status == "failed"
assert dataset_version.provenance_status == "incomplete"
assert dataset_version.lineage_status == "incomplete"
def test_ingest_keys_are_scoped_and_migration_keeps_unknown_legacy_unbound() -> None:
project_id = uuid4()
dataset = _dataset()
dataset.project_id = project_id
dataset.ingest_key = "grb:2026-08-01:gbg:area-sha"
version = DatasetVersion(
id=uuid4(),
dataset_id=dataset.id,
ingest_key=dataset.ingest_key,
validation_status="not_validated",
provenance_status="incomplete",
lineage_status="incomplete",
)
session = InMemorySession(dataset, version)
assert (
SourceRegistryService.find_dataset_by_ingest_key(
session, project_id, dataset.ingest_key
)
is dataset
)
assert (
SourceRegistryService.find_dataset_version_by_ingest_key(
session, dataset.id, dataset.ingest_key
)
is version
)
with pytest.raises(AppError) as invalid_key:
SourceRegistryService.find_dataset_by_ingest_key(session, project_id, " ")
assert invalid_key.value.code == "INGEST_KEY_INVALID"
migration = (
Path(__file__).resolve().parents[1]
/ "alembic"
/ "versions"
/ "202608010001_source_registry_provenance.py"
).read_text(encoding="utf-8")
assert "uuid_generate_v5" not in migration
assert "__unregistered_legacy_source__" in migration
assert "uq_datasets_project_ingest_key" in migration
assert "uq_dataset_versions_dataset_ingest_key" in migration
def test_migration_contains_database_guards_for_snapshot_pairing_contract_lineage_and_quarantine() -> (
None
):
migration = (
Path(__file__).resolve().parents[1]
/ "alembic"
/ "versions"
/ "202608010001_source_registry_provenance.py"
).read_text(encoding="utf-8")
assert "trg_datasets_snapshot_registry_guard" in migration
assert "trg_dataset_versions_snapshot_registry_guard" in migration
assert "trg_source_registry_write_guard" in migration
assert "trg_source_snapshots_evidence_immutable" in migration
assert "geointel_phase2_contract_report_guard" in migration
assert "trg_datasets_contract_report_guard" in migration
assert "trg_dataset_versions_contract_report_guard" in migration
assert "matching complete validation report" in migration
assert "geointel_phase2_lineage_cycle_guard" in migration
assert "trg_dataset_lineage_edges_cycle_guard" in migration
assert "geointel_phase2_lineage_edge_immutable_guard" in migration
assert "trg_dataset_lineage_edges_immutable" in migration
assert "WITH RECURSIVE descendants" in migration
assert "geointel_phase2_quarantine_lineage_descendants" in migration
assert "geointel_phase2_quarantine_state_guard" in migration
assert "trg_dataset_quarantines_state_guard" in migration
assert "accepted dataset artifact and provenance evidence is immutable" in migration
+4 -1
View File
@@ -154,6 +154,7 @@ def test_guest_login_exposes_models_but_rejects_management_and_cross_project_req
other_project = client.get("/api/v1/projects/00000000-0000-0000-0000-000000000999")
detection_models = client.get("/api/v1/detection/models")
segmentation_models = client.get("/api/v1/segmentation/models")
global_source_registry = client.get("/api/v1/source-registry/grb")
cross_project_runs = client.get(
"/api/v1/detection/runs?project_id=00000000-0000-0000-0000-000000000999"
)
@@ -179,6 +180,8 @@ def test_guest_login_exposes_models_but_rejects_management_and_cross_project_req
assert detection_models.status_code == 200
assert detection_models.json()["data"]["models"]
assert segmentation_models.status_code == 200
assert global_source_registry.status_code == 403
assert global_source_registry.json()["error"] == "GUEST_ROUTE_NOT_AVAILABLE"
assert cross_project_runs.status_code == 403
assert cross_project_runs.json()["error"] == "GUEST_PROJECT_SCOPE_REQUIRED"
assert cross_project_coverage.status_code == 403
@@ -186,7 +189,7 @@ def test_guest_login_exposes_models_but_rejects_management_and_cross_project_req
def test_guest_session_tokens_fail_closed_without_a_project_scope(monkeypatch) -> None:
client = auth_client(monkeypatch, guest_access=True)
auth_client(monkeypatch, guest_access=True)
settings = get_settings()
try:
+193 -7
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import importlib.util
import hashlib
import json
import subprocess
import sys
@@ -13,6 +14,94 @@ assert SPEC and SPEC.loader
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
from training_release_manifest import create_training_release_manifest # noqa: E402
def write_fixture_manifest(path: Path) -> None:
policy = "geointel-training-source-eligibility/v1"
def eligible(sample_slug: str) -> dict[str, object]:
return {
"policy_version": policy,
"eligible": True,
"fixture_mode": True,
"raster": {
"eligible": True,
"reasons": [],
"evidence": {
"dataset_id": f"raster:{sample_slug}",
"checksum_sha256": "a" * 64,
"source_registry_id": "fixture-raster",
"source_snapshot_id": "fixture-raster-snapshot",
},
},
"reference": {
"eligible": True,
"reasons": [],
"evidence": {
"dataset_id": f"reference:{sample_slug}",
"checksum_sha256": "b" * 64,
"source_registry_id": "fixture-reference",
"source_snapshot_id": "fixture-reference-snapshot",
},
},
}
path.write_text(
json.dumps(
{
"training_eligibility": {
"policy_version": policy,
"status": "eligible",
"fixture_mode": True,
},
"samples": [
{
"sample_slug": sample_slug,
"split": split,
"raster_dataset_id": f"raster:{sample_slug}",
"reference_dataset_id": f"reference:{sample_slug}",
"training_eligibility": eligible(sample_slug),
}
for sample_slug, split in (("fixture-train", "train"), ("fixture-val", "val"))
],
}
),
encoding="utf-8",
)
(path.parent / "corpus-freeze.json").write_text(
json.dumps(
{
"schema_version": 2,
"manifest_sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
"immutable": True,
"training_eligibility_policy": policy,
"fixture_mode": True,
}
),
encoding="utf-8",
)
def write_fixture_training_release(tmp_path: Path, manifest: Path) -> Path:
dataset_dir = tmp_path / "fixture-dataset"
for split, sample_slug in (("train", "fixture-train"), ("val", "fixture-val")):
image = dataset_dir / "images" / split / f"{sample_slug}.png"
label = dataset_dir / "labels" / split / f"{sample_slug}.txt"
image.parent.mkdir(parents=True, exist_ok=True)
label.parent.mkdir(parents=True, exist_ok=True)
image.write_bytes(split.encode("utf-8"))
label.write_text("0 0.5 0.5 0.2 0.2\n", encoding="utf-8")
yaml_path = dataset_dir / "dataset.yaml"
yaml_path.write_text(
f"path: {dataset_dir}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
encoding="utf-8",
)
create_training_release_manifest(
train_yaml=yaml_path,
corpus_manifest=manifest,
fixture_mode=True,
)
return yaml_path
def test_training_command_is_cuda_deterministic_and_bound_to_frozen_inputs(tmp_path: Path) -> None:
command = MODULE.training_command(
@@ -78,6 +167,7 @@ def test_failed_iteration_builds_train_only_sampling_for_next_checkpoint(tmp_pat
corpus_manifest=tmp_path / "manifest.json",
assessment=tmp_path / "assessment.json",
output_dir=tmp_path / "iteration-001" / "failure-driven-training",
review_audit=tmp_path / "review-audit.json",
)
assert command[1].endswith("build_failure_driven_yolo_sampling.py")
assert command[command.index("--summary") + 1].endswith("train-summary.json")
@@ -103,20 +193,23 @@ def test_dry_run_can_gate_existing_checkpoint_without_training(tmp_path: Path) -
"status": "ok", "low_variance_positive_tile_count": 0,
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
}))
manifest = tmp_path / "manifest.json"
write_fixture_manifest(manifest)
train_yaml = write_fixture_training_release(tmp_path, manifest)
result = subprocess.run(
[
sys.executable, str(SCRIPT),
"--initial-model", str(tmp_path / "candidate.pt"),
"--train-yaml", str(tmp_path / "dataset.yaml"),
"--train-yaml", str(train_yaml),
"--train-summary", str(tmp_path / "train-summary.json"),
"--dataset-audit", str(audit),
"--train-quality-audit", str(quality),
"--calibration-summary", str(tmp_path / "cal.json"),
"--test-summary", str(tmp_path / "test.json"),
"--background-summary", str(tmp_path / "background.json"),
"--corpus-manifest", str(tmp_path / "manifest.json"),
"--corpus-manifest", str(manifest),
"--output-dir", str(tmp_path / "output"),
"--evaluate-initial-model", "--dry-run",
"--evaluate-initial-model", "--fixture-mode", "--dry-run",
], capture_output=True, text=True, check=False,
)
assert result.returncode == 0
@@ -139,6 +232,9 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
"status": "ok", "low_variance_positive_tile_count": 0,
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
}))
manifest = tmp_path / "manifest.json"
write_fixture_manifest(manifest)
train_yaml = write_fixture_training_release(tmp_path, manifest)
result = subprocess.run(
[
sys.executable,
@@ -146,7 +242,7 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
"--initial-model",
str(tmp_path / "base.pt"),
"--train-yaml",
str(tmp_path / "dataset.yaml"),
str(train_yaml),
"--train-summary",
str(tmp_path / "train-summary.json"),
"--dataset-audit",
@@ -160,9 +256,10 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
"--background-summary",
str(tmp_path / "background.json"),
"--corpus-manifest",
str(tmp_path / "manifest.json"),
str(manifest),
"--output-dir",
str(tmp_path / "output"),
"--fixture-mode",
],
capture_output=True,
text=True,
@@ -172,7 +269,45 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
assert "Dataset audit is not eligible for training" in result.stderr
def test_pending_human_review_does_not_block_objective_training() -> None:
def test_loop_rejects_manifest_without_source_eligibility_before_cuda_training(tmp_path: Path) -> None:
audit = tmp_path / "audit.json"
audit.write_text(json.dumps({
"status": "needs_human_review", "failures": [],
"manifest_immutable": True, "spatial_leakage_status": "ok",
}))
quality = tmp_path / "quality.json"
quality.write_text(json.dumps({
"status": "ok", "low_variance_positive_tile_count": 0,
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
}))
manifest = tmp_path / "manifest.json"
manifest.write_text(json.dumps({"samples": [{"sample_slug": "unproven"}]}), encoding="utf-8")
result = subprocess.run(
[
sys.executable, str(SCRIPT),
"--initial-model", str(tmp_path / "candidate.pt"),
"--train-yaml", str(tmp_path / "dataset.yaml"),
"--train-summary", str(tmp_path / "train-summary.json"),
"--dataset-audit", str(audit),
"--train-quality-audit", str(quality),
"--calibration-summary", str(tmp_path / "cal.json"),
"--test-summary", str(tmp_path / "test.json"),
"--background-summary", str(tmp_path / "background.json"),
"--corpus-manifest", str(manifest),
"--output-dir", str(tmp_path / "output"),
"--evaluate-initial-model", "--dry-run",
],
capture_output=True,
text=True,
check=False,
)
assert result.returncode != 0
assert "manifest_training_eligibility_missing" in result.stderr
def test_pending_human_review_blocks_operational_training() -> None:
audit = {
"status": "needs_human_review",
"failures": [],
@@ -185,7 +320,58 @@ def test_pending_human_review_does_not_block_objective_training() -> None:
"status": "ok", "low_variance_positive_tile_count": 0,
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
}
assert MODULE.dataset_audit_failures(audit, quality) == []
failures = MODULE.dataset_audit_failures(audit, quality)
assert "unsupported audit status: needs_human_review" in failures
assert "review_complete_not_true" in failures
assert "accepted_human_review_evidence_missing" in failures
def test_fixture_mode_can_relax_review_only_after_fixture_manifest_gate() -> None:
audit = {
"status": "needs_human_review",
"failures": [],
"manifest_immutable": True,
"spatial_leakage_status": "ok",
"review_complete": False,
}
quality = {
"status": "ok", "low_variance_positive_tile_count": 0,
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
}
assert MODULE.dataset_audit_failures(audit, quality, fixture_mode=True) == []
def test_operational_dataset_audit_must_be_the_one_bound_into_the_release(tmp_path: Path) -> None:
bound = tmp_path / "bound-audit.json"
other = tmp_path / "other-audit.json"
bound.write_text("{}", encoding="utf-8")
other.write_text("{}", encoding="utf-8")
release = {"human_review": {"audit_path": str(bound.resolve())}}
MODULE.assert_dataset_audit_bound_to_release(
release=release,
dataset_audit=bound,
fixture_mode=False,
)
try:
MODULE.assert_dataset_audit_bound_to_release(
release=release,
dataset_audit=other,
fixture_mode=False,
)
except MODULE.TrainingReleaseError as exc:
assert "does not match" in str(exc)
else:
raise AssertionError("unbound dataset audit was accepted")
def test_protected_assessment_feedback_is_terminal_and_cannot_seed_another_yaml() -> None:
assert MODULE.protected_feedback_roles(
{"status": "continue_training_loop", "test": {"aggregate": {}}, "background": None}
) == ["test"]
assert MODULE.protected_feedback_roles(
{"status": "continue_training_loop", "test": None, "background": {"aggregate": {}}}
) == ["background"]
def test_training_audit_still_fails_closed_on_automated_integrity_gates() -> None:
@@ -0,0 +1,449 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from hashlib import sha256
import json
from pathlib import Path
import pytest
from shapely.geometry import box
from app.core.errors import AppError
from app.services.data_contract_validation import (
AttributeRule,
BoundingBox,
ContractKind,
DataAssetValidationInput,
DataContract,
DataContractRegistry,
DataContractValidator,
FreshnessRules,
GeometryRecord,
GeometryRules,
LineageEvidence,
LineageRules,
RasterRules,
RequirementLevel,
Resolution,
ResolutionRules,
TransformationEvidence,
ValidationStatus,
build_default_data_contract_registry,
build_label_validation_input,
build_model_validation_input,
build_raster_ingest_input,
build_vector_ingest_input,
validate_registered_asset,
)
from app.services.data_quarantine_service import AssetUse, DataQuarantineService
FIXTURE_ROOT = Path(__file__).resolve().parents[2] / "tests" / "fixtures" / "data-contracts"
NOW = datetime(2026, 8, 1, 12, 0, tzinfo=timezone.utc)
CHECKSUM_A = "a" * 64
def _fixture_json(name: str) -> tuple[bytes, object]:
raw = (FIXTURE_ROOT / name).read_bytes()
return raw, json.loads(raw)
def _checksum(raw: bytes) -> str:
return sha256(raw).hexdigest()
def _lineage_with_transform() -> LineageEvidence:
return LineageEvidence(
transformations=(
TransformationEvidence(
name="epsg31370-to-epsg4326",
version="1.0.0",
checksum_sha256=CHECKSUM_A,
),
),
)
def _vector_input_from_fixture(name: str, *, source_crs: str = "EPSG:31370", storage_crs: str = "EPSG:4326") -> DataAssetValidationInput:
raw, payload = _fixture_json(name)
assert isinstance(payload, dict)
return build_vector_ingest_input(
asset_id=f"fixture:{name}",
source_crs=source_crs,
storage_crs=storage_crs,
feature_collection=payload,
checksum_sha256=_checksum(raw),
computed_checksum_sha256=_checksum(raw),
content=raw,
source_registry_id="source:digitaal-vlaanderen:grb",
source_snapshot_id="snapshot:grb:2026-07-31",
imported_at=NOW,
metadata={"license": "Open Data Licence", "provider": "Digitaal Vlaanderen"},
observed_at=NOW - timedelta(days=1),
source_version="2026.07.31",
lineage=_lineage_with_transform() if source_crs != storage_crs else LineageEvidence(),
)
def _issue_codes(report) -> set[str]:
return {issue.code for issue in report.issues}
def test_default_vector_contract_accepts_transformed_geojson_with_complete_provenance() -> None:
report = validate_registered_asset(_vector_input_from_fixture("vector-building-valid.geojson"), now=NOW)
assert report.validation_status == ValidationStatus.PASSED
assert report.quarantine_status == "not_quarantined"
assert report.provenance_status == "complete"
assert report.lineage_status == "complete"
persisted = report.persistence_fields()
assert persisted["data_contract_key"] == "geointel.vector.geojson"
assert persisted["data_contract_version"] == "1.0.0"
assert persisted["validation_report_json"]["report_sha256"] == report.report_sha256
def test_default_vector_contract_quarantines_lambert_coordinates_mislabelled_as_epsg4326() -> None:
report = validate_registered_asset(
_vector_input_from_fixture(
"vector-lambert-mislabelled-as-4326.geojson",
source_crs="EPSG:4326",
storage_crs="EPSG:4326",
),
now=NOW,
)
assert report.validation_status == ValidationStatus.FAILED
assert report.quarantine_status == "quarantined"
assert "CRS_COORDINATE_DOMAIN_VIOLATION" in _issue_codes(report)
def test_vector_contract_checks_geometry_attributes_bounds_and_topology_fail_closed() -> None:
contract = DataContract(
key="test.vector.buildings",
version="1.0.0",
kind=ContractKind.VECTOR,
accepted_source_crs=frozenset({"EPSG:4326"}),
canonical_storage_crs="EPSG:4326",
spatial_domain=BoundingBox(2.0, 49.0, 7.0, 52.0),
require_bounds=True,
geometry_rules=GeometryRules(
allowed_geometry_types=frozenset({"Polygon"}),
attribute_rules=(AttributeRule("native_id", accepted_types=("integer",)),),
forbid_shared_area=True,
),
)
raw = b"overlapping-vector"
asset = DataAssetValidationInput(
asset_id="vector:bad",
data_contract_key=contract.key,
data_contract_version=contract.version,
kind=ContractKind.VECTOR,
source_crs="EPSG:4326",
storage_crs="EPSG:4326",
bounds=BoundingBox(4.0, 51.0, 4.1, 51.1),
checksum_sha256=_checksum(raw),
computed_checksum_sha256=_checksum(raw),
content=raw,
geometry_records=(
GeometryRecord(box(4.0, 51.0, 4.05, 51.05), {"native_id": "wrong-type"}),
GeometryRecord(box(4.025, 51.025, 4.075, 51.075), {}),
),
source_registry_id="source:test",
source_snapshot_id="snapshot:test",
imported_at=NOW,
)
report = DataContractValidator.validate(contract, asset, now=NOW)
assert report.validation_status == ValidationStatus.FAILED
assert {"ATTRIBUTE_TYPE_INVALID", "ATTRIBUTE_REQUIRED", "TOPOLOGY_SHARED_AREA"} <= _issue_codes(report)
assert "BOUNDS_GEOMETRY_MISMATCH" in _issue_codes(report)
def test_default_vector_contract_validates_replayable_large_partition_stream_without_materialising_geometry_list() -> None:
"""Regional imports may be large but remain fully schema/domain checked.
The default contract has no source-specific shared-area rule, so the
validator must make its bounds/schema passes over a replayable stream
without accumulating every Shapely geometry in memory. A stricter
source-specific contract can still opt into a bounded topology batch.
"""
class ReplayableRecords:
def __init__(self, count: int) -> None:
self.count = count
self.iterations = 0
def __iter__(self):
self.iterations += 1
for index in range(self.count):
yield GeometryRecord(
box(4.69, 51.09, 4.70, 51.10),
{"partition_feature": index},
)
raw = b"partitioned-vector-stream"
records = ReplayableRecords(12_000)
asset = DataAssetValidationInput(
asset_id="vector:partitioned-stream",
data_contract_key="geointel.vector.geojson",
data_contract_version="1.0.0",
kind=ContractKind.VECTOR,
source_crs="EPSG:4326",
storage_crs="EPSG:4326",
bounds=BoundingBox(4.69, 51.09, 4.70, 51.10),
checksum_sha256=_checksum(raw),
computed_checksum_sha256=_checksum(raw),
content=raw,
metadata={"license": "Open Data"},
geometry_records=records,
source_registry_id="source:grb",
source_snapshot_id="snapshot:grb:partitioned",
imported_at=NOW,
observed_at=NOW,
source_version="2026-08-01",
)
report = validate_registered_asset(asset, now=NOW)
assert report.validation_status == ValidationStatus.PASSED
assert records.iterations >= 2
def test_raster_contract_accepts_explicit_units_and_quarantines_stale_bad_profile() -> None:
raw = b"raster-stage"
valid = build_raster_ingest_input(
asset_id="raster:valid",
source_crs="EPSG:31370",
storage_crs="EPSG:31370",
raster_profile={"width": 512, "height": 512, "band_count": 3, "dtype": ["uint8"]},
bounds=BoundingBox(193_277.5, 205_708.3, 193_777.5, 206_208.3),
resolution=Resolution(0.9765625, 0.9765625, "m"),
checksum_sha256=_checksum(raw),
computed_checksum_sha256=_checksum(raw),
content=raw,
source_registry_id="source:orthophoto",
source_snapshot_id="snapshot:orthophoto:2026.01",
imported_at=NOW,
metadata={"license": "Open Data"},
observed_at=None,
temporal_unknown_reason="latest mosaic has no per-pixel observation date",
source_version=None,
source_version_unknown_reason="provider did not publish an edition",
)
assert validate_registered_asset(valid, now=NOW).validation_status == ValidationStatus.PASSED
strict = DataContract(
key="test.raster.strict",
version="1.0.0",
kind=ContractKind.RASTER,
accepted_source_crs=frozenset({"EPSG:31370"}),
require_bounds=True,
raster_rules=RasterRules(allowed_band_counts=frozenset({3}), allowed_dtypes=frozenset({"uint8"})),
resolution_rules=ResolutionRules(allowed_units=frozenset({"m"}), min_x=0.2, max_x=1.0, min_y=0.2, max_y=1.0),
freshness_rules=FreshnessRules(observed_at=RequirementLevel.REQUIRED, max_age=timedelta(days=30)),
lineage_rules=LineageRules(require_transformation_when_crs_changes=False),
)
invalid = DataAssetValidationInput(
asset_id="raster:bad",
data_contract_key=strict.key,
data_contract_version=strict.version,
kind=ContractKind.RASTER,
source_crs="EPSG:31370",
storage_crs="EPSG:31370",
bounds=BoundingBox(100.0, 100.0, 200.0, 200.0),
checksum_sha256=_checksum(raw),
computed_checksum_sha256=_checksum(raw),
content=raw,
raster_profile={"width": 0, "height": 10, "band_count": 2, "dtype": ["float32"]},
resolution=Resolution(2.0, 0.1, "degree"),
source_registry_id="source:raster",
source_snapshot_id="snapshot:raster",
imported_at=NOW,
observed_at=NOW - timedelta(days=31),
)
report = DataContractValidator.validate(strict, invalid, now=NOW)
assert report.validation_status == ValidationStatus.FAILED
assert {
"RASTER_PROFILE_VALUE_INVALID",
"RASTER_BAND_COUNT_NOT_ALLOWED",
"RASTER_DTYPE_NOT_ALLOWED",
"RESOLUTION_UNIT_NOT_ALLOWED",
"RESOLUTION_OUT_OF_RANGE",
"FRESHNESS_EXCEEDED",
} <= _issue_codes(report)
def test_default_label_and_model_contracts_validate_good_and_bad_fixtures() -> None:
valid_raw, valid_labels = _fixture_json("labels-valid.json")
invalid_raw, invalid_labels = _fixture_json("labels-invalid.json")
assert isinstance(valid_labels, list)
assert isinstance(invalid_labels, list)
lineage = LineageEvidence(upstream_asset_ids=("image:1",), upstream_checksums_sha256=(CHECKSUM_A,))
valid_label = build_label_validation_input(
asset_id="label:valid",
label_records=valid_labels,
checksum_sha256=_checksum(valid_raw),
computed_checksum_sha256=_checksum(valid_raw),
content=valid_raw,
source_registry_id="source:labels",
source_snapshot_id="snapshot:labels:1",
imported_at=NOW,
metadata={
"image_checksum_sha256": CHECKSUM_A,
"class_ontology_version": "buildings-v1",
"source_corpus_manifest_sha256": CHECKSUM_A,
},
temporal_unknown_reason="labels inherit image observation handling",
source_version_unknown_reason="label release is represented by its snapshot",
lineage=lineage,
)
valid_report = validate_registered_asset(valid_label, now=NOW)
assert valid_report.validation_status == ValidationStatus.PASSED
invalid_label = build_label_validation_input(
asset_id="label:invalid",
label_records=invalid_labels,
checksum_sha256=_checksum(invalid_raw),
computed_checksum_sha256=_checksum(invalid_raw),
content=invalid_raw,
source_registry_id="source:labels",
source_snapshot_id="snapshot:labels:1",
imported_at=NOW,
metadata={
"image_checksum_sha256": "not-a-sha256",
"class_ontology_version": "buildings-v1",
"source_corpus_manifest_sha256": CHECKSUM_A,
},
temporal_unknown_reason="labels inherit image observation handling",
source_version_unknown_reason="label release is represented by its snapshot",
lineage=lineage,
)
invalid_report = validate_registered_asset(invalid_label, now=NOW)
assert invalid_report.validation_status == ValidationStatus.FAILED
assert {
"LABEL_CLASS_ID_NOT_ALLOWED",
"LABEL_NORMALIZED_COORDINATE_INVALID",
"METADATA_CHECKSUM_INVALID",
} <= _issue_codes(invalid_report)
pure_background_raw = b""
pure_background_metadata = {
"image_checksum_sha256": CHECKSUM_A,
"class_ontology_version": "buildings-v1",
"source_corpus_manifest_sha256": CHECKSUM_A,
"label_mode": "pure_background",
"sample_slug": "forest-background-aoi",
"split": "train",
"raster_dataset_id": "dataset:raster:1",
"reference_dataset_id": "dataset:reference:1",
"review_decision": "accepted",
"reviewer_id": "reviewer@example.test",
"reviewed_at": "2026-08-01T11:00:00+00:00",
"review_artifact_sha256": CHECKSUM_A,
}
pure_background = build_label_validation_input(
asset_id="label:pure-background",
label_records=(),
label_mode="pure_background",
checksum_sha256=_checksum(pure_background_raw),
computed_checksum_sha256=_checksum(pure_background_raw),
content=pure_background_raw,
source_registry_id="source:labels",
source_snapshot_id="snapshot:labels:1",
imported_at=NOW,
metadata=pure_background_metadata,
temporal_unknown_reason="labels inherit image observation handling",
source_version_unknown_reason="label release is represented by its snapshot",
lineage=LineageEvidence(
upstream_asset_ids=("dataset:raster:1", "dataset:reference:1"),
upstream_checksums_sha256=(CHECKSUM_A, CHECKSUM_A),
),
)
assert validate_registered_asset(pure_background, now=NOW).validation_status == ValidationStatus.PASSED
unmarked_empty = build_label_validation_input(
asset_id="label:unmarked-empty",
label_records=(),
checksum_sha256=_checksum(pure_background_raw),
computed_checksum_sha256=_checksum(pure_background_raw),
content=pure_background_raw,
source_registry_id="source:labels",
source_snapshot_id="snapshot:labels:1",
imported_at=NOW,
metadata={
"image_checksum_sha256": CHECKSUM_A,
"class_ontology_version": "buildings-v1",
"source_corpus_manifest_sha256": CHECKSUM_A,
},
temporal_unknown_reason="labels inherit image observation handling",
source_version_unknown_reason="label release is represented by its snapshot",
lineage=LineageEvidence(
upstream_asset_ids=("dataset:raster:1", "dataset:reference:1"),
upstream_checksums_sha256=(CHECKSUM_A, CHECKSUM_A),
),
)
assert "PURE_BACKGROUND_MODE_REQUIRED" in _issue_codes(validate_registered_asset(unmarked_empty, now=NOW))
model_raw = b"model-asset"
model = build_model_validation_input(
asset_id="model:valid",
model_metadata={"model_format": "pytorch", "framework": "torch", "class_mapping": {"0": "building"}},
checksum_sha256=_checksum(model_raw),
computed_checksum_sha256=_checksum(model_raw),
content=model_raw,
source_registry_id="source:model-registry",
source_snapshot_id="snapshot:model:1",
imported_at=NOW,
source_version="candidate-1",
metadata={"training_manifest_sha256": CHECKSUM_A, "runtime_manifest_sha256": CHECKSUM_A},
lineage=lineage,
)
assert validate_registered_asset(model, now=NOW).validation_status == ValidationStatus.PASSED
def test_unknown_contract_and_quarantine_gate_are_deterministic_and_fail_closed() -> None:
unknown = DataAssetValidationInput(
asset_id="asset:unknown",
data_contract_key="does.not.exist",
data_contract_version="9.9.9",
kind=ContractKind.VECTOR,
)
report = DataContractRegistry().validate(unknown, now=NOW)
assert report.validation_status == ValidationStatus.FAILED
assert report.quarantine_status == "quarantined"
assert _issue_codes(report) == {"DATA_CONTRACT_UNKNOWN"}
first = DataQuarantineService.decide(report)
second = DataQuarantineService.decide(report)
assert first.idempotency_key == second.idempotency_key
assert first.reason_codes == ("DATA_CONTRACT_UNKNOWN",)
with pytest.raises(AppError, match="cannot enter this pipeline") as exc_info:
DataQuarantineService.require_eligible(first, use=AssetUse.PRODUCTION_INFERENCE)
assert exc_info.value.code == "DATASET_QUARANTINED"
assert exc_info.value.details["use"] == "production_inference"
clean_report = validate_registered_asset(_vector_input_from_fixture("vector-building-valid.geojson"), now=NOW)
release_request = DataQuarantineService.decide(clean_report, previous=first)
assert release_request.quarantine_status == "quarantined"
assert release_request.requires_explicit_release is True
assert release_request.reason_codes == ("QUARANTINE_RELEASE_REQUIRES_EXPLICIT_PERSISTENCE",)
def test_registry_requires_exact_contract_version_and_fingerprints_schema() -> None:
registry = build_default_data_contract_registry()
version_mismatch = _vector_input_from_fixture("vector-building-valid.geojson")
mismatched = DataAssetValidationInput(
**{**version_mismatch.__dict__, "data_contract_version": "2.0.0"},
)
report = registry.validate(mismatched, now=NOW)
assert report.validation_status == ValidationStatus.FAILED
assert "DATA_CONTRACT_UNKNOWN" in _issue_codes(report)
contract = registry.resolve("geointel.vector.geojson", "1.0.0")
assert contract is not None
direct_report = DataContractValidator.validate(contract, mismatched, now=NOW)
assert direct_report.validation_status == ValidationStatus.FAILED
assert "DATA_CONTRACT_IDENTITY_MISMATCH" in _issue_codes(direct_report)
@@ -0,0 +1,316 @@
from __future__ import annotations
from types import SimpleNamespace
from uuid import uuid4
import pytest
from shapely.geometry import box
from app.core.errors import AppError
from app.models import Dataset, SourceRegistry, SourceSnapshot
from app.services.coverage_registry_service import CoverageRegistryService, SOURCE_DEFINITIONS
import app.services.dataset_consumption_gate_service as gate_module
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
from app.services.export_service import ExportService
def _governed_dataset(
*,
source_key: str = "grb",
classification: str = "authoritative",
snapshot_freshness_status: str = "current",
) -> Dataset:
source_id = uuid4()
snapshot_id = uuid4()
checksum = "a" * 64
source = SourceRegistry(
id=source_id,
source_key=source_key,
display_name=f"{source_key} test source",
classification=classification,
authority_name="GeoIntel test authority",
authority_scope_json={"scope": "test"},
usage_policy_json={
"ground_truth_allowed": classification == "authoritative",
"validation_authority": {"building_validation": "primary"}
if classification == "authoritative"
else {},
},
)
snapshot = SourceSnapshot(
id=snapshot_id,
source_registry_id=source_id,
snapshot_key="test-snapshot",
checksum_sha256=checksum,
freshness_status=snapshot_freshness_status,
ingest_status="ingested",
)
dataset = Dataset(
id=uuid4(),
project_id=uuid4(),
name="governed.tif",
dataset_type="raster",
source=source_key,
source_name=source_key,
dataset_role="source",
checksum_sha256=checksum,
source_registry_id=source_id,
source_snapshot_id=snapshot_id,
data_contract_key="geointel.raster.geotiff",
data_contract_version="1.0.0",
validation_status="passed",
provenance_status="complete",
lineage_status="not_applicable",
quarantine_status="not_quarantined",
status="ready",
)
dataset.source_registry = source
dataset.source_snapshot = snapshot
return dataset
def test_governed_dataset_passes_production_inference_and_authoritative_coverage() -> None:
dataset = _governed_dataset()
inference = DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
coverage = DatasetConsumptionGate.assert_eligible(dataset, purpose="authoritative_coverage")
assert inference.eligible is True
assert coverage.eligible is True
@pytest.mark.parametrize(
("field", "value", "error_code"),
(
("provenance_status", "incomplete", "DATASET_PROVENANCE_INCOMPLETE"),
("validation_status", "failed", "DATASET_QUARANTINED"),
("quarantine_status", "quarantined", "DATASET_QUARANTINED"),
),
)
def test_explicit_unsafe_states_can_never_be_relaxed(field: str, value: str, error_code: str) -> None:
dataset = _governed_dataset()
setattr(dataset, field, value)
with pytest.raises(AppError) as exc_info:
DatasetConsumptionGate.assert_eligible(
dataset,
purpose="production_inference",
fixture_mode=True,
)
assert exc_info.value.code == error_code
assert field.replace("_status", "") in " ".join(exc_info.value.details["reasons"])
def test_legacy_fixture_can_support_fixture_qa_but_never_authoritative_coverage() -> None:
fixture = Dataset(
id=uuid4(),
project_id=uuid4(),
name="fixture.tif",
dataset_type="raster",
source="fixture",
)
qa = DatasetConsumptionGate.assert_eligible(fixture, purpose="quality_assessment")
with pytest.raises(AppError) as inference_error:
DatasetConsumptionGate.assert_eligible(
fixture,
purpose="production_inference",
fixture_mode=True,
)
with pytest.raises(AppError) as export_error:
DatasetConsumptionGate.assert_eligible(fixture, purpose="export")
with pytest.raises(AppError) as fixture_export_error:
DatasetConsumptionGate.assert_eligible(fixture, purpose="export", fixture_mode=True)
coverage = DatasetConsumptionGate.evaluate(fixture, purpose="authoritative_coverage")
assert qa.fixture_legacy_exception is True
assert inference_error.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert "fixture_qa_only" in inference_error.value.details["reasons"]
assert export_error.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert fixture_export_error.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert "fixture_qa_only" in fixture_export_error.value.details["reasons"]
assert coverage.eligible is False
assert "fixture_not_authoritative_coverage" in coverage.reasons
def test_unprovenanced_persistent_dataset_is_blocked(monkeypatch) -> None:
dataset = Dataset(
id=uuid4(),
project_id=uuid4(),
name="manual.tif",
dataset_type="raster",
source="manual_upload",
)
monkeypatch.setattr(gate_module, "sa_inspect", lambda _dataset: SimpleNamespace(transient=False))
with pytest.raises(AppError) as exc_info:
DatasetConsumptionGate.assert_eligible(
dataset,
purpose="production_inference",
fixture_mode=True,
)
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert "phase2_provenance_missing" in exc_info.value.details["reasons"]
assert "fixture_source_required" in exc_info.value.details["reasons"]
def test_transient_orm_test_double_can_only_bypass_missing_legacy_fields_for_qa() -> None:
transient = Dataset(
id=uuid4(),
project_id=uuid4(),
name="transient-test.tif",
dataset_type="raster",
source="manual_upload",
)
decision = DatasetConsumptionGate.assert_eligible(transient, purpose="quality_assessment")
coverage = DatasetConsumptionGate.evaluate(transient, purpose="authoritative_coverage")
with pytest.raises(AppError) as production_error:
DatasetConsumptionGate.assert_eligible(transient, purpose="production_inference")
assert decision.fixture_legacy_exception is True
assert production_error.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert coverage.eligible is False
assert "phase2_provenance_missing" in coverage.reasons
@pytest.mark.parametrize("purpose", ("production_inference", "derived_processing", "export"))
def test_passed_manual_or_experimental_dataset_cannot_cross_production_boundary(purpose: str) -> None:
"""A syntactically valid manual upload remains experimental, never production-ready."""
manual = _governed_dataset(source_key="manual", classification="experimental")
manual.source = "manual_upload"
with pytest.raises(AppError) as exc_info:
DatasetConsumptionGate.assert_eligible(manual, purpose=purpose) # type: ignore[arg-type]
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert "experimental_source_not_allowed_for_purpose" in exc_info.value.details["reasons"]
def test_reference_validation_requires_authoritative_ground_truth_reference() -> None:
reference = _governed_dataset()
reference.dataset_type = "vector"
reference.dataset_role = "reference"
decision = DatasetConsumptionGate.assert_eligible(
reference,
purpose="reference_validation",
reference_task="building_validation",
)
assert decision.eligible is True
reference.source_registry.classification = "corroborative"
with pytest.raises(AppError) as exc_info:
DatasetConsumptionGate.assert_eligible(
reference,
purpose="reference_validation",
reference_task="building_validation",
)
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert "reference_source_not_authoritative" in exc_info.value.details["reasons"]
def test_pending_regional_building_authority_cannot_become_truth_without_approval() -> None:
reference = _governed_dataset(source_key="spw_picc", classification="authoritative")
reference.dataset_type = "vector"
reference.dataset_role = "reference"
reference.source_registry.authority_scope_json = {"zone": "Wallonia"}
reference.source_registry.usage_policy_json = {
"ground_truth_allowed": True,
"validation_authority": {"building_validation": "regional_primary_pending_contract"},
}
with pytest.raises(AppError) as exc_info:
DatasetConsumptionGate.assert_eligible(
reference,
purpose="reference_validation",
reference_task="building_validation",
)
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert "reference_task_authority_not_approved" in exc_info.value.details["reasons"]
def test_source_snapshot_must_belong_to_the_dataset_source_registry() -> None:
dataset = _governed_dataset()
dataset.source_snapshot.source_registry_id = uuid4()
with pytest.raises(AppError) as exc_info:
DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert "source_snapshot_registry_mismatch" in exc_info.value.details["reasons"]
@pytest.mark.parametrize("freshness_status", ("unknown", "review_required", "due", "stale"))
def test_non_consumable_source_snapshot_freshness_is_blocked_at_production_boundaries(
freshness_status: str,
) -> None:
dataset = _governed_dataset(snapshot_freshness_status=freshness_status)
for purpose in ("production_inference", "authoritative_coverage"):
with pytest.raises(AppError) as exc_info:
DatasetConsumptionGate.assert_eligible(dataset, purpose=purpose) # type: ignore[arg-type]
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
assert "source_snapshot_freshness_not_eligible" in exc_info.value.details["reasons"]
def test_coverage_registry_ignores_explicitly_incomplete_materialization() -> None:
definition = next(item for item in SOURCE_DEFINITIONS if item.contract.source_name == "digitaal_vlaanderen")
unsafe_materialization = SimpleNamespace(
id=uuid4(),
status="ready",
source_name="grb",
validation_status="passed",
provenance_status="incomplete",
lineage_status="complete",
quarantine_status="not_quarantined",
)
matches, fully_covered = CoverageRegistryService._matching_datasets(
[unsafe_materialization],
definition,
"buildings",
"flanders",
box(4.0, 50.8, 4.1, 50.9),
)
assert matches == []
assert fully_covered is False
def test_vector_export_is_fail_closed_before_selection(monkeypatch) -> None:
dataset = _governed_dataset()
dataset.dataset_type = "vector"
dataset.status = "quarantined"
queried = False
class _Session:
@staticmethod
def get(model, item_id):
return dataset if model is Dataset and item_id == dataset.id else None
def _unexpected_selection(*_args, **_kwargs):
nonlocal queried
queried = True
raise AssertionError("unsafe dataset must be rejected before querying vector features")
monkeypatch.setattr(
"app.services.export_service.VectorFeatureService.select_features_by_bbox",
_unexpected_selection,
)
with pytest.raises(AppError) as exc_info:
ExportService.export_vector_selection_geojson(
_Session(),
dataset.id,
{"min_x": 4.0, "min_y": 50.0, "max_x": 4.1, "max_y": 50.1, "crs": "EPSG:4326"},
)
assert exc_info.value.code == "DATASET_QUARANTINED"
assert queried is False
@@ -74,6 +74,8 @@ def test_all_in_one_dockerfile_copies_operator_scripts_for_runtime_use() -> None
"audit_operator_yolo_dataset_quality.py",
"render_operator_yolo_label_qa_contact_sheets.py",
"train_operator_yolo_detector.sh",
"training_dataset_eligibility.py",
"training_release_manifest.py",
"verify_real_data_detection_qa_workflow.sh",
"run_detection_quality_matrix.sh",
"run_multi_sample_detection_quality_matrix.sh",
@@ -3,6 +3,8 @@ from __future__ import annotations
import importlib.util
from pathlib import Path
import pytest
SCRIPT = Path(__file__).parents[2] / "scripts" / "build_failure_driven_yolo_sampling.py"
SPEC = importlib.util.spec_from_file_location("failure_sampling", SCRIPT)
@@ -22,7 +24,7 @@ def test_dataset_validation_source_preserves_manifest_path(tmp_path: Path):
assert MODULE.dataset_validation_source(source) == "/data/source/internal-val.txt"
def test_sampling_repeats_only_failed_region_train_tiles() -> None:
def test_sampling_rejects_protected_test_and_background_feedback() -> None:
manifest = {
"samples": [
{"sample_slug": "train-fl", "split": "train", "region": "flanders"},
@@ -54,15 +56,10 @@ def test_sampling_repeats_only_failed_region_train_tiles() -> None:
},
"background": {"pure_empty_false_positives": 2},
}
paths, metadata = MODULE.build_sampling(
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0
)
assert paths.count(str(Path("/tmp/fl-pos.png").resolve())) == 3
assert paths.count(str(Path("/tmp/fl-neg.png").resolve())) == 4
assert paths.count(str(Path("/tmp/wa-pos.png").resolve())) == 1
assert not any("protected" in path for path in paths)
assert metadata["protected_samples_in_training"] == []
assert metadata["weak_recall_regions"] == ["flanders"]
with pytest.raises(ValueError, match="protected test/background evidence"):
MODULE.build_sampling(
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0
)
def test_sampling_can_use_calibration_before_test_is_opened() -> None:
@@ -171,7 +168,7 @@ def test_sampling_targets_failed_calibration_contexts_without_using_protected_ti
assert metadata["recall_dominant_regions"] == ["flanders"]
def test_recall_dominance_does_not_suppress_negatives_when_background_gate_failed() -> None:
def test_sampling_rejects_background_feedback_after_a_protected_background_opening() -> None:
manifest = {"samples": [
{"sample_slug": "positive", "split": "train", "region": "flanders", "context": "industrial"},
{"sample_slug": "negative", "split": "train", "region": "flanders", "context": "industrial-hard-negative"},
@@ -190,12 +187,10 @@ def test_recall_dominance_does_not_suppress_negatives_when_background_gate_faile
"background": {"pure_empty_false_positives": 1},
}
paths, metadata = MODULE.build_sampling(
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0,
)
assert paths.count(str(Path("/tmp/negative.png").resolve())) == 4
assert metadata["recall_dominant_regions"] == []
with pytest.raises(ValueError, match="protected background evidence"):
MODULE.build_sampling(
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0,
)
def test_region_cap_drops_only_repeats_and_preserves_every_unique_tile() -> None:
@@ -0,0 +1,765 @@
from __future__ import annotations
from datetime import UTC, datetime
from hashlib import sha256
import json
from pathlib import Path
from uuid import UUID, uuid4
import pytest
from pyproj import Transformer
from app.core.errors import AppError
from app.models import (
Area,
Dataset,
DatasetQuarantine,
DatasetVersion,
Project,
SourceRegistry,
SourceSnapshot,
VectorFeature,
)
from app.services.dataset_service import DatasetService, _PartitionedGeoJsonRecords
from app.services.vector_operations_service import VectorOperationsService
class _Query:
def __init__(self, session: "_Session", model: type) -> None:
self.session = session
self.model = model
self.predicates = []
def filter(self, *predicates):
self.predicates.extend(predicates)
return self
def one_or_none(self):
matches = self._matches()
if len(matches) > 1:
raise AssertionError(
f"expected one {self.model.__name__}, found {len(matches)}"
)
return matches[0] if matches else None
def all(self):
return self._matches()
def _matches(self):
matches = list(self.session.rows.get(self.model, []))
for predicate in self.predicates:
field_name = predicate.left.key
expected = predicate.right.value
operator_name = getattr(predicate.operator, "__name__", "")
if operator_name == "in_op":
matches = [
item for item in matches if getattr(item, field_name) in expected
]
else:
matches = [
item for item in matches if getattr(item, field_name) == expected
]
return matches
class _Session:
"""Small ORM-shaped harness that exercises the real governed path."""
def __init__(self, project: Project) -> None:
self.rows: dict[type, list[object]] = {Project: [project]}
self.commits = 0
self.rollbacks = 0
self.flushes = 0
def get(self, model: type, item_id: UUID):
return next(
(
item
for item in self.rows.get(model, [])
if getattr(item, "id", None) == item_id
),
None,
)
def query(self, model: type) -> _Query:
return _Query(self, model)
def add(self, item: object) -> None:
if getattr(item, "id", None) is None:
setattr(item, "id", uuid4())
self.rows.setdefault(type(item), []).append(item)
def flush(self) -> None:
self.flushes += 1
def commit(self) -> None:
self.commits += 1
def rollback(self) -> None:
self.rollbacks += 1
def refresh(self, _item: object) -> None:
return None
def expunge(self, _item: object) -> None:
return None
def _storage_info(tmp_path: Path, content: bytes) -> dict[str, object]:
path = tmp_path / "grb-buildings.geojson"
path.write_bytes(content)
return {
"storage_path": str(path),
"original_filename": path.name,
"stored_filename": path.name,
"content_type": "application/geo+json",
"size_bytes": len(content),
"checksum_sha256": sha256(content).hexdigest(),
}
def _valid_payload() -> bytes:
return json.dumps(
{
"type": "FeatureCollection",
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
"features": [
{
"type": "Feature",
"id": "gbg-1",
"properties": {"id": "gbg-1"},
"geometry": {
"type": "Polygon",
"coordinates": [
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
],
},
}
],
}
).encode("utf-8")
def _grb_payload_without_required_id() -> bytes:
return json.dumps(
{
"type": "FeatureCollection",
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
"features": [
{
"type": "Feature",
"properties": {"unrelated": "not a GRB identity"},
"geometry": {
"type": "Polygon",
"coordinates": [
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
],
},
}
],
}
).encode("utf-8")
def _lambert_grb_payload() -> tuple[bytes, tuple[float, float, float, float]]:
"""Create a valid GRB-shaped source artifact in its declared native CRS."""
longitude, latitude = 4.70, 51.10
max_longitude, max_latitude = 4.7001, 51.1001
to_lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
lambert_ring = [
to_lambert.transform(longitude, latitude),
to_lambert.transform(max_longitude, latitude),
to_lambert.transform(max_longitude, max_latitude),
to_lambert.transform(longitude, latitude),
]
return (
json.dumps(
{
"type": "FeatureCollection",
"crs": {"type": "name", "properties": {"name": "EPSG:31370"}},
"features": [
{
"type": "Feature",
"id": "GBG.lambert.1",
"properties": {"id": "GBG.lambert.1"},
"geometry": {"type": "Polygon", "coordinates": [lambert_ring]},
}
],
}
).encode("utf-8"),
(longitude, latitude, max_longitude, max_latitude),
)
def test_governed_vector_import_persists_snapshot_contract_and_queryable_features(
monkeypatch, tmp_path: Path
) -> None:
project = Project(id=uuid4(), name="Phase 2 governed ingest")
db = _Session(project)
raw = _valid_payload()
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_dataset_file",
lambda **_kwargs: _storage_info(tmp_path, raw),
)
result = DatasetService.import_vector_bytes(
db,
project_id=project.id,
filename="grb-buildings.geojson",
content=raw,
source="grb_wfs",
source_name="grb",
dataset_role="reference",
reference_layer_name="buildings",
source_metadata={
"license": "Open data",
"source_url": "https://example.invalid/grb",
},
provenance_metadata={"adapter": "test"},
temporal_series_key="grb:2026-08",
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
source_version="2026-08-01",
temporal_granularity="snapshot",
)
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
snapshot = db.rows[SourceSnapshot][0]
source = db.rows[SourceRegistry][0]
assert result.status == "ready"
assert dataset.source_name == "grb"
assert dataset.source_registry_id == source.id
assert dataset.source_snapshot_id == snapshot.id
assert dataset.validation_status == "passed"
assert dataset.provenance_status == "complete"
assert dataset.lineage_status == "complete"
assert dataset.quarantine_status == "not_quarantined"
assert dataset.crs == "EPSG:4326"
assert snapshot.checksum_sha256 == sha256(raw).hexdigest()
assert len(db.rows[VectorFeature]) == 1
assert db.commits == 1
# A retry with identical governed evidence is idempotent and does not
# create a second source snapshot, dataset or vector feature.
repeated = DatasetService.import_vector_bytes(
db,
project_id=project.id,
filename="grb-buildings.geojson",
content=raw,
source="grb_wfs",
source_name="grb",
dataset_role="reference",
reference_layer_name="buildings",
source_metadata={"license": "Open data"},
provenance_metadata={"adapter": "test"},
temporal_series_key="grb:2026-08",
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
source_version="2026-08-01",
temporal_granularity="snapshot",
)
assert repeated.id == result.id
assert len(db.rows[Dataset]) == 1
assert len(db.rows[SourceSnapshot]) == 1
assert len(db.rows[VectorFeature]) == 1
def test_governed_lambert_geojson_persists_canonical_consumption_bytes_and_provenance_evidence(
monkeypatch, tmp_path: Path
) -> None:
"""Projected source bytes must never be the file that vector operations consume."""
project = Project(id=uuid4(), name="Canonical GeoJSON storage")
db = _Session(project)
raw, (longitude, latitude, max_longitude, max_latitude) = _lambert_grb_payload()
consumption_path = tmp_path / "consumption" / "grb-buildings.geojson"
provenance_path = tmp_path / "provenance" / "grb-buildings.geojson"
def _persist_dataset_file(**kwargs):
stored = kwargs["content"]
consumption_path.parent.mkdir(parents=True, exist_ok=True)
consumption_path.write_bytes(stored)
return _storage_info(consumption_path.parent, stored)
def _persist_file(storage_path, content, original_filename, content_type):
del storage_path, original_filename, content_type
provenance_path.parent.mkdir(parents=True, exist_ok=True)
provenance_path.write_bytes(content)
return _storage_info(provenance_path.parent, content)
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_dataset_file",
_persist_dataset_file,
)
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_file",
_persist_file,
)
result = DatasetService.import_vector_bytes(
db,
project_id=project.id,
filename="grb-lambert.geojson",
content=raw,
source="grb_wfs",
source_name="grb",
dataset_role="reference",
reference_layer_name="buildings",
source_metadata={"license": "Open data"},
provenance_metadata={"adapter": "test"},
temporal_series_key="grb:lambert:2026-08",
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
source_version="2026-08-01-lambert",
temporal_granularity="snapshot",
)
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
dataset_version = db.rows[DatasetVersion][0]
snapshot = db.rows[SourceSnapshot][0]
canonical_bytes = Path(str(dataset.storage_path)).read_bytes()
canonical_payload = json.loads(canonical_bytes)
source_artifact = dataset.provenance_metadata["source_artifact"]
assert result.status == "ready"
assert dataset.crs == "EPSG:4326"
assert canonical_payload["crs"]["properties"]["name"] == "EPSG:4326"
assert canonical_payload["features"][0]["geometry"]["coordinates"][0][0] == pytest.approx(
[longitude, latitude], abs=0.000001
)
assert sha256(canonical_bytes).hexdigest() == dataset.checksum_sha256
assert dataset_version.checksum_sha256 == dataset.checksum_sha256
assert snapshot.checksum_sha256 == dataset.checksum_sha256
assert source_artifact["retention"] == "provenance_evidence_only"
assert source_artifact["checksum_sha256"] == sha256(raw).hexdigest()
assert source_artifact["storage_path"] != dataset.storage_path
assert Path(source_artifact["storage_path"]).read_bytes() == raw
assert dataset.provenance_metadata["canonical_consumption_artifact"] == {
"checksum_sha256": dataset.checksum_sha256,
"crs": "EPSG:4326",
"storage_role": "dataset_consumption",
}
inspection = VectorOperationsService.inspect(db, dataset.id)
assert inspection.crs == "EPSG:4326"
assert inspection.bounds_json == {
"min_x": pytest.approx(longitude, abs=0.000001),
"min_y": pytest.approx(latitude, abs=0.000001),
"max_x": pytest.approx(max_longitude, abs=0.000001),
"max_y": pytest.approx(max_latitude, abs=0.000001),
}
response_payload = DatasetService.get_dataset_geojson(db, dataset.id)
assert response_payload["features"][0]["geometry"]["coordinates"][0][0] == pytest.approx(
[longitude, latitude], abs=0.000001
)
# The storage identity is enforced at the operation boundary too; a
# replacement with different canonical bytes is not silently processed.
Path(str(dataset.storage_path)).write_bytes(canonical_bytes + b"\n")
with pytest.raises(AppError) as exc_info:
VectorOperationsService.inspect(db, dataset.id)
assert exc_info.value.code == "DATASET_STORAGE_CHECKSUM_MISMATCH"
def test_metadata_refresh_refuses_mutated_governed_artifact(
monkeypatch, tmp_path: Path
) -> None:
"""A passed snapshot cannot be silently re-described from mutable storage."""
project = Project(id=uuid4(), name="Phase 2 immutable refresh")
db = _Session(project)
raw = _valid_payload()
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_dataset_file",
lambda **_kwargs: _storage_info(tmp_path, raw),
)
result = DatasetService.import_vector_bytes(
db,
project_id=project.id,
filename="grb-buildings.geojson",
content=raw,
source="grb_wfs",
source_name="grb",
dataset_role="reference",
reference_layer_name="buildings",
source_metadata={"license": "Open data"},
provenance_metadata={"adapter": "test"},
temporal_series_key="grb:2026-08",
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
source_version="2026-08-01",
temporal_granularity="snapshot",
)
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
original_checksum = dataset.checksum_sha256
original_metadata = dict(dataset.metadata_json or {})
original_commit_count = db.commits
# Simulate an out-of-band storage replacement at the same path. The
# refresh endpoint must not parse it into an already-passed contract row.
Path(str(dataset.storage_path)).write_bytes(_grb_payload_without_required_id())
with pytest.raises(AppError) as exc_info:
DatasetService.refresh_metadata(db, dataset.id)
assert exc_info.value.code == "GOVERNED_DATASET_REINGEST_REQUIRED"
assert dataset.status == "ready"
assert dataset.validation_status == "passed"
assert dataset.checksum_sha256 == original_checksum
assert dataset.metadata_json == original_metadata
assert db.commits == original_commit_count
def test_governed_import_quarantines_bad_artifacts_and_refuses_unknown_source(
monkeypatch, tmp_path: Path
) -> None:
project = Project(id=uuid4(), name="Phase 2 quarantine")
db = _Session(project)
raw = b'{"type":"FeatureCollection","features":[]}'
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_dataset_file",
lambda **_kwargs: _storage_info(tmp_path, raw),
)
quarantined = DatasetService.import_vector_bytes(
db,
project_id=project.id,
filename="empty.geojson",
content=raw,
source="grb_wfs",
source_name="grb",
dataset_role="reference",
reference_layer_name="buildings",
source_metadata={"license": "Open data"},
provenance_metadata={},
temporal_series_key="grb:2026-08-empty",
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
source_version="2026-08-01-empty",
temporal_granularity="snapshot",
)
assert quarantined.status == "quarantined"
assert quarantined.validation_status == "failed"
assert quarantined.quarantine_status == "quarantined"
assert len(db.rows[DatasetQuarantine]) == 1
assert db.rows[SourceSnapshot][0].ingest_status == "quarantined"
with pytest.raises(AppError) as exc_info:
DatasetService.import_vector_bytes(
db,
project_id=project.id,
filename="unregistered.geojson",
content=_valid_payload(),
source="caller_controlled",
source_name="caller_claimed_grb",
dataset_role="reference",
reference_layer_name="buildings",
source_metadata={"license": "Open data"},
provenance_metadata={},
)
assert exc_info.value.code == "SOURCE_REGISTRY_ENTRY_NOT_FOUND"
def test_governed_grb_vector_quarantines_missing_server_owned_required_attribute(
monkeypatch, tmp_path: Path
) -> None:
project = Project(id=uuid4(), name="Phase 2 source schema")
db = _Session(project)
raw = _grb_payload_without_required_id()
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_dataset_file",
lambda **kwargs: _storage_info(tmp_path, kwargs["content"]),
)
quarantined = DatasetService.import_vector_bytes(
db,
project_id=project.id,
filename="grb-missing-id.geojson",
content=raw,
source="grb_wfs",
source_name="grb",
dataset_role="reference",
reference_layer_name="buildings",
source_metadata={"license": "Open data"},
provenance_metadata={"adapter": "test"},
temporal_series_key="grb:missing-id",
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
source_version="2026-08-01-missing-id",
temporal_granularity="snapshot",
)
dataset = next(item for item in db.rows[Dataset] if item.id == quarantined.id)
assert quarantined.status == "quarantined"
assert dataset.validation_status == "failed"
assert dataset.quarantine_status == "quarantined"
issue = dataset.validation_report_json["issues"][0]
assert issue["code"] == "SOURCE_SCHEMA_REQUIRED_ATTRIBUTE_MISSING"
assert issue["category"] == "source_schema"
assert len(db.rows[DatasetQuarantine]) == 1
def test_partitioned_vector_ingest_is_idempotent_and_quarantines_noncanonical_partition_coordinates(
monkeypatch,
tmp_path: Path,
) -> None:
project = Project(id=uuid4(), name="Partitioned governed ingest")
area = Area(id=uuid4(), project_id=project.id, name="Partitioned AOI")
db = _Session(project)
db.rows[Area] = [area]
feature = {
"type": "Feature",
"id": "GBG.1",
"properties": {"id": "GBG.1", "source_feature_id": "GBG.1"},
"geometry": {
"type": "Polygon",
"coordinates": [
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
],
},
}
partition_payload = {"type": "FeatureCollection", "features": [feature]}
partition_path = tmp_path / "partition-01.geojson"
partition_path.write_text(json.dumps(partition_payload), encoding="utf-8")
artifact_payload = {
"type": "FeatureCollection",
"crs": "EPSG:4326",
"features": [feature],
}
artifact_path = tmp_path / "grb-partitioned.geojson"
artifact_raw = json.dumps(artifact_payload).encode("utf-8")
artifact_path.write_bytes(artifact_raw)
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
lambda **_kwargs: _storage_info(tmp_path, artifact_raw),
)
result = DatasetService.import_partitioned_vector_artifact(
db,
project_id=project.id,
area_id=area.id,
artifact_path=artifact_path,
partition_paths=[partition_path],
original_filename="grb-partitioned.geojson",
source="operator_official_import",
dataset_role="reference",
source_name="grb",
reference_layer_name="buildings",
metadata_json={
"feature_count": 1,
"crs": "EPSG:4326",
"bounds_json": {
"min_x": 4.69,
"min_y": 51.09,
"max_x": 4.70,
"max_y": 51.10,
},
},
source_metadata={"license": "Open data"},
provenance_metadata={
"artifact_sha256": sha256(artifact_raw).hexdigest(),
"partition_checksums": {
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
},
},
temporal_series_key="grb:partitioned:test",
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
source_version="2026-08-01",
)
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
assert result.status == "ready"
assert dataset.validation_status == "passed"
assert dataset.provenance_status == "complete"
assert dataset.source_name == "grb"
assert len(db.rows[SourceSnapshot]) == 1
assert len(db.rows[VectorFeature]) == 1
assert dataset.metadata_json["partitioned_geometry_audit"][
"partition_checksums_sha256"
] == {partition_path.name: sha256(partition_path.read_bytes()).hexdigest()}
assert dataset.provenance_metadata["partition_checksum_manifest_sha256"]
assert dataset.provenance_metadata["partitioned_artifact_binding_sha256"]
repeated = DatasetService.import_partitioned_vector_artifact(
db,
project_id=project.id,
area_id=area.id,
artifact_path=artifact_path,
partition_paths=[partition_path],
original_filename="grb-partitioned.geojson",
source="operator_official_import",
dataset_role="reference",
source_name="grb",
reference_layer_name="buildings",
metadata_json={
"feature_count": 1,
"crs": "EPSG:4326",
"bounds_json": {
"min_x": 4.69,
"min_y": 51.09,
"max_x": 4.70,
"max_y": 51.10,
},
},
source_metadata={"license": "Open data"},
provenance_metadata={
"artifact_sha256": sha256(artifact_raw).hexdigest(),
"partition_checksums": {
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
},
},
temporal_series_key="grb:partitioned:test",
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
source_version="2026-08-01",
)
assert repeated.id == result.id
assert len(db.rows[Dataset]) == 1
assert len(db.rows[VectorFeature]) == 1
lambert_feature = {
**feature,
"id": "GBG.lambert",
"properties": {"id": "GBG.lambert"},
"geometry": {
"type": "Polygon",
"coordinates": [
[[150000, 170000], [150010, 170000], [150010, 170010], [150000, 170000]]
],
},
}
lambert_partition = tmp_path / "partition-lambert.geojson"
lambert_partition.write_text(
json.dumps({"type": "FeatureCollection", "features": [lambert_feature]}),
encoding="utf-8",
)
lambert_artifact = tmp_path / "grb-lambert.geojson"
lambert_raw = json.dumps(
{"type": "FeatureCollection", "features": [lambert_feature]}
).encode("utf-8")
lambert_artifact.write_bytes(lambert_raw)
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
lambda **_kwargs: _storage_info(tmp_path, lambert_raw),
)
quarantined = DatasetService.import_partitioned_vector_artifact(
db,
project_id=project.id,
area_id=area.id,
artifact_path=lambert_artifact,
partition_paths=[lambert_partition],
original_filename="grb-lambert.geojson",
source="operator_official_import",
dataset_role="reference",
source_name="grb",
reference_layer_name="buildings",
metadata_json={
"feature_count": 1,
"crs": "EPSG:4326",
"bounds_json": {
"min_x": 150000,
"min_y": 170000,
"max_x": 150010,
"max_y": 170010,
},
},
source_metadata={"license": "Open data"},
provenance_metadata={
"artifact_sha256": sha256(lambert_raw).hexdigest(),
"partition_checksums": {
lambert_partition.name: sha256(
lambert_partition.read_bytes()
).hexdigest()
},
},
temporal_series_key="grb:partitioned:lambert",
observed_at=datetime(2026, 8, 2, tzinfo=UTC),
source_version="2026-08-02",
)
assert quarantined.status == "quarantined"
assert quarantined.validation_status == "failed"
assert quarantined.quarantine_status == "quarantined"
missing_manifest_partition = tmp_path / "partition-missing-manifest.geojson"
missing_manifest_partition.write_text(
json.dumps(partition_payload), encoding="utf-8"
)
missing_manifest_artifact = tmp_path / "grb-missing-manifest.geojson"
missing_manifest_raw = json.dumps(
{"type": "FeatureCollection", "features": [feature]}
).encode("utf-8")
missing_manifest_artifact.write_bytes(missing_manifest_raw)
monkeypatch.setattr(
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
lambda **_kwargs: _storage_info(tmp_path, missing_manifest_raw),
)
missing_manifest = DatasetService.import_partitioned_vector_artifact(
db,
project_id=project.id,
area_id=area.id,
artifact_path=missing_manifest_artifact,
partition_paths=[missing_manifest_partition],
original_filename="grb-missing-manifest.geojson",
source="operator_official_import",
dataset_role="reference",
source_name="grb",
reference_layer_name="buildings",
metadata_json={
"feature_count": 1,
"crs": "EPSG:4326",
"bounds_json": {
"min_x": 4.69,
"min_y": 51.09,
"max_x": 4.70,
"max_y": 51.10,
},
},
source_metadata={"license": "Open data"},
provenance_metadata={
"artifact_sha256": sha256(missing_manifest_raw).hexdigest()
},
temporal_series_key="grb:partitioned:missing-manifest",
observed_at=datetime(2026, 8, 3, tzinfo=UTC),
source_version="2026-08-03",
)
assert missing_manifest.status == "quarantined"
assert (
missing_manifest.validation_report_json["issues"][0]["code"]
== "PARTITION_CHECKSUM_MANIFEST_REQUIRED"
)
def test_partitioned_geometry_audit_handles_more_than_generic_topology_limit_without_materializing_geometries(
tmp_path: Path,
) -> None:
feature_count = 10_001
partition_path = tmp_path / "large-partition.geojson"
partition_path.write_text(
json.dumps(
{
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"id": f"GBG.{index}",
"properties": {"id": f"GBG.{index}"},
"geometry": {
"type": "Point",
"coordinates": [4.0 + index / 10_000_000, 51.0],
},
}
for index in range(feature_count)
],
}
),
encoding="utf-8",
)
audit = _PartitionedGeoJsonRecords(
[partition_path],
expected_feature_count=feature_count,
declared_partition_checksums={
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
},
).audit()
assert audit.feature_count == feature_count
assert audit.bounds_json["min_x"] == 4.0
assert audit.bounds_json["max_x"] > audit.bounds_json["min_x"]
assert audit.representative_record.geometry.geom_type == "MultiPoint"
+96 -16
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import json
import subprocess
import sys
from hashlib import sha256
from pathlib import Path
from PIL import Image
@@ -13,35 +14,114 @@ SCRIPT = Path(__file__).parents[2] / "scripts" / "build_grayscale_yolo_dataset.p
def test_grayscale_builder_preserves_labels_and_split(tmp_path: Path) -> None:
source = tmp_path / "source"
(source / "images" / "train").mkdir(parents=True)
(source / "labels" / "train").mkdir(parents=True)
image = source / "images" / "train" / "tile.png"
label = source / "labels" / "train" / "tile.txt"
Image.new("RGB", (8, 8), (255, 0, 0)).save(image)
label.write_text("0 0.5 0.5 0.5 0.5\n", encoding="utf-8")
entries = []
for split, sample_slug, colour in (("train", "fixture-train", (255, 0, 0)), ("val", "fixture-val", (0, 255, 0))):
image = source / "images" / split / f"{sample_slug}.png"
label = source / "labels" / split / f"{sample_slug}.txt"
image.parent.mkdir(parents=True, exist_ok=True)
label.parent.mkdir(parents=True, exist_ok=True)
Image.new("RGB", (8, 8), colour).save(image)
label.write_text("0 0.5 0.5 0.5 0.5\n", encoding="utf-8")
entries.append((split, sample_slug, image, label))
manifest = source / "operator_samples_manifest.json"
policy = "geointel-training-source-eligibility/v1"
manifest.write_text(
json.dumps(
{
"immutable": True,
"training_eligibility": {"policy_version": policy, "status": "eligible", "fixture_mode": True},
"samples": [
{
"sample_slug": sample_slug,
"split": split,
"raster_dataset_id": f"raster:{sample_slug}",
"reference_dataset_id": f"reference:{sample_slug}",
"training_eligibility": {
"policy_version": policy,
"eligible": True,
"fixture_mode": True,
"raster": {"eligible": True, "reasons": [], "evidence": {"dataset_id": f"raster:{sample_slug}", "checksum_sha256": "a" * 64, "source_registry_id": "fixture-raster", "source_snapshot_id": "fixture-raster-snapshot"}},
"reference": {"eligible": True, "reasons": [], "evidence": {"dataset_id": f"reference:{sample_slug}", "checksum_sha256": "b" * 64, "source_registry_id": "fixture-reference", "source_snapshot_id": "fixture-reference-snapshot"}},
},
}
for split, sample_slug, _image, _label in entries
],
}
),
encoding="utf-8",
)
(source / "corpus-freeze.json").write_text(
json.dumps(
{
"schema_version": 2,
"immutable": True,
"fixture_mode": True,
"training_eligibility_policy": policy,
"manifest_sha256": sha256(manifest.read_bytes()).hexdigest(),
}
),
encoding="utf-8",
)
dataset_yaml = source / "dataset.yaml"
dataset_yaml.write_text(
f"path: {source}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
encoding="utf-8",
)
release_script = SCRIPT.parent / "training_release_manifest.py"
subprocess.run(
[
sys.executable,
str(release_script),
"create",
"--train-yaml",
str(dataset_yaml),
"--corpus-manifest",
str(manifest),
"--fixture-mode",
],
check=True,
)
release_path = dataset_yaml.with_name(dataset_yaml.name + ".geointel-training-release.json")
asset_path = dataset_yaml.with_name(dataset_yaml.name + ".geointel-training-assets.json")
assets = json.loads(asset_path.read_text(encoding="utf-8"))
summary = source / "yolo_tile_dataset_summary.json"
summary.write_text(
json.dumps(
{
"dataset_yaml": str(dataset_yaml.resolve()),
"training_release_manifest": str(release_path.resolve()),
"training_release_manifest_sha256": sha256(release_path.read_bytes()).hexdigest(),
"training_asset_manifest": str(asset_path.resolve()),
"source_manifest_sha256": sha256(manifest.read_bytes()).hexdigest(),
"tiles": [
{
"split": "train",
"image_path": str(image),
"label_path": str(label),
}
]
{"split": entry["split"], "image_path": entry["image_path"], "label_path": entry["label_path"]}
for entry in assets["entries"]
],
}
),
encoding="utf-8",
)
output = tmp_path / "gray"
subprocess.run(
[sys.executable, str(SCRIPT), "--summary", str(summary), "--output-dir", str(output)],
[
sys.executable,
str(SCRIPT),
"--summary",
str(summary),
"--train-yaml",
str(dataset_yaml),
"--corpus-manifest",
str(manifest),
"--fixture-mode",
"--output-dir",
str(output),
],
check=True,
)
converted = Image.open(output / "images" / "train" / "tile.png")
converted = Image.open(output / "images" / "train" / "fixture-train.png")
r, g, b = converted.getpixel((0, 0))
assert r == g == b
assert (output / "labels" / "train" / "tile.txt").read_text() == label.read_text()
assert (output / "labels" / "train" / "fixture-train.txt").read_text() == "0 0.5 0.5 0.5 0.5\n"
evidence = json.loads((output / "grayscale-dataset-evidence.json").read_text())
assert evidence["converted_tile_count"] == 1
assert evidence["converted_tile_count"] == 2
assert evidence["training_eligible"] is False
+103 -2
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
from hashlib import sha256
import json
from pathlib import Path
from uuid import uuid4
@@ -10,9 +11,10 @@ from fastapi.testclient import TestClient
from app.core.config import Settings
from app.core.errors import AppError
from app.main import app
from app.models import AnalysisRun, Dataset, Detection, Job, Project
from app.models import AnalysisRun, Dataset, Detection, Job, Project, SourceRegistry, SourceSnapshot
from app.services.detection_service import DetectionService
from app.services.model_asset_catalog_service import ModelAssetCatalogService
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
class FakeSession:
@@ -63,15 +65,47 @@ class MockYoloAdapter:
def _project_and_raster_dataset():
project_id = uuid4()
dataset_id = uuid4()
source_registry_id = uuid4()
source_snapshot_id = uuid4()
checksum = "a" * 64
project = Project(id=project_id, name="Geel")
source_registry = SourceRegistry(
id=source_registry_id,
source_key="test-derived-raster",
display_name="Governed test-derived raster",
classification="derived",
authority_name="GeoIntel test fixture",
usage_policy_json={"ground_truth_allowed": False},
)
source_snapshot = SourceSnapshot(
id=source_snapshot_id,
source_registry_id=source_registry_id,
snapshot_key="test-derived-raster-v1",
checksum_sha256=checksum,
freshness_status="current",
ingest_status="ingested",
)
dataset = Dataset(
id=dataset_id,
project_id=project_id,
name="source.tif",
dataset_type="raster",
source="user_upload",
source="test-derived-raster",
source_name="test-derived-raster",
storage_path="storage/uploads/source.tif",
checksum_sha256=checksum,
source_registry_id=source_registry_id,
source_snapshot_id=source_snapshot_id,
data_contract_key="geointel.raster.geotiff",
data_contract_version="1.0.0",
validation_status="passed",
provenance_status="complete",
lineage_status="not_applicable",
quarantine_status="not_quarantined",
status="ready",
)
dataset.source_registry = source_registry
dataset.source_snapshot = source_snapshot
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
return db, project_id, dataset_id
@@ -101,6 +135,72 @@ def _manifest(tmp_path: Path) -> Path:
return manifest_path
def _write_model_sidecar(
model_path: Path,
settings: Settings,
*,
db: FakeSession | None = None,
) -> None:
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
source_registry_id = uuid4()
source_snapshot_id = uuid4()
source_version = settings.yolo_model_version or "test-v1"
if db is not None:
source_registry = SourceRegistry(
id=source_registry_id,
source_key="model",
display_name="Governed test model artifact",
classification="experimental",
authority_name="GeoIntel test fixture",
freshness_status="current",
ingest_status="configured",
)
source_snapshot = SourceSnapshot(
id=source_snapshot_id,
source_registry_id=source_registry_id,
snapshot_key=f"model-{source_version}",
source_version=source_version,
checksum_sha256=model_sha256,
freshness_status="current",
ingest_status="ingested",
)
db.objects[(SourceRegistry, source_registry_id)] = source_registry
db.objects[(SourceSnapshot, source_snapshot_id)] = source_snapshot
payload = {
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
"data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"},
"model": {
"model_id": settings.yolo_model_id,
"task_type": "object_detection",
"sha256": model_sha256,
"model_format": "pytorch",
"framework": "ultralytics/pytorch",
"class_mapping": {"0": "building"},
"source_version": source_version,
},
"source": {
"source_registry_id": str(source_registry_id),
"source_snapshot_id": str(source_snapshot_id),
"source_registry_key": "model",
"source_snapshot_checksum_sha256": model_sha256,
},
"lineage": {
"upstream_asset_ids": ["test-training-corpus"],
"upstream_checksums_sha256": ["a" * 64],
"transformations": [
{"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64}
],
},
"metadata": {"training_manifest_sha256": "c" * 64},
"imported_at": "2026-08-01T10:00:00+00:00",
}
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text(
json.dumps(payload, sort_keys=True),
encoding="utf-8",
)
def test_model_asset_catalog_lists_supported_local_model_files(tmp_path: Path) -> None:
model_file = tmp_path / "building-detector.pt"
model_file.write_bytes(b"local model")
@@ -190,6 +290,7 @@ def test_detection_run_persists_selected_model_asset_parameters(tmp_path: Path)
yolo_models_dir=str(tmp_path),
yolo_max_tiles=4,
)
_write_model_sidecar(model_file, settings, db=db)
result = DetectionService.run_detection(
db=db,
@@ -0,0 +1,644 @@
from __future__ import annotations
from datetime import datetime, timezone
from hashlib import sha256
import json
from pathlib import Path
from types import SimpleNamespace
from uuid import uuid4
from app.models import Dataset, DatasetVersion, SourceRegistry, SourceSnapshot
from app.services.data_contract_validation import (
build_vector_ingest_input,
validate_registered_asset,
)
from app.services.demo_workflow_service import DemoWorkflowService
from app.services.derived_dataset_governance_service import (
DerivedDatasetGovernanceService,
)
from app.services.raster_operations_service import RasterOperationsService
from app.services.storage_service import StorageService
from app.services.vector_feature_service import VectorFeatureService
from app.services.vector_operations_service import VectorOperationsService
_CHECKSUM = "a" * 64
class _FakeSession:
def __init__(self, rows=None) -> None:
self.rows = rows or {}
self.added = []
def get(self, model, row_id):
row = self.rows.get((model, row_id))
if row is not None:
return row
return next(
(
item
for item in self.added
if isinstance(item, model) and item.id == row_id
),
None,
)
def add(self, row) -> None:
self.added.append(row)
def commit(self) -> None:
return None
def refresh(self, row) -> None:
return None
class _GovernedSession:
"""Small ORM-shaped session for the real governance branch.
Registry persistence is monkeypatched below; the test exercises the
service's orchestration and report decisions without needing PostGIS.
"""
def __init__(self) -> None:
self.added = []
self.flushes = 0
class _EmptyQuery:
def filter(self, *_args, **_kwargs):
return self
@staticmethod
def all():
return []
@staticmethod
def one_or_none():
return None
def query(self, *_args, **_kwargs):
# Governance now performs a bounded lineage traversal during quarantine.
# This focused harness intentionally has no persisted siblings/edges.
return self._EmptyQuery()
def add(self, row) -> None:
self.added.append(row)
def flush(self) -> None:
self.flushes += 1
def _governed_parent() -> Dataset:
source_id = uuid4()
snapshot_id = uuid4()
source = SourceRegistry(
id=source_id,
source_key="grb",
display_name="GRB parent fixture",
classification="authoritative",
authority_name="Digitaal Vlaanderen",
authority_scope_json={"zone": "Flanders"},
usage_policy_json={"ground_truth_allowed": True},
)
snapshot = SourceSnapshot(
id=snapshot_id,
source_registry_id=source_id,
snapshot_key="governed-parent",
checksum_sha256=_CHECKSUM,
freshness_status="current",
ingest_status="ingested",
)
dataset = Dataset(
id=uuid4(),
project_id=uuid4(),
name="governed.geojson",
dataset_type="vector",
source="grb",
source_name="grb",
status="ready",
checksum_sha256=_CHECKSUM,
data_contract_key="geointel.vector.geojson",
data_contract_version="1.0.0",
validation_status="passed",
provenance_status="complete",
lineage_status="not_applicable",
quarantine_status="not_quarantined",
source_registry_id=source_id,
source_snapshot_id=snapshot_id,
)
dataset.source_registry = source
dataset.source_snapshot = snapshot
return dataset
def test_lineage_evidence_quarantines_ungoverned_parent_without_inventing_a_checksum() -> (
None
):
parent = _governed_parent()
valid_lineage = DerivedDatasetGovernanceService._lineage_evidence(
parent, "vector.clip", {"area_id": "a"}
)
valid_report = validate_registered_asset(
build_vector_ingest_input(
asset_id="derived-valid",
source_crs="EPSG:4326",
storage_crs="EPSG:4326",
feature_collection={
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
"properties": {},
}
],
},
checksum_sha256=_CHECKSUM,
computed_checksum_sha256=_CHECKSUM,
source_registry_id="derived-source",
source_snapshot_id="derived-snapshot",
imported_at=datetime.now(timezone.utc),
metadata={
"license": "internal derived artifact",
"bounds_json": {
"min_x": 5.0,
"min_y": 51.0,
"max_x": 5.0,
"max_y": 51.0,
},
},
temporal_unknown_reason="derived input has no precise observation timestamp",
source_version_unknown_reason="transform version is recorded separately",
lineage=valid_lineage,
)
)
assert valid_report.validation_status.value == "passed"
assert valid_lineage.upstream_asset_ids == (str(parent.id),)
assert valid_lineage.upstream_checksums_sha256 == (_CHECKSUM,)
parent.validation_status = "not_validated"
rejected_lineage = DerivedDatasetGovernanceService._lineage_evidence(
parent, "vector.clip", {}
)
rejected_report = validate_registered_asset(
build_vector_ingest_input(
asset_id="derived-rejected",
source_crs="EPSG:4326",
storage_crs="EPSG:4326",
feature_collection={
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
"properties": {},
}
],
},
checksum_sha256=_CHECKSUM,
computed_checksum_sha256=_CHECKSUM,
source_registry_id="derived-source",
source_snapshot_id="derived-snapshot",
imported_at=datetime.now(timezone.utc),
metadata={
"license": "internal derived artifact",
"bounds_json": {
"min_x": 5.0,
"min_y": 51.0,
"max_x": 5.0,
"max_y": 51.0,
},
},
temporal_unknown_reason="derived input has no precise observation timestamp",
source_version_unknown_reason="transform version is recorded separately",
lineage=rejected_lineage,
)
)
assert rejected_lineage.upstream_checksums_sha256 == (
"parent_dataset_not_governed",
)
assert rejected_report.validation_status.value == "failed"
assert rejected_report.quarantine_status.value == "quarantined"
assert any(
issue.code == "UPSTREAM_CHECKSUM_FORMAT_INVALID"
for issue in rejected_report.issues
)
def test_govern_vector_binds_snapshot_contract_and_lineage_before_marking_ready(
monkeypatch,
) -> None:
from app.services.source_registry_service import SourceRegistryService
db = _GovernedSession()
parent = _governed_parent()
parent_version_id = uuid4()
dataset = Dataset(
id=uuid4(),
project_id=parent.project_id,
name="derived.geojson",
dataset_type="vector",
source="operation:clip",
source_name="derived",
checksum_sha256=_CHECKSUM,
imported_at=datetime.now(timezone.utc),
crs="EPSG:4326",
metadata_json={
"bounds_json": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.0, "max_y": 51.0}
},
status="validating",
)
version = DatasetVersion(
id=uuid4(), dataset_id=dataset.id, version=1, checksum_sha256=_CHECKSUM
)
source = SimpleNamespace(id=uuid4(), license_name="internal derived artifact")
snapshot = SimpleNamespace(id=uuid4(), source_registry_id=source.id)
edges = []
monkeypatch.setattr(
SourceRegistryService,
"ensure_server_owned_source",
lambda *_args, **_kwargs: source,
)
monkeypatch.setattr(
SourceRegistryService, "record_snapshot", lambda *_args, **_kwargs: snapshot
)
monkeypatch.setattr(
DerivedDatasetGovernanceService,
"_latest_parent_version_id",
lambda *_args: parent_version_id,
)
def _bind(target, **kwargs):
target.source_registry_id = kwargs["source"].id
target.source_snapshot_id = kwargs["snapshot"].id
target.data_contract_key = kwargs["data_contract_key"]
target.data_contract_version = kwargs["data_contract_version"]
target.validation_status = kwargs["validation_status"]
target.provenance_status = kwargs["provenance_status"]
target.lineage_status = kwargs["lineage_status"]
return target
monkeypatch.setattr(SourceRegistryService, "bind_dataset_provenance", _bind)
monkeypatch.setattr(SourceRegistryService, "bind_dataset_version_provenance", _bind)
monkeypatch.setattr(
SourceRegistryService,
"record_lineage_edge",
lambda *_args, **kwargs: edges.append(kwargs),
)
ready = DerivedDatasetGovernanceService.govern_vector(
db,
dataset=dataset,
dataset_version=version,
feature_collection={
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
"properties": {},
}
],
},
source_key="derived",
operation="vector.clip",
parent_dataset=parent,
operation_parameters={"area_id": "a"},
)
assert ready is True
assert dataset.status == "ready"
assert dataset.validation_status == "passed"
assert dataset.provenance_status == "complete"
assert dataset.source_registry_id == source.id
assert version.source_snapshot_id == snapshot.id
assert db.flushes >= 1
assert edges[0]["parent_dataset_id"] == parent.id
assert edges[0]["parent_dataset_version_id"] == parent_version_id
assert edges[0]["child_dataset_version_id"] == version.id
def test_govern_vector_quarantines_output_when_parent_is_manual_or_experimental(
monkeypatch,
) -> None:
from app.services.source_registry_service import SourceRegistryService
db = _GovernedSession()
parent = _governed_parent()
parent.source = "manual"
parent.source_name = "manual"
parent.source_registry.source_key = "manual"
parent.source_registry.classification = "experimental"
parent.source_registry.usage_policy_json = {"ground_truth_allowed": False}
dataset = Dataset(
id=uuid4(),
project_id=parent.project_id,
name="manual-derived.geojson",
dataset_type="vector",
source="operation:clip",
source_name="derived",
checksum_sha256=_CHECKSUM,
imported_at=datetime.now(timezone.utc),
crs="EPSG:4326",
metadata_json={
"bounds_json": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.0, "max_y": 51.0}
},
status="validating",
)
version = DatasetVersion(
id=uuid4(), dataset_id=dataset.id, version=1, checksum_sha256=_CHECKSUM
)
source = SimpleNamespace(id=uuid4(), license_name="internal derived artifact")
snapshot = SimpleNamespace(
id=uuid4(), source_registry_id=source.id, ingest_status="ingested"
)
monkeypatch.setattr(
SourceRegistryService,
"ensure_server_owned_source",
lambda *_args, **_kwargs: source,
)
monkeypatch.setattr(
SourceRegistryService, "record_snapshot", lambda *_args, **_kwargs: snapshot
)
monkeypatch.setattr(
DerivedDatasetGovernanceService,
"_latest_parent_version_id",
lambda *_args: None,
)
monkeypatch.setattr(
SourceRegistryService, "record_lineage_edge", lambda *_args, **_kwargs: None
)
ready = DerivedDatasetGovernanceService.govern_vector(
db,
dataset=dataset,
dataset_version=version,
feature_collection={
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
"properties": {},
}
],
},
source_key="derived",
operation="vector.clip",
parent_dataset=parent,
)
assert ready is False
assert dataset.status == "quarantined"
assert dataset.quarantine_status == "quarantined"
assert dataset.validation_status == "failed"
assert any(
issue["code"] == "PARENT_DATASET_NOT_ELIGIBLE_FOR_DERIVED_PROCESSING"
for issue in dataset.validation_report_json["issues"]
)
assert snapshot.ingest_status == "quarantined"
def test_vector_selection_uses_map_selection_registry_and_skips_features_when_quarantined(
monkeypatch, tmp_path
) -> None:
source = _governed_parent()
source.area_id = None
source.storage_path = str(tmp_path / "source.geojson")
db = _FakeSession({(Dataset, source.id): source})
output_path = tmp_path / "selection.geojson"
calls = []
persisted_features = []
monkeypatch.setattr(
VectorFeatureService,
"select_features_by_bbox",
lambda *_args, **_kwargs: {
"selection_bbox": {
"min_x": 4.9,
"min_y": 50.9,
"max_x": 5.2,
"max_y": 51.2,
"crs": "EPSG:4326",
},
"feature_count": 1,
"limit": 250,
"truncated": False,
"geojson": {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"id": "source-feature",
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
"properties": {
"vector_feature_id": "source-feature",
"dataset_id": str(source.id),
},
}
],
},
},
)
def _persist_dataset_file(**kwargs):
output_path.write_bytes(kwargs["content"])
return {
"original_filename": kwargs["original_filename"],
"stored_filename": output_path.name,
"content_type": kwargs["content_type"],
"size_bytes": len(kwargs["content"]),
"checksum_sha256": _CHECKSUM,
"storage_path": str(output_path),
}
monkeypatch.setattr(StorageService, "persist_dataset_file", _persist_dataset_file)
def _quarantine(db, **kwargs):
calls.append(kwargs)
kwargs["dataset"].status = "quarantined"
return False
monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_vector", _quarantine)
monkeypatch.setattr(
VectorFeatureService,
"persist_geojson_features",
lambda **kwargs: persisted_features.append(kwargs),
)
response = VectorOperationsService.derive_selection_dataset(
db=db,
dataset_id=source.id,
bbox={
"min_x": 4.9,
"min_y": 50.9,
"max_x": 5.2,
"max_y": 51.2,
"crs": "EPSG:4326",
},
)
assert response.status == "quarantined"
assert calls[0]["source_key"] == "map_selection"
assert calls[0]["parent_dataset"] is source
assert calls[0]["operation"] == "vector.selection"
assert persisted_features == []
def test_vector_buffer_uses_projected_metres_instead_of_wgs84_degrees(
monkeypatch, tmp_path
) -> None:
source = _governed_parent()
source.storage_path = str(tmp_path / "source.geojson")
source.crs = "EPSG:4326"
Path(source.storage_path).write_text(
json.dumps(
{
"type": "FeatureCollection",
"crs": "EPSG:4326",
"features": [
{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
"properties": {},
},
],
}
),
encoding="utf-8",
)
# A governed consumption artifact must carry the checksum of these exact
# bytes; vector operations deliberately refuse a stale fixture checksum.
source.checksum_sha256 = sha256(Path(source.storage_path).read_bytes()).hexdigest()
source.source_snapshot.checksum_sha256 = source.checksum_sha256
db = _FakeSession({(Dataset, source.id): source})
captured = {}
def _persist(**kwargs):
captured.update(kwargs)
return uuid4()
monkeypatch.setattr(VectorOperationsService, "_persist_derived_dataset", _persist)
VectorOperationsService.buffer(
db, source.id, distance_m=100.0, dissolve=False, output_name=None
)
coordinates = captured["feature_collection"]["features"][0]["geometry"][
"coordinates"
][0]
longitudes = [coordinate[0] for coordinate in coordinates]
latitudes = [coordinate[1] for coordinate in coordinates]
assert max(longitudes) - min(longitudes) < 0.01
assert max(latitudes) - min(latitudes) < 0.01
def test_raster_operation_uses_derived_registry_before_commit(
monkeypatch, tmp_path
) -> None:
source = _governed_parent()
source.dataset_type = "raster"
source.storage_path = str(tmp_path / "source.tif")
output_path = tmp_path / "derived.tif"
output_path.write_bytes(b"derived-raster")
db = _FakeSession()
calls = []
def _govern(db, **kwargs):
calls.append(kwargs)
kwargs["dataset"].status = "quarantined"
return False
monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_raster", _govern)
result = RasterOperationsService._persist_derived_dataset(
db,
source_dataset=source,
source_dataset_id=source.id,
operation="ndvi",
output_path=str(output_path),
output_name="derived.tif",
metadata={
"crs": "EPSG:31370",
"bounds": [100000.0, 100000.0, 100001.0, 100001.0],
"resolution": [1.0, 1.0],
"width": 1,
"height": 1,
"band_count": 1,
"dtype": ["float32"],
"operation_parameters": {"nir_band": 4, "red_band": 3},
},
)
derived = next(item for item in db.added if isinstance(item, Dataset))
assert result == derived.id
assert derived.status == "quarantined"
assert derived.source_name == "derived"
assert calls[0]["source_key"] == "derived"
assert calls[0]["parent_dataset"] is source
assert calls[0]["operation"] == "raster.ndvi"
def test_demo_fixture_creation_is_governed_and_does_not_persist_features_when_rejected(
monkeypatch, tmp_path
) -> None:
project_id = uuid4()
area_id = uuid4()
db = _FakeSession()
calls = []
features = []
monkeypatch.setattr(
StorageService,
"persist_dataset_file",
lambda **kwargs: {
"storage_path": str(tmp_path / kwargs["original_filename"]),
"original_filename": kwargs["original_filename"],
"stored_filename": kwargs["original_filename"],
"content_type": kwargs["content_type"],
"size_bytes": len(kwargs["content"]),
"checksum_sha256": _CHECKSUM,
},
)
def _quarantine(db, **kwargs):
calls.append(kwargs)
kwargs["dataset"].status = "quarantined"
return False
monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_vector", _quarantine)
monkeypatch.setattr(
VectorFeatureService,
"persist_geojson_features",
lambda **kwargs: features.append(kwargs),
)
payload = {
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
"properties": {},
}
],
}
dataset = DemoWorkflowService._create_dataset(
db,
project_id=project_id,
area_id=area_id,
filename="fixture.geojson",
payload=payload,
raw=json.dumps(payload).encode("utf-8"),
role="source",
source_name="fixture",
reference_layer_name=None,
)
assert dataset.status == "quarantined"
assert calls[0]["source_key"] == "fixture"
assert calls[0]["operation"] == "demo.fixture_vector"
assert features == []
+47 -6
View File
@@ -1,11 +1,11 @@
from __future__ import annotations
import json
from hashlib import sha256
from pathlib import Path
from types import SimpleNamespace
from uuid import uuid4
from app.models import Area, Dataset
from app.models import Dataset, SourceRegistry, SourceSnapshot
from app.services.qa_service import QaService
@@ -46,6 +46,47 @@ def _write_features(path: Path, features: list[dict]) -> None:
path.write_text(json.dumps({"type": "FeatureCollection", "features": features}), encoding="utf-8")
def _authoritative_reference(dataset: Dataset) -> Dataset:
"""Give QA reference fixtures the same durable authority proof as GRB."""
source_id = uuid4()
snapshot_id = uuid4()
checksum = sha256(Path(str(dataset.storage_path)).read_bytes()).hexdigest()
source = SourceRegistry(
id=source_id,
source_key="grb",
display_name="GRB test reference",
classification="authoritative",
authority_name="Digitaal Vlaanderen",
authority_scope_json={"zone": "Flanders"},
usage_policy_json={"ground_truth_allowed": True, "validation_authority": {"building_validation": "primary"}},
)
snapshot = SourceSnapshot(
id=snapshot_id,
source_registry_id=source_id,
snapshot_key=f"qa-grb-{dataset.id}",
checksum_sha256=checksum,
ingest_status="ingested",
freshness_status="current",
)
dataset.source = "grb"
dataset.source_name = "grb"
dataset.dataset_role = "reference"
dataset.status = "ready"
dataset.checksum_sha256 = checksum
dataset.source_registry_id = source_id
dataset.source_snapshot_id = snapshot_id
dataset.data_contract_key = "geointel.vector.geojson"
dataset.data_contract_version = "1.0.0"
dataset.validation_status = "passed"
dataset.provenance_status = "complete"
dataset.lineage_status = "complete"
dataset.quarantine_status = "not_quarantined"
dataset.source_registry = source
dataset.source_snapshot = snapshot
return dataset
def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None:
project_id = uuid4()
candidate_id = uuid4()
@@ -66,7 +107,7 @@ def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None:
crs="EPSG:4326",
metadata_json={"crs_assumed": False},
)
reference = Dataset(
reference = _authoritative_reference(Dataset(
id=reference_id,
project_id=project_id,
name="reference.geojson",
@@ -75,7 +116,7 @@ def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None:
storage_path=str(reference_path),
crs="EPSG:4326",
metadata_json={"crs_assumed": False},
)
))
result = QaService.compare_candidate_with_reference(
db=FakeSession([candidate, reference]),
@@ -117,7 +158,7 @@ def test_qa_compare_candidate_with_reference_returns_feature_level_evidence(tmp_
crs="EPSG:4326",
metadata_json={"crs_assumed": False},
)
reference = Dataset(
reference = _authoritative_reference(Dataset(
id=reference_id,
project_id=project_id,
name="reference.geojson",
@@ -126,7 +167,7 @@ def test_qa_compare_candidate_with_reference_returns_feature_level_evidence(tmp_
storage_path=str(reference_path),
crs="EPSG:4326",
metadata_json={"crs_assumed": False},
)
))
result = QaService.compare_candidate_with_reference(
db=FakeSession([candidate, reference]),
+80 -24
View File
@@ -49,6 +49,54 @@ def scope_area(name: str, geometry):
return SimpleNamespace(name=name, geometry=geometry)
def governed_materialization(
*,
source_name: str,
reference_layer_name: str | None,
source_metadata: dict[str, object],
dataset_id=None,
) -> SimpleNamespace:
"""Build a complete authoritative materialization for coverage tests.
Coverage is a production-facing statement. These fixtures must therefore
carry the same registry, immutable snapshot, checksum and freshness state
that a materialized official dataset needs in production.
"""
source_registry_id = uuid4()
source_snapshot_id = uuid4()
checksum_sha256 = "a" * 64
return SimpleNamespace(
id=dataset_id or uuid4(),
status="ready",
source=source_name,
source_name=source_name,
reference_layer_name=reference_layer_name,
source_metadata=dict(source_metadata),
checksum_sha256=checksum_sha256,
source_registry_id=source_registry_id,
source_snapshot_id=source_snapshot_id,
data_contract_key="geointel.vector.geojson",
data_contract_version="1.0.0",
validation_status="passed",
provenance_status="complete",
lineage_status="not_applicable",
quarantine_status="not_quarantined",
source_registry=SimpleNamespace(
source_key=source_name,
classification="authoritative",
authority_scope_json={"scope": "coverage test"},
usage_policy_json={},
),
source_snapshot=SimpleNamespace(
source_registry_id=source_registry_id,
checksum_sha256=checksum_sha256,
freshness_status="current",
ingest_status="ingested",
),
)
def test_coverage_catalog_uses_normalized_contracts_and_does_not_change_provider_registry() -> None:
catalog = CoverageRegistryService.catalog()
@@ -135,9 +183,8 @@ def test_coverage_resolver_only_reports_operational_for_materialized_ready_datas
assert without_materialized.items[0].materialized_dataset_ids == []
dataset_id = uuid4()
materialized = SimpleNamespace(
id=dataset_id,
status="ready",
materialized = governed_materialization(
dataset_id=dataset_id,
source_name="ngi_adminvector",
reference_layer_name="belgium_regions",
source_metadata={"coverage_zones": ["belgium", "flanders", "wallonia", "brussels"]},
@@ -156,9 +203,8 @@ def test_coverage_resolver_only_reports_operational_for_materialized_ready_datas
def test_statbel_population_materialization_does_not_masquerade_as_admin_data() -> None:
project_id = uuid4()
statbel_id = uuid4()
statbel = SimpleNamespace(
id=statbel_id,
status="ready",
statbel = governed_materialization(
dataset_id=statbel_id,
source_name="statbel",
reference_layer_name="population",
source_metadata={"coverage_zones": ["belgium", "flanders", "wallonia", "brussels"]},
@@ -187,9 +233,8 @@ def test_statbel_population_materialization_does_not_masquerade_as_admin_data()
def test_bounded_api_materialization_only_covers_its_persisted_bbox() -> None:
project_id = uuid4()
dataset_id = uuid4()
dataset = SimpleNamespace(
id=dataset_id,
status="ready",
dataset = governed_materialization(
dataset_id=dataset_id,
source_name="spw_picc",
reference_layer_name="buildings",
source_metadata={
@@ -230,8 +275,24 @@ def test_bounded_partition_union_can_be_operational() -> None:
left_id = uuid4()
right_id = uuid4()
datasets = [
SimpleNamespace(id=left_id, status="ready", source_name="spw_picc", reference_layer_name="buildings", source_metadata={"coverage_zones": ["wallonia"], "bbox_epsg4326": [4.50, 50.50, 4.60, 50.60]}),
SimpleNamespace(id=right_id, status="ready", source_name="spw_picc", reference_layer_name="buildings", source_metadata={"coverage_zones": ["wallonia"], "bbox_epsg4326": [4.60, 50.50, 4.70, 50.60]}),
governed_materialization(
dataset_id=left_id,
source_name="spw_picc",
reference_layer_name="buildings",
source_metadata={
"coverage_zones": ["wallonia"],
"bbox_epsg4326": [4.50, 50.50, 4.60, 50.60],
},
),
governed_materialization(
dataset_id=right_id,
source_name="spw_picc",
reference_layer_name="buildings",
source_metadata={
"coverage_zones": ["wallonia"],
"bbox_epsg4326": [4.60, 50.50, 4.70, 50.60],
},
),
]
session = FakeSession(
project=SimpleNamespace(id=project_id),
@@ -252,9 +313,7 @@ def test_spw_bathymetry_materialization_is_source_specific() -> None:
scope_area("Wallonia", box(2.5, 49.5, 6.4, 50.8)),
]
selection = CoverageBBox(minx=4.851, miny=50.451, maxx=4.869, maxy=50.469)
spw_picc = SimpleNamespace(
id=uuid4(),
status="ready",
spw_picc = governed_materialization(
source_name="spw_picc",
reference_layer_name="buildings",
source_metadata={
@@ -272,9 +331,8 @@ def test_spw_bathymetry_materialization_is_source_specific() -> None:
assert without_bathymetry.items[0].materialized_dataset_ids == []
bathymetry_id = uuid4()
bathymetry = SimpleNamespace(
id=bathymetry_id,
status="ready",
bathymetry = governed_materialization(
dataset_id=bathymetry_id,
source_name="spw_bathymetry",
reference_layer_name=None,
source_metadata={
@@ -309,9 +367,8 @@ def test_vha_bathymetry_profiles_are_operational_only_inside_the_persisted_selec
assert without_profiles.items[0].source_names == ["vmm_vha_bathymetry_profiles"]
profile_id = uuid4()
profiles = SimpleNamespace(
id=profile_id,
status="ready",
profiles = governed_materialization(
dataset_id=profile_id,
source_name="vmm_vha_bathymetry_profiles",
reference_layer_name="bathymetry_profile_points",
source_metadata={
@@ -360,9 +417,8 @@ def test_flemish_materialization_is_theme_specific() -> None:
project_id = uuid4()
project = SimpleNamespace(id=project_id)
orthophoto_id = uuid4()
orthophoto = SimpleNamespace(
id=orthophoto_id,
status="ready",
orthophoto = governed_materialization(
dataset_id=orthophoto_id,
source_name="digitaal_vlaanderen_orthophoto",
reference_layer_name="orthophoto",
source_metadata={"coverage_zones": ["flanders"]},
@@ -429,5 +485,5 @@ def test_frontend_prefers_materialized_national_workspace_and_resolves_drawn_bbo
assert "externalApi.resolveCoverage" in coverage_hook
assert "coverage.outside_supported_scope" in map_workspace
assert "coverageStatusLabel" in map_workspace
assert "coverageSelectionAvailable" in map_workspace
assert "activeThemeSupportsCurrentSelection" in map_workspace
assert "activeThemeAvailable && !regionalPartitionedThemeActive" in map_workspace
+3 -1
View File
@@ -44,8 +44,10 @@ def _project_and_dataset():
project_id=project_id,
name="ortho.tif",
dataset_type="raster",
source="user_upload",
source="fixture",
source_name="fixture",
storage_path="storage/uploads/ortho.tif",
source_metadata={"fixture": True},
)
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
return db, project_id, dataset_id
@@ -0,0 +1,325 @@
from __future__ import annotations
from hashlib import sha256
import json
from pathlib import Path
from uuid import uuid4
import pytest
from app.core.errors import AppError
from app.models import DatasetQuarantine, SourceRegistry, SourceSnapshot
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
class FakeSession:
"""Explicit database double for production-runtime provenance tests."""
def __init__(self, objects: dict[tuple[type, object], object] | None = None) -> None:
self.objects = objects or {}
def get(self, model, item_id):
return self.objects.get((model, item_id))
def _write_sidecar(
model_path: Path,
*,
model_id: str = "yolo-configured",
task_type: str = "object_detection",
framework: str = "ultralytics/pytorch",
source_version: str = "test-v1",
source_registry_id: str | None = None,
source_snapshot_id: str | None = None,
) -> Path:
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
payload = {
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
"data_contract": {
"key": "geointel.model.pytorch",
"version": "1.0.0",
},
"model": {
"model_id": model_id,
"task_type": task_type,
"sha256": model_sha256,
"model_format": "pytorch",
"framework": framework,
"class_mapping": {"0": "building"},
"source_version": source_version,
},
"source": {
"source_registry_id": source_registry_id or str(uuid4()),
"source_snapshot_id": source_snapshot_id or str(uuid4()),
"source_registry_key": "model",
"source_snapshot_checksum_sha256": model_sha256,
},
"lineage": {
"upstream_asset_ids": ["training-corpus:test-v1"],
"upstream_checksums_sha256": ["a" * 64],
"transformations": [
{
"name": "pytorch-training",
"version": "1.0.0",
"checksum_sha256": "b" * 64,
}
],
},
"metadata": {
"training_manifest_sha256": "c" * 64,
},
"imported_at": "2026-08-01T10:00:00+00:00",
}
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
sidecar_path = RuntimeModelProvenanceService.manifest_path_for_model(model_path)
sidecar_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
return sidecar_path
def _governed_model_database(
*,
source_registry_id,
source_snapshot_id,
model_checksum: str,
source_version: str = "test-v1",
) -> tuple[FakeSession, SourceRegistry, SourceSnapshot]:
registry = SourceRegistry(
id=source_registry_id,
source_key="model",
display_name="Governed test model artifacts",
classification="experimental",
authority_name="GeoIntel test fixture",
freshness_status="current",
ingest_status="configured",
)
snapshot = SourceSnapshot(
id=source_snapshot_id,
source_registry_id=source_registry_id,
snapshot_key=f"model-{source_version}",
source_version=source_version,
checksum_sha256=model_checksum,
freshness_status="current",
ingest_status="ingested",
)
return (
FakeSession(
{
(SourceRegistry, source_registry_id): registry,
(SourceSnapshot, source_snapshot_id): snapshot,
}
),
registry,
snapshot,
)
def test_runtime_model_provenance_accepts_byte_bound_pytorch_sidecar_for_structural_preflight(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"trusted local model bytes")
sidecar_path = _write_sidecar(model_path, source_version="v1")
evidence = RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id="yolo-configured",
task_type="object_detection",
expected_model_version="v1",
allowed_frameworks=("ultralytics/pytorch",),
)
assert evidence.model_sha256 == sha256(model_path.read_bytes()).hexdigest()
assert evidence.manifest_path == str(sidecar_path.resolve())
assert evidence.data_contract_key == "geointel.model.pytorch"
assert evidence.data_contract_version == "1.0.0"
assert len(evidence.validation_report_sha256) == 64
def test_production_runtime_requires_db_bound_model_source_snapshot(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"governed local model bytes")
source_registry_id = uuid4()
source_snapshot_id = uuid4()
_write_sidecar(
model_path,
source_version="v1",
source_registry_id=str(source_registry_id),
source_snapshot_id=str(source_snapshot_id),
)
db, _, _ = _governed_model_database(
source_registry_id=source_registry_id,
source_snapshot_id=source_snapshot_id,
model_checksum=sha256(model_path.read_bytes()).hexdigest(),
source_version="v1",
)
evidence = RuntimeModelProvenanceService.validate_for_production_runtime(
db=db,
model_path=model_path,
model_id="yolo-configured",
task_type="object_detection",
expected_model_version="v1",
allowed_frameworks=("ultralytics/pytorch",),
)
assert evidence.source_registry_id == str(source_registry_id)
assert evidence.source_snapshot_id == str(source_snapshot_id)
assert evidence.source_snapshot_checksum_sha256 == evidence.model_sha256
def test_production_runtime_rejects_missing_database_source_binding(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"unbound model bytes")
_write_sidecar(model_path)
with pytest.raises(AppError) as exc_info:
RuntimeModelProvenanceService.validate_for_production_runtime(
db=FakeSession(),
model_path=model_path,
model_id="yolo-configured",
task_type="object_detection",
)
assert exc_info.value.code == "MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND"
def test_production_runtime_requires_a_database_session(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"model bytes")
_write_sidecar(model_path)
with pytest.raises(AppError) as exc_info:
RuntimeModelProvenanceService.validate_for_production_runtime(
db=None,
model_path=model_path,
model_id="yolo-configured",
task_type="object_detection",
)
assert exc_info.value.code == "MODEL_PROVENANCE_DATABASE_REQUIRED"
@pytest.mark.parametrize(
("mutation", "expected_code"),
(
("registry_unsafe", "MODEL_PROVENANCE_SOURCE_REGISTRY_UNSAFE"),
("snapshot_registry_mismatch", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_REGISTRY_MISMATCH"),
("snapshot_missing", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_NOT_FOUND"),
("snapshot_quarantined", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_UNSAFE"),
("snapshot_checksum_mismatch", "MODEL_PROVENANCE_DATABASE_SNAPSHOT_CHECKSUM_MISMATCH"),
("active_quarantine", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_QUARANTINED"),
),
)
def test_production_runtime_rejects_unsafe_or_inconsistent_database_snapshot(
tmp_path: Path,
mutation: str,
expected_code: str,
) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"governed model bytes")
source_registry_id = uuid4()
source_snapshot_id = uuid4()
_write_sidecar(
model_path,
source_registry_id=str(source_registry_id),
source_snapshot_id=str(source_snapshot_id),
)
db, registry, snapshot = _governed_model_database(
source_registry_id=source_registry_id,
source_snapshot_id=source_snapshot_id,
model_checksum=sha256(model_path.read_bytes()).hexdigest(),
)
if mutation == "registry_unsafe":
registry.ingest_status = "quarantined"
elif mutation == "snapshot_registry_mismatch":
snapshot.source_registry_id = uuid4()
elif mutation == "snapshot_missing":
db.objects.pop((SourceSnapshot, source_snapshot_id))
elif mutation == "snapshot_quarantined":
snapshot.ingest_status = "quarantined"
elif mutation == "snapshot_checksum_mismatch":
snapshot.checksum_sha256 = "f" * 64
elif mutation == "active_quarantine":
snapshot.quarantines = [
DatasetQuarantine(
source_snapshot_id=source_snapshot_id,
stage="test",
reason_code="test_active_quarantine",
status="quarantined",
)
]
with pytest.raises(AppError) as exc_info:
RuntimeModelProvenanceService.validate_for_production_runtime(
db=db,
model_path=model_path,
model_id="yolo-configured",
task_type="object_detection",
)
assert exc_info.value.code == expected_code
def test_runtime_model_provenance_rejects_missing_sidecar(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"unmanifested local model bytes")
with pytest.raises(AppError) as exc_info:
RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id="yolo-configured",
task_type="object_detection",
)
assert exc_info.value.code == "MODEL_PROVENANCE_MANIFEST_MISSING"
def test_runtime_model_provenance_rejects_model_bytes_tampered_after_manifest(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"original local model bytes")
_write_sidecar(model_path)
model_path.write_bytes(b"tampered local model bytes")
with pytest.raises(AppError) as exc_info:
RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id="yolo-configured",
task_type="object_detection",
)
assert exc_info.value.code == "MODEL_PROVENANCE_MODEL_CHECKSUM_MISMATCH"
def test_runtime_model_provenance_rejects_tampered_manifest_contents(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"original local model bytes")
sidecar_path = _write_sidecar(model_path)
payload = json.loads(sidecar_path.read_text(encoding="utf-8"))
payload["model"]["class_mapping"]["1"] = "road"
sidecar_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
with pytest.raises(AppError) as exc_info:
RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id="yolo-configured",
task_type="object_detection",
)
assert exc_info.value.code == "MODEL_PROVENANCE_MANIFEST_CHECKSUM_MISMATCH"
def test_runtime_model_provenance_rejects_other_contract_even_if_structurally_valid(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"local model bytes")
sidecar_path = _write_sidecar(model_path)
payload = json.loads(sidecar_path.read_text(encoding="utf-8"))
payload["data_contract"]["key"] = "geointel.vector.geojson"
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
sidecar_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
with pytest.raises(AppError) as exc_info:
RuntimeModelProvenanceService.validate_for_runtime(
model_path=model_path,
model_id="yolo-configured",
task_type="object_detection",
)
assert exc_info.value.code == "MODEL_PROVENANCE_MANIFEST_INVALID"
@@ -1,5 +1,6 @@
from __future__ import annotations
from hashlib import sha256
import json
from pathlib import Path
from uuid import uuid4
@@ -8,9 +9,10 @@ import pytest
from geoalchemy2.shape import to_shape
from app.core.config import Settings
from app.models import Dataset, Project, Segmentation
from app.models import Dataset, Project, Segmentation, SourceRegistry, SourceSnapshot
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
from app.services.model_registry_service import ModelRegistryService
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
from app.services.segmentation_service import SegmentationService
ROOT = Path(__file__).resolve().parents[2]
@@ -80,18 +82,58 @@ class MissingDependencySegAdapter(AvailableSegAdapter):
return False
class NeverLoadSegAdapter(AvailableSegAdapter):
load_calls = 0
def load_model(self, model_path: Path):
type(self).load_calls += 1
raise AssertionError("unmanifested weights must not reach adapter.load_model")
def _project_and_dataset(dataset_type: str = "raster"):
project_id = uuid4()
dataset_id = uuid4()
source_id = uuid4()
snapshot_id = uuid4()
checksum = "a" * 64
project = Project(id=project_id, name="Mol")
source = SourceRegistry(
id=source_id,
source_key="digitaal_vlaanderen_orthophoto",
display_name="Governed orthophoto test source",
classification="contextual",
authority_name="Digitaal Vlaanderen",
authority_scope_json={"zone": "Flanders", "role": "imagery"},
)
snapshot = SourceSnapshot(
id=snapshot_id,
source_registry_id=source_id,
snapshot_key="configured-segmentation-orthophoto",
checksum_sha256=checksum,
ingest_status="ingested",
freshness_status="current",
)
dataset = Dataset(
id=dataset_id,
project_id=project_id,
name="ortho.tif",
dataset_type=dataset_type,
source="user_upload",
source="digitaal_vlaanderen_orthophoto",
source_name="digitaal_vlaanderen_orthophoto",
storage_path="storage/uploads/ortho.tif",
checksum_sha256=checksum,
source_registry_id=source_id,
source_snapshot_id=snapshot_id,
data_contract_key="geointel.raster.geotiff",
data_contract_version="1.0.0",
validation_status="passed",
provenance_status="complete",
lineage_status="not_applicable",
quarantine_status="not_quarantined",
status="ready",
)
dataset.source_registry = source
dataset.source_snapshot = snapshot
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
return db, project_id, dataset_id
@@ -108,6 +150,102 @@ def _settings(tmp_path: Path, **overrides) -> Settings:
return Settings(**values)
def _write_model_sidecar(
model_path: Path,
*,
model_id: str,
framework: str,
source_version: str | None,
db: FakeSession | None = None,
) -> None:
"""Create explicit local test evidence; no production code creates sidecars."""
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
source_registry_id = uuid4()
source_snapshot_id = uuid4()
resolved_source_version = source_version or "test-v1"
if db is not None:
source_registry = SourceRegistry(
id=source_registry_id,
source_key="model",
display_name="Governed test model artifact",
classification="experimental",
authority_name="GeoIntel test fixture",
freshness_status="current",
ingest_status="configured",
)
source_snapshot = SourceSnapshot(
id=source_snapshot_id,
source_registry_id=source_registry_id,
snapshot_key=f"model-{model_id}-{resolved_source_version}",
source_version=resolved_source_version,
checksum_sha256=model_sha256,
freshness_status="current",
ingest_status="ingested",
)
db.objects[(SourceRegistry, source_registry_id)] = source_registry
db.objects[(SourceSnapshot, source_snapshot_id)] = source_snapshot
payload = {
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
"data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"},
"model": {
"model_id": model_id,
"task_type": "segmentation",
"sha256": model_sha256,
"model_format": "pytorch",
"framework": framework,
"class_mapping": {"0": "segment"},
"source_version": resolved_source_version,
},
"source": {
"source_registry_id": str(source_registry_id),
"source_snapshot_id": str(source_snapshot_id),
"source_registry_key": "model",
"source_snapshot_checksum_sha256": model_sha256,
},
"lineage": {
"upstream_asset_ids": ["test-training-corpus"],
"upstream_checksums_sha256": ["a" * 64],
"transformations": [
{"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64}
],
},
"metadata": {"training_manifest_sha256": "c" * 64},
"imported_at": "2026-08-01T10:00:00+00:00",
}
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text(
json.dumps(payload, sort_keys=True),
encoding="utf-8",
)
def _write_configured_model_sidecars(
tmp_path: Path,
settings: Settings,
*,
include_yolo: bool = True,
include_sam: bool = True,
db: FakeSession | None = None,
) -> None:
if include_yolo:
_write_model_sidecar(
tmp_path / "seg.pt",
model_id=settings.yolo_seg_model_id,
framework="ultralytics/pytorch",
source_version=settings.yolo_seg_model_version,
db=db,
)
if include_sam:
_write_model_sidecar(
tmp_path / "sam.pt",
model_id=settings.sam_model_id,
framework="ultralytics/sam",
source_version=settings.sam_model_version,
db=db,
)
def _manifest(tmp_path: Path, tile_count: int = 1) -> Path:
tiles = []
for index in range(tile_count):
@@ -172,10 +310,31 @@ def test_segmentation_models_report_dependency_unavailable(tmp_path: Path) -> No
assert models["sam-configured"].status == "dependency_unavailable"
def test_segmentation_models_require_runtime_provenance_sidecars(tmp_path: Path) -> None:
(tmp_path / "seg.pt").write_bytes(b"unmanifested yolo segmentation weights")
(tmp_path / "sam.pt").write_bytes(b"unmanifested sam weights")
settings = _settings(tmp_path)
models = {
model.model_id: model
for model in ModelRegistryService.list_segmentation_model_capabilities(
settings=settings,
yolo_seg_adapter_class=AvailableSegAdapter,
sam_adapter_class=ClassAgnosticSamAdapter,
)
}
assert models["yolo-seg-configured"].configured is False
assert models["yolo-seg-configured"].status == "contract_incomplete"
assert models["sam-configured"].configured is False
assert models["sam-configured"].status == "contract_incomplete"
def test_segmentation_models_report_configured_with_local_weights(tmp_path: Path) -> None:
(tmp_path / "seg.pt").write_bytes(b"weights")
(tmp_path / "sam.pt").write_bytes(b"weights")
settings = _settings(tmp_path)
_write_configured_model_sidecars(tmp_path, settings)
models = {
model.model_id: model
@@ -204,6 +363,7 @@ def test_configured_segmentation_requires_tile_manifest(tmp_path: Path) -> None:
(tmp_path / "seg.pt").write_bytes(b"weights")
db, project_id, dataset_id = _project_and_dataset()
settings = _settings(tmp_path)
_write_configured_model_sidecars(tmp_path, settings, include_sam=False, db=db)
with pytest.raises(Exception) as exc_info:
SegmentationService.run_segmentation(
@@ -220,10 +380,60 @@ def test_configured_segmentation_requires_tile_manifest(tmp_path: Path) -> None:
assert getattr(exc_info.value, "code", None) == "SEGMENTATION_TILE_MANIFEST_REQUIRED"
def test_configured_segmentation_fails_before_adapter_load_without_sidecar(tmp_path: Path) -> None:
(tmp_path / "seg.pt").write_bytes(b"unmanifested weights")
db, project_id, dataset_id = _project_and_dataset()
settings = _settings(tmp_path)
NeverLoadSegAdapter.load_calls = 0
response = SegmentationService.run_segmentation(
db=db,
project_id=project_id,
dataset_id=dataset_id,
model_id="yolo-seg-configured",
confidence_threshold=0.5,
tile_manifest_path=str(_manifest(tmp_path)),
settings=settings,
yolo_seg_adapter_class=NeverLoadSegAdapter,
sam_adapter_class=ClassAgnosticSamAdapter,
)
assert response.status == "failed"
assert response.error_code == "SEGMENTATION_MODEL_UNAVAILABLE"
assert NeverLoadSegAdapter.load_calls == 0
def test_configured_segmentation_rejects_unbound_model_snapshot_before_adapter_load(tmp_path: Path) -> None:
(tmp_path / "seg.pt").write_bytes(b"structurally valid but unbound weights")
db, project_id, dataset_id = _project_and_dataset()
settings = _settings(tmp_path)
# The sidecar passes catalog validation but its source registry/snapshot
# was never registered in this production-session fixture.
_write_configured_model_sidecars(tmp_path, settings, include_sam=False)
NeverLoadSegAdapter.load_calls = 0
response = SegmentationService.run_segmentation(
db=db,
project_id=project_id,
dataset_id=dataset_id,
model_id="yolo-seg-configured",
confidence_threshold=0.5,
tile_manifest_path=str(_manifest(tmp_path)),
settings=settings,
yolo_seg_adapter_class=NeverLoadSegAdapter,
sam_adapter_class=ClassAgnosticSamAdapter,
)
assert response.status == "failed"
assert response.error_code == "MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND"
assert NeverLoadSegAdapter.load_calls == 0
def test_configured_yolo_seg_run_persists_georeferenced_masks(tmp_path: Path) -> None:
(tmp_path / "seg.pt").write_bytes(b"weights")
db, project_id, dataset_id = _project_and_dataset()
settings = _settings(tmp_path)
_write_configured_model_sidecars(tmp_path, settings, include_sam=False, db=db)
manifest_path = _manifest(tmp_path)
response = SegmentationService.run_segmentation(
@@ -255,12 +465,14 @@ def test_configured_yolo_seg_run_persists_georeferenced_masks(tmp_path: Path) ->
assert segmentation.area_m2 is not None and segmentation.area_m2 > 0
assert segmentation.provenance_json["inference"] == "local"
assert segmentation.provenance_json["model_id"] == "yolo-seg-configured"
assert segmentation.provenance_json["runtime_model_provenance"]["data_contract_key"] == "geointel.model.pytorch"
def test_configured_sam_run_is_class_agnostic(tmp_path: Path) -> None:
(tmp_path / "sam.pt").write_bytes(b"weights")
db, project_id, dataset_id = _project_and_dataset()
settings = _settings(tmp_path)
_write_configured_model_sidecars(tmp_path, settings, include_yolo=False, db=db)
manifest_path = _manifest(tmp_path)
response = SegmentationService.run_segmentation(
@@ -11,7 +11,7 @@ from shapely.geometry import box
from app.core.errors import AppError
from app.main import app
from app.models import Area, Dataset, Export
from app.models import Area, Dataset, Export, SourceRegistry, SourceSnapshot
from app.schemas.export import ExportCreateResponse
from app.services.export_service import ExportService
from app.services.storage_service import StorageService
@@ -45,18 +45,56 @@ class FakeSession:
return row
def _govern_fixture_dataset(dataset: Dataset) -> Dataset:
source_id = uuid4()
snapshot_id = uuid4()
checksum = "a" * 64
source = SourceRegistry(
id=source_id,
source_key="grb",
display_name="GRB map export test source",
classification="authoritative",
authority_name="Digitaal Vlaanderen",
authority_scope_json={"zone": "Flanders"},
usage_policy_json={"ground_truth_allowed": True},
)
snapshot = SourceSnapshot(
id=snapshot_id,
source_registry_id=source_id,
snapshot_key=f"map-export-grb-{dataset.id}",
checksum_sha256=checksum,
ingest_status="ingested",
freshness_status="current",
)
dataset.source = "grb"
dataset.source_name = "grb"
dataset.checksum_sha256 = checksum
dataset.source_registry_id = source_id
dataset.source_snapshot_id = snapshot_id
dataset.data_contract_key = "geointel.vector.geojson"
dataset.data_contract_version = "1.0.0"
dataset.validation_status = "passed"
dataset.provenance_status = "complete"
dataset.lineage_status = "not_applicable"
dataset.quarantine_status = "not_quarantined"
dataset.status = "ready"
dataset.source_registry = source
dataset.source_snapshot = snapshot
return dataset
def test_vector_selection_geojson_export_persists_handoff_artifact(tmp_path, monkeypatch) -> None:
project_id = uuid4()
dataset_id = uuid4()
export_path = tmp_path / "exports" / "selection.geojson"
dataset = Dataset(
dataset = _govern_fixture_dataset(Dataset(
id=dataset_id,
project_id=project_id,
name="candidate.geojson",
dataset_type="vector",
source="fixture",
status="ready",
)
))
db = FakeSession({(Dataset, dataset_id): dataset})
selection_bbox = {"min_x": 4.9, "min_y": 50.9, "max_x": 5.2, "max_y": 51.2, "crs": "EPSG:4326"}
selection_payload = {
@@ -135,14 +173,14 @@ def test_vector_selection_export_uses_exact_area_scope_when_requested(tmp_path,
project_id = uuid4()
dataset_id = uuid4()
area_id = uuid4()
dataset = Dataset(
dataset = _govern_fixture_dataset(Dataset(
id=dataset_id,
project_id=project_id,
name="regional-buildings.geojson",
dataset_type="vector",
source="fixture",
status="ready",
)
))
area_shape = box(5.0, 51.1, 5.2, 51.3)
area_geometry = from_shape(area_shape, srid=4326)
area = SimpleNamespace(id=area_id, project_id=project_id, name="Gemeente Mol", geometry=area_geometry)
@@ -184,7 +222,7 @@ def test_area_constrained_bbox_uses_intersection_and_disables_full_area_fast_pat
project_id = uuid4()
dataset_id = uuid4()
area_id = uuid4()
dataset = Dataset(
dataset = _govern_fixture_dataset(Dataset(
id=dataset_id,
project_id=project_id,
area_id=area_id,
@@ -193,7 +231,7 @@ def test_area_constrained_bbox_uses_intersection_and_disables_full_area_fast_pat
source="fixture",
source_metadata={"geometry_clipped_to_area": True},
status="ready",
)
))
area_shape = box(5.0, 51.0, 5.2, 51.2)
area = SimpleNamespace(
id=area_id,
@@ -7,7 +7,7 @@ from uuid import uuid4
from fastapi.testclient import TestClient
from app.main import app
from app.models import Dataset, VectorFeature
from app.models import Dataset
from app.schemas.dataset import DatasetCreateResponse
from app.services.storage_service import StorageService
from app.services.vector_feature_service import VectorFeatureService
@@ -124,7 +124,7 @@ def test_vector_selection_derive_persists_queryable_derived_dataset(tmp_path, mo
assert response.provenance_metadata["source_dataset_id"] == str(dataset_id)
assert response.provenance_metadata["source_table"] == "vector_features"
assert persisted_features[0]["dataset_id"] == derived.id
assert persisted_features[0]["commit"] is True
assert persisted_features[0]["commit"] is False
derived_payload = json.loads(output_path.read_text(encoding="utf-8"))
props = derived_payload["features"][0]["properties"]
assert props["source_vector_feature_id"] == "source-row-1"
@@ -75,5 +75,7 @@ def test_operator_yolo_train_smoke_script_contract() -> None:
assert '"dataset_summary_sha256"' in script
assert '"base_model_sha256"' in script
assert '"trained_model_sha256"' in script
assert "training_release_manifest.py" in script
assert "verify" in script
assert "download" not in script.lower()
assert "fixture_mode" not in script
+69 -4
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
from hashlib import sha256
import json
import subprocess
import sys
@@ -9,6 +10,7 @@ from fastapi.testclient import TestClient
from app.core.config import Settings
from app.main import app
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
from app.services.yolo_preflight_service import YoloPreflightService
@@ -57,6 +59,43 @@ def _manifest(tmp_path: Path, tile_count: int = 1) -> Path:
return manifest_path
def _write_model_sidecar(model_path: Path, settings: Settings) -> None:
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
payload = {
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
"data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"},
"model": {
"model_id": settings.yolo_model_id,
"task_type": "object_detection",
"sha256": model_sha256,
"model_format": "pytorch",
"framework": "ultralytics/pytorch",
"class_mapping": {"0": "building"},
"source_version": settings.yolo_model_version or "test-v1",
},
"source": {
"source_registry_id": "11111111-1111-4111-8111-111111111111",
"source_snapshot_id": "22222222-2222-4222-8222-222222222222",
"source_registry_key": "model",
"source_snapshot_checksum_sha256": model_sha256,
},
"lineage": {
"upstream_asset_ids": ["test-training-corpus"],
"upstream_checksums_sha256": ["a" * 64],
"transformations": [
{"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64}
],
},
"metadata": {"training_manifest_sha256": "c" * 64},
"imported_at": "2026-08-01T10:00:00+00:00",
}
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text(
json.dumps(payload, sort_keys=True),
encoding="utf-8",
)
def test_yolo_preflight_reports_disabled_without_loading_model(tmp_path: Path, monkeypatch) -> None:
monkeypatch.setenv("YOLO_CONFIG_DIR", str(tmp_path / "ultralytics"))
@@ -99,9 +138,11 @@ def test_yolo_preflight_validates_model_and_manifest_without_importing_yolo(tmp_
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"weights")
manifest_path = _manifest(tmp_path, tile_count=2)
settings = Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4)
_write_model_sidecar(model_path, settings)
result = YoloPreflightService.run(
settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4),
settings=settings,
tile_manifest_path=str(manifest_path),
yolo_adapter_class=AvailableAdapter,
)
@@ -109,6 +150,7 @@ def test_yolo_preflight_validates_model_and_manifest_without_importing_yolo(tmp_
assert result["status"] == "ready"
assert result["checks"]["dependencies_available"] is True
assert result["checks"]["model_file_exists"] is True
assert result["checks"]["model_provenance_valid"] is True
assert result["checks"]["manifest_valid"] is True
assert result["tile_count"] == 2
assert result["will_download_models"] is False
@@ -120,9 +162,11 @@ def test_yolo_preflight_marks_assumed_dependencies_in_runtime_details(tmp_path:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"weights")
manifest_path = _manifest(tmp_path, tile_count=1)
settings = Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4)
_write_model_sidecar(model_path, settings)
result = YoloPreflightService.run(
settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4),
settings=settings,
tile_manifest_path=str(manifest_path),
yolo_adapter_class=MissingDependencyAdapter,
assume_dependencies=True,
@@ -138,9 +182,11 @@ def test_yolo_preflight_can_explicitly_smoke_load_local_model(tmp_path: Path) ->
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"weights")
manifest_path = _manifest(tmp_path)
settings = Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4)
_write_model_sidecar(model_path, settings)
result = YoloPreflightService.run(
settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4),
settings=settings,
tile_manifest_path=str(manifest_path),
yolo_adapter_class=AvailableAdapter,
check_model_load=True,
@@ -156,9 +202,11 @@ def test_yolo_preflight_can_explicitly_smoke_load_local_model(tmp_path: Path) ->
def test_yolo_preflight_reports_explicit_model_load_failure(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"weights")
settings = Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4)
_write_model_sidecar(model_path, settings)
result = YoloPreflightService.run(
settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4),
settings=settings,
tile_manifest_path=str(_manifest(tmp_path)),
yolo_adapter_class=FailingLoadAdapter,
check_model_load=True,
@@ -173,6 +221,7 @@ def test_yolo_preflight_script_outputs_json(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"weights")
manifest_path = _manifest(tmp_path)
_write_model_sidecar(model_path, Settings(yolo_enabled=True, yolo_model_path=str(model_path)))
result = subprocess.run(
[
@@ -201,6 +250,7 @@ def test_yolo_preflight_script_uses_environment_configuration(tmp_path: Path, mo
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"weights")
manifest_path = _manifest(tmp_path)
_write_model_sidecar(model_path, Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4))
monkeypatch.setenv("YOLO_ENABLED", "true")
monkeypatch.setenv("YOLO_MODEL_PATH", str(model_path))
monkeypatch.setenv("YOLO_MAX_TILES", "4")
@@ -227,6 +277,21 @@ def test_yolo_preflight_script_uses_environment_configuration(tmp_path: Path, mo
assert payload["max_tiles"] == 4
def test_yolo_preflight_refuses_unmanifested_local_weights(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"unmanifested weights")
result = YoloPreflightService.run(
settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path)),
tile_manifest_path=str(_manifest(tmp_path)),
yolo_adapter_class=AvailableAdapter,
)
assert result["status"] == "contract_incomplete"
assert result["checks"]["model_provenance_valid"] is False
assert result["error_code"] == "MODEL_PROVENANCE_MANIFEST_MISSING"
def test_yolo_preflight_script_rejects_assumed_dependencies_for_model_load(tmp_path: Path) -> None:
result = subprocess.run(
[
@@ -8,7 +8,7 @@ from fastapi.testclient import TestClient
from app.core.errors import AppError
from app.main import app
from app.models import Area, Dataset, Export, Project, QualityCheck
from app.models import Area, Dataset, Export, Project, QualityCheck, SourceRegistry, SourceSnapshot
from app.schemas.export import ExportCreateResponse
from app.services.export_service import ExportService
from app.services.storage_service import StorageService
@@ -66,13 +66,57 @@ class FakeSession:
return row
def _govern_fixture_dataset(dataset: Dataset) -> Dataset:
"""Give an export fixture a governed authoritative source identity.
Export is a production boundary: test data must model a source that could
cross it, rather than using the deliberately QA-only ``fixture`` source.
"""
source_id = uuid4()
snapshot_id = uuid4()
checksum = "a" * 64
source = SourceRegistry(
id=source_id,
source_key="grb",
display_name="GRB export test source",
classification="authoritative",
authority_name="Digitaal Vlaanderen",
authority_scope_json={"zone": "Flanders"},
usage_policy_json={"ground_truth_allowed": True},
)
snapshot = SourceSnapshot(
id=snapshot_id,
source_registry_id=source_id,
snapshot_key=f"export-grb-{dataset.id}",
checksum_sha256=checksum,
ingest_status="ingested",
freshness_status="current",
)
dataset.source = "grb"
dataset.source_name = "grb"
dataset.checksum_sha256 = checksum
dataset.source_registry_id = source_id
dataset.source_snapshot_id = snapshot_id
dataset.data_contract_key = "geointel.vector.geojson"
dataset.data_contract_version = "1.0.0"
dataset.validation_status = "passed"
dataset.provenance_status = "complete"
dataset.lineage_status = "not_applicable"
dataset.quarantine_status = "not_quarantined"
dataset.status = "ready"
dataset.source_registry = source
dataset.source_snapshot = snapshot
return dataset
def test_dataset_geojson_export_persists_export_and_writes_artifact(tmp_path, monkeypatch) -> None:
project_id = uuid4()
dataset_id = uuid4()
dataset_path = tmp_path / "input.geojson"
dataset_path.write_text(json.dumps({"type": "FeatureCollection", "features": []}), encoding="utf-8")
export_path = tmp_path / "exports" / "buildings.geojson"
dataset = Dataset(
dataset = _govern_fixture_dataset(Dataset(
id=dataset_id,
project_id=project_id,
name="buildings.geojson",
@@ -80,7 +124,7 @@ def test_dataset_geojson_export_persists_export_and_writes_artifact(tmp_path, mo
source="fixture",
storage_path=str(dataset_path),
status="ready",
)
))
db = FakeSession({(Dataset, dataset_id): dataset})
monkeypatch.setattr(StorageService, "dataset_export_path", lambda *_args: str(export_path))
@@ -18,7 +18,7 @@ from app.core.config import Settings
from app.core.errors import AppError
from app.db.session import get_db
from app.main import app
from app.models import Area, Dataset, DatasetVersion, Job, Project
from app.models import Area, Dataset, DatasetVersion, Job, Project, SourceRegistry, SourceSnapshot
from app.schemas.orthophoto import OrthophotoAcquireRequest
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
@@ -41,6 +41,15 @@ class FakeSession:
def add(self, row):
self.added.append(row)
def flush(self):
# The governed importer persists source identities and immutable
# snapshots before the Dataset. Mirror the database-generated UUIDs
# so this harness exercises that Phase 2 path rather than the legacy
# no-registry fallback.
for row in self.added:
if getattr(row, "id", None) is None:
row.id = uuid4()
def commit(self):
return None
@@ -50,13 +59,23 @@ class FakeSession:
def refresh(self, row):
return row
def query(self, _model):
return FakeQuery(self.query_result)
def query(self, model):
rows = [
row
for (row_model, _row_id), row in self.rows.items()
if row_model is model and isinstance(row, model)
]
rows.extend(row for row in self.added if isinstance(row, model))
if isinstance(self.query_result, model):
rows.append(self.query_result)
elif isinstance(self.query_result, list):
rows.extend(row for row in self.query_result if isinstance(row, model))
return FakeQuery(rows)
class FakeQuery:
def __init__(self, result):
self.result = result
def __init__(self, results):
self.results = list(results)
def filter(self, *_args):
return self
@@ -65,7 +84,10 @@ class FakeQuery:
return self
def first(self):
return self.result
return self.results[0] if self.results else None
def one_or_none(self):
return self.first()
class FakeImageResponse:
@@ -222,12 +244,23 @@ def test_regional_orthophoto_products_bind_provider_and_governed_scope(
)
dataset = next(row for row in db.added if isinstance(row, Dataset))
source = next(row for row in db.added if isinstance(row, SourceRegistry))
snapshot = next(row for row in db.added if isinstance(row, SourceSnapshot))
assert result["provider"] == provider
assert result["layer"] == layer
assert dataset.source_name == provider
assert dataset.source_metadata["coverage_zone"] == coverage_zone
assert dataset.source_metadata["license_note"]
assert dataset.provenance_metadata["request_url"].startswith(prepared["product"].wms_url)
assert dataset.source_registry_id == source.id
assert dataset.source_snapshot_id == snapshot.id
assert dataset.validation_status == "passed"
assert dataset.provenance_status == "complete"
assert dataset.quarantine_status == "not_quarantined"
assert snapshot.source_registry_id == source.id
assert snapshot.checksum_sha256 == dataset.checksum_sha256
assert snapshot.ingest_status == "ingested"
assert snapshot.freshness_status == "current"
prepared = OrthophotoAcquisitionService._prepared_request(_selection_payload(product_key="1971"), settings)
assert prepared["params"]["LAYERS"] == "OKZPAN71VL"
@@ -291,6 +324,8 @@ def test_orthophoto_acquisition_persists_georeferenced_raster_and_provenance(tmp
assert len(datasets) == 1
assert len(versions) == 1
dataset = datasets[0]
source = next(row for row in db.added if isinstance(row, SourceRegistry))
snapshot = next(row for row in db.added if isinstance(row, SourceSnapshot))
assert result["output_dataset_id"] == str(dataset.id)
assert result["reused"] is False
assert dataset.project_id == project_id
@@ -301,6 +336,15 @@ def test_orthophoto_acquisition_persists_georeferenced_raster_and_provenance(tmp
assert dataset.crs == "EPSG:31370"
assert dataset.provenance_metadata["acquisition"] == "explicit_bounded_map_selection"
assert dataset.provenance_metadata["request_hash"] == prepared["request_hash"]
assert dataset.source_registry_id == source.id
assert dataset.source_snapshot_id == snapshot.id
assert dataset.validation_status == "passed"
assert dataset.provenance_status == "complete"
assert dataset.lineage_status == "not_applicable"
assert dataset.quarantine_status == "not_quarantined"
assert snapshot.source_registry_id == source.id
assert snapshot.checksum_sha256 == dataset.checksum_sha256
assert snapshot.freshness_status == "current"
assert dataset.source_metadata["attribution"].startswith("Bron: Orthofotomozaiek Vlaanderen")
assert dataset.storage_path is not None
with rasterio.open(dataset.storage_path) as stored:
+40 -7
View File
@@ -17,7 +17,7 @@ from app.core.config import Settings
from app.core.errors import AppError
from app.db.session import get_db
from app.main import app
from app.models import Area, Dataset, DatasetVersion, Job, Project
from app.models import Area, Dataset, DatasetVersion, Job, Project, SourceRegistry, SourceSnapshot
from app.schemas.dhmv import DhmvAcquireRequest, TerrainPartitionSelectionRequest, TerrainSelectionRequest
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
from app.services.terrain_analysis_service import TerrainAnalysisService
@@ -27,8 +27,8 @@ ROOT = Path(__file__).resolve().parents[2]
class FakeQuery:
def __init__(self, result=None):
self.result = result
def __init__(self, results=None):
self.results = list(results or [])
def filter(self, *_args):
return self
@@ -37,10 +37,13 @@ class FakeQuery:
return self
def first(self):
return self.result
return self.results[0] if self.results else None
def one_or_none(self):
return self.first()
def all(self):
return self.result if isinstance(self.result, list) else []
return list(self.results)
class FakeSession:
@@ -58,6 +61,14 @@ class FakeSession:
def add(self, row):
self.added.append(row)
def flush(self):
# Exercise the governed source/snapshot import path with database-like
# primary-key assignment instead of silently falling back to legacy
# fixture behavior.
for row in self.added:
if getattr(row, "id", None) is None:
row.id = uuid4()
def commit(self):
return None
@@ -67,8 +78,18 @@ class FakeSession:
def refresh(self, row):
return row
def query(self, _model):
return FakeQuery(self.query_result)
def query(self, model):
rows = [
row
for (row_model, _row_id), row in self.rows.items()
if row_model is model and isinstance(row, model)
]
rows.extend(row for row in self.added if isinstance(row, model))
if isinstance(self.query_result, model):
rows.append(self.query_result)
elif isinstance(self.query_result, list):
rows.extend(row for row in self.query_result if isinstance(row, model))
return FakeQuery(rows)
class FakeResponse:
@@ -318,6 +339,8 @@ def test_dhmv_acquisition_clips_validates_and_persists_via_dataset_service(tmp_p
dataset = next(item for item in db.added if isinstance(item, Dataset))
version = next(item for item in db.added if isinstance(item, DatasetVersion))
source = next(item for item in db.added if isinstance(item, SourceRegistry))
snapshot = next(item for item in db.added if isinstance(item, SourceSnapshot))
assert result["output_dataset_id"] == str(dataset.id)
assert dataset.source_name == "digitaal_vlaanderen_dhmv"
assert dataset.area_id == area_id
@@ -331,6 +354,16 @@ def test_dhmv_acquisition_clips_validates_and_persists_via_dataset_service(tmp_p
assert dataset.provenance_metadata["water_depth_available"] is False
assert dataset.provenance_metadata["water_volume_available"] is False
assert len(dataset.provenance_metadata["response_sha256"]) == 64
assert dataset.source_registry_id == source.id
assert dataset.source_snapshot_id == snapshot.id
assert dataset.validation_status == "passed"
assert dataset.provenance_status == "complete"
assert dataset.lineage_status == "not_applicable"
assert dataset.quarantine_status == "not_quarantined"
assert snapshot.source_registry_id == source.id
assert snapshot.checksum_sha256 == dataset.checksum_sha256
assert snapshot.ingest_status == "ingested"
assert snapshot.freshness_status == "current"
with rasterio.open(dataset.storage_path) as stored:
assert stored.crs.to_epsg() == 31370
assert stored.count == 1
@@ -1,8 +1,9 @@
from __future__ import annotations
import json
from datetime import UTC, datetime
from hashlib import sha256
from pathlib import Path
from types import SimpleNamespace
from uuid import uuid4
import pytest
@@ -10,12 +11,13 @@ from fastapi.testclient import TestClient
from pydantic import ValidationError
from app.main import app
from app.models import Dataset, Export
from app.models import Dataset, Export, SourceRegistry, SourceSnapshot
from app.schemas.export import ExportCreateResponse, MapResultExportRequest
from app.schemas.project import ProjectRead
from app.services.export_service import ExportService
from app.services.project_service import ProjectService
from app.services.storage_service import StorageService
from app.services.source_registry_service import SourceRegistryService
from app.services.temporal_analysis_service import TemporalAnalysisService
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
@@ -48,6 +50,71 @@ def bbox_payload() -> dict:
}
def governed_dataset(
*,
project_id,
dataset_id,
name: str,
dataset_type: str,
source_key: str,
dataset_role: str = "source",
) -> Dataset:
"""Build an in-memory stand-in for a passed governed dataset.
Map-result export is an operational consumption boundary. These tests
must therefore model the same source registry/snapshot, checksum and
passed-contract evidence supplied by a real adapter rather than relying
on an old transient Dataset fixture.
"""
source = SourceRegistry(
id=uuid4(),
**SourceRegistryService.definition_for(source_key).as_model_values(),
)
checksum = sha256(f"{dataset_id}:{source_key}:{dataset_type}".encode("utf-8")).hexdigest()
snapshot = SourceSnapshot(
id=uuid4(),
source_registry_id=source.id,
snapshot_key=f"test:{source_key}:{checksum}",
checksum_sha256=checksum,
fetched_at=datetime.now(UTC),
crs="EPSG:31370",
units=source.default_units,
spatial_resolution_json={"x": 1.0, "y": 1.0, "unit": "m"},
temporal_coverage_json={"status": "test-fixture"},
geographic_coverage_json={"zone": "Flanders"},
observed_schema_json={"dataset_type": dataset_type},
freshness_status="current",
ingest_status="ingested",
known_limitations_json=["In-memory governed fixture used only by this export test."],
snapshot_metadata_json={"fixture_mode": True},
)
return Dataset(
id=dataset_id,
project_id=project_id,
name=name,
dataset_type=dataset_type,
source="governed test fixture",
dataset_role=dataset_role,
source_name=source.source_key,
source_registry_id=source.id,
source_snapshot_id=snapshot.id,
source_registry=source,
source_snapshot=snapshot,
data_contract_key=("geointel.raster.geotiff" if dataset_type == "raster" else "geointel.vector.geojson"),
data_contract_version="1.0.0",
validation_status="passed",
provenance_status="complete",
lineage_status="not_applicable",
quarantine_status="not_quarantined",
checksum_sha256=checksum,
metadata_json={"fixture_mode": True},
source_metadata={"fixture_mode": True, "source_registry_key": source.source_key},
provenance_metadata={"fixture_mode": True, "source_snapshot_id": str(snapshot.id)},
status="ready",
)
def test_map_result_export_request_requires_a_complete_target() -> None:
with pytest.raises(ValidationError):
MapResultExportRequest(project_id=uuid4(), mode="current", bbox=bbox_payload())
@@ -67,13 +134,12 @@ def test_current_vector_map_result_uses_authoritative_selection_export(monkeypat
project_id = uuid4()
dataset_id = uuid4()
area_id = uuid4()
dataset = Dataset(
id=dataset_id,
dataset = governed_dataset(
project_id=project_id,
dataset_id=dataset_id,
name="buildings.geojson",
dataset_type="vector",
source="fixture",
status="ready",
source_key="grb",
)
db = FakeSession({(Dataset, dataset_id): dataset})
expected = ExportCreateResponse(
@@ -109,14 +175,12 @@ def test_current_vector_map_result_uses_authoritative_selection_export(monkeypat
def test_partitioned_vector_map_result_uses_governed_partition_export(monkeypatch) -> None:
project_id = uuid4()
dataset_id = uuid4()
dataset = Dataset(
id=dataset_id,
dataset = governed_dataset(
project_id=project_id,
dataset_id=dataset_id,
name="vha-municipality.geojson",
dataset_type="vector",
source="VHA",
source_name="vmm_vha_bathymetry_profiles",
status="ready",
source_key="vmm_vha_bathymetry_profiles",
)
db = FakeSession({(Dataset, dataset_id): dataset})
expected = ExportCreateResponse(
@@ -159,14 +223,12 @@ def test_partitioned_vector_map_result_uses_governed_partition_export(monkeypatc
def test_raster_map_result_is_recomputed_and_persisted(tmp_path, monkeypatch) -> None:
project_id = uuid4()
dataset_id = uuid4()
dataset = Dataset(
id=dataset_id,
dataset = governed_dataset(
project_id=project_id,
dataset_id=dataset_id,
name="space-occupation.tif",
dataset_type="raster",
source="official",
source_name="department_omgeving_thematic_raster",
status="ready",
source_key="department_omgeving_thematic_raster",
)
db = FakeSession({(Dataset, dataset_id): dataset})
export_path = tmp_path / "space-occupation-analysis.json"
@@ -209,7 +271,26 @@ def test_evolution_map_result_is_recomputed_and_persisted(tmp_path, monkeypatch)
project_id = uuid4()
earlier_id = uuid4()
later_id = uuid4()
db = FakeSession()
earlier_dataset = governed_dataset(
project_id=project_id,
dataset_id=earlier_id,
name="forest-earlier.geojson",
dataset_type="vector",
source_key="inbo_bwk_natura2000",
)
later_dataset = governed_dataset(
project_id=project_id,
dataset_id=later_id,
name="forest-later.geojson",
dataset_type="vector",
source_key="inbo_bwk_natura2000",
)
db = FakeSession(
{
(Dataset, earlier_id): earlier_dataset,
(Dataset, later_id): later_dataset,
}
)
export_path = tmp_path / "forest-evolution.json"
captured: dict = {}
@@ -6,6 +6,7 @@ from uuid import uuid4
import pytest
from geoalchemy2.shape import to_shape
from pyproj import Transformer
from app.api.routes.qa import compare_candidate_with_reference
from app.models import Dataset, Metric, Project, QualityCheck, VectorFeature
@@ -117,6 +118,54 @@ def test_vector_feature_service_normalizes_source_z_coordinates_to_canonical_2d(
assert to_shape(persisted[0].geometry).has_z is False
def test_vector_feature_service_transforms_declared_source_crs_before_epsg4326_storage() -> None:
to_lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
x, y = to_lambert.transform(4.7, 51.1)
db = FakeSession()
persisted = VectorFeatureService.persist_geojson_features(
db=db,
dataset_id=uuid4(),
payload={
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"id": "lambert-point",
"properties": {},
"geometry": {"type": "Point", "coordinates": [x, y]},
}
],
},
source_crs="EPSG:31370",
)
geometry = to_shape(persisted[0].geometry)
assert geometry.x == pytest.approx(4.7, abs=0.000001)
assert geometry.y == pytest.approx(51.1, abs=0.000001)
def test_vector_feature_service_rejects_invalid_declared_source_crs() -> None:
with pytest.raises(Exception) as exc_info:
VectorFeatureService.persist_geojson_features(
db=FakeSession(),
dataset_id=uuid4(),
payload={
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {},
"geometry": {"type": "Point", "coordinates": [4.7, 51.1]},
}
],
},
source_crs="EPSG:not-a-crs",
)
assert getattr(exc_info.value, "code", None) == "INVALID_DATASET_CRS"
def test_dataset_upload_persists_vector_features(monkeypatch, tmp_path) -> None:
project_id = uuid4()
db = FakeSession(objects={(Project, project_id): Project(id=project_id, name="Geel")})
@@ -43,7 +43,7 @@ def _project_and_dataset(dataset_type: str = "raster"):
project_id=project_id,
name="source.tif",
dataset_type=dataset_type,
source="user_upload",
source="test",
storage_path="storage/uploads/source.tif",
)
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
+191 -3
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
from hashlib import sha256
import json
from pathlib import Path
import sys
@@ -10,10 +11,11 @@ import pytest
from app.core.config import Settings
from app.core.errors import AppError
from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project
from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project, SourceRegistry, SourceSnapshot
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
from app.services.detection_service import DetectionService
from app.services.model_registry_service import ModelRegistryService
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
from app.services.yolo_adapter import YoloDetectionAdapter
ROOT = Path(__file__).resolve().parents[2]
@@ -79,6 +81,14 @@ class MockYoloAdapter:
]
class NeverLoadUnboundModelAdapter(MockYoloAdapter):
load_calls = 0
def load_model(self, model_path: Path):
type(self).load_calls += 1
raise AssertionError("unbound model provenance must be rejected before adapter.load_model")
class MixedCaseYoloAdapter(MockYoloAdapter):
def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict]:
return [
@@ -141,15 +151,47 @@ class ExplodingPredictModel:
def _project_and_dataset(dataset_type: str = "raster"):
project_id = uuid4()
dataset_id = uuid4()
source_registry_id = uuid4()
source_snapshot_id = uuid4()
checksum = "a" * 64
project = Project(id=project_id, name="Geel")
source_registry = SourceRegistry(
id=source_registry_id,
source_key="test-derived-raster",
display_name="Governed test-derived raster",
classification="derived",
authority_name="GeoIntel test fixture",
usage_policy_json={"ground_truth_allowed": False},
)
source_snapshot = SourceSnapshot(
id=source_snapshot_id,
source_registry_id=source_registry_id,
snapshot_key="test-derived-raster-v1",
checksum_sha256=checksum,
freshness_status="current",
ingest_status="ingested",
)
dataset = Dataset(
id=dataset_id,
project_id=project_id,
name="source.tif",
dataset_type=dataset_type,
source="user_upload",
source="test-derived-raster",
source_name="test-derived-raster",
storage_path="storage/uploads/source.tif",
checksum_sha256=checksum,
source_registry_id=source_registry_id,
source_snapshot_id=source_snapshot_id,
data_contract_key="geointel.raster.geotiff",
data_contract_version="1.0.0",
validation_status="passed",
provenance_status="complete",
lineage_status="not_applicable",
quarantine_status="not_quarantined",
status="ready",
)
dataset.source_registry = source_registry
dataset.source_snapshot = source_snapshot
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
return db, project_id, dataset_id
@@ -165,6 +207,74 @@ def _settings(tmp_path: Path, **overrides) -> Settings:
return Settings(**values)
def _write_model_sidecar(
model_path: Path,
settings: Settings,
*,
db: FakeSession | None = None,
) -> None:
"""Create explicit test-only evidence; production never self-generates it."""
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
source_registry_id = uuid4()
source_snapshot_id = uuid4()
source_version = settings.yolo_model_version or "test-v1"
if db is not None:
source_registry = SourceRegistry(
id=source_registry_id,
source_key="model",
display_name="Governed test model artifact",
classification="experimental",
authority_name="GeoIntel test fixture",
freshness_status="current",
ingest_status="configured",
)
source_snapshot = SourceSnapshot(
id=source_snapshot_id,
source_registry_id=source_registry_id,
snapshot_key=f"model-{source_version}",
source_version=source_version,
checksum_sha256=model_sha256,
freshness_status="current",
ingest_status="ingested",
)
db.objects[(SourceRegistry, source_registry_id)] = source_registry
db.objects[(SourceSnapshot, source_snapshot_id)] = source_snapshot
payload = {
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
"data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"},
"model": {
"model_id": settings.yolo_model_id,
"task_type": "object_detection",
"sha256": model_sha256,
"model_format": "pytorch",
"framework": "ultralytics/pytorch",
"class_mapping": {"0": "building"},
"source_version": source_version,
},
"source": {
"source_registry_id": str(source_registry_id),
"source_snapshot_id": str(source_snapshot_id),
"source_registry_key": "model",
"source_snapshot_checksum_sha256": model_sha256,
},
"lineage": {
"upstream_asset_ids": ["test-training-corpus"],
"upstream_checksums_sha256": ["a" * 64],
"transformations": [
{"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64}
],
},
"metadata": {"training_manifest_sha256": "c" * 64},
"imported_at": "2026-08-01T10:00:00+00:00",
}
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text(
json.dumps(payload, sort_keys=True),
encoding="utf-8",
)
def _manifest(tmp_path: Path, tile_count: int = 1) -> Path:
tiles = []
for index in range(tile_count):
@@ -223,10 +333,28 @@ def test_yolo_configured_model_reports_dependency_unavailable(tmp_path: Path) ->
assert model.status == "dependency_unavailable"
def test_yolo_configured_model_requires_a_runtime_provenance_sidecar(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"unmanifested local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path))
model = ModelRegistryService.get_model_capability(
"yolo-configured",
settings=settings,
yolo_adapter_class=AvailableAdapter,
)
assert model is not None
assert model.configured is False
assert model.status == "contract_incomplete"
assert "sidecar" in model.limitation_message
def test_yolo_configured_model_reports_configured_with_local_model_and_dependencies(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path))
_write_model_sidecar(model_path, settings)
model = ModelRegistryService.get_model_capability("yolo-configured", settings=settings, yolo_adapter_class=AvailableAdapter)
@@ -306,11 +434,37 @@ def test_yolo_run_requires_tile_manifest_path(tmp_path: Path) -> None:
assert getattr(exc_info.value, "code", None) == "DETECTION_TILE_MANIFEST_REQUIRED"
def test_yolo_run_fails_closed_before_adapter_load_without_sidecar(tmp_path: Path) -> None:
db, project_id, dataset_id = _project_and_dataset()
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"unmanifested local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path))
# AvailableAdapter intentionally has no load_model method. If runtime
# provenance were checked after adapter loading, this would raise instead
# of returning the explicit unavailable capability state.
result = DetectionService.run_detection(
db=db,
project_id=project_id,
dataset_id=dataset_id,
model_id="yolo-configured",
confidence_threshold=0.5,
tile_manifest_path=str(_manifest(tmp_path)),
settings=settings,
yolo_adapter_class=AvailableAdapter,
)
assert result.status == "failed"
assert result.error_code == "DETECTION_MODEL_UNAVAILABLE"
assert "sidecar" in result.message
def test_yolo_run_rejects_manifest_over_tile_limit(tmp_path: Path) -> None:
db, project_id, dataset_id = _project_and_dataset()
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_max_tiles=1)
_write_model_sidecar(model_path, settings, db=db)
manifest_path = _manifest(tmp_path, tile_count=2)
result = DetectionService.run_detection(
@@ -333,6 +487,7 @@ def test_yolo_run_rejects_missing_tile_manifest_file(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path))
_write_model_sidecar(model_path, settings, db=db)
result = DetectionService.run_detection(
db=db,
@@ -354,6 +509,7 @@ def test_yolo_run_rejects_invalid_tile_manifest_json(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path))
_write_model_sidecar(model_path, settings, db=db)
manifest_path = tmp_path / "manifest.json"
manifest_path.write_text("{not-json", encoding="utf-8")
@@ -372,6 +528,32 @@ def test_yolo_run_rejects_invalid_tile_manifest_json(tmp_path: Path) -> None:
assert result.error_code == "DETECTION_TILE_MANIFEST_INVALID"
def test_yolo_run_rejects_unbound_model_snapshot_before_adapter_load(tmp_path: Path) -> None:
db, project_id, dataset_id = _project_and_dataset()
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"structurally valid but unbound model")
settings = _settings(tmp_path, yolo_model_path=str(model_path))
# A catalog/preflight sidecar alone is deliberately insufficient for a
# production call. Do not register the declared source IDs in ``db``.
_write_model_sidecar(model_path, settings)
NeverLoadUnboundModelAdapter.load_calls = 0
result = DetectionService.run_detection(
db=db,
project_id=project_id,
dataset_id=dataset_id,
model_id="yolo-configured",
confidence_threshold=0.5,
tile_manifest_path=str(_manifest(tmp_path)),
settings=settings,
yolo_adapter_class=NeverLoadUnboundModelAdapter,
)
assert result.status == "failed"
assert result.error_code == "MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND"
assert NeverLoadUnboundModelAdapter.load_calls == 0
def test_pixel_bbox_to_epsg4326_polygon_from_gdal_transform() -> None:
polygon = pixel_bbox_to_epsg4326_polygon(
bbox=[10, 20, 30, 40],
@@ -390,6 +572,7 @@ def test_yolo_run_persists_mocked_georeferenced_detections(tmp_path: Path) -> No
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_model_version="local-test")
_write_model_sidecar(model_path, settings, db=db)
manifest_path = _manifest(tmp_path, tile_count=1)
result = DetectionService.run_detection(
@@ -416,7 +599,10 @@ def test_yolo_run_persists_mocked_georeferenced_detections(tmp_path: Path) -> No
assert detections[0].confidence == 0.91
assert detections[0].source_tile_path.endswith("tile_0000.tif")
assert detections[0].bbox_json == {"x_min": 10.0, "y_min": 20.0, "x_max": 30.0, "y_max": 40.0}
assert detections[0].properties_json == {"adapter": "mock", "tile_index": 0}
assert detections[0].properties_json["adapter"] == "mock"
assert detections[0].properties_json["tile_index"] == 0
assert detections[0].properties_json["runtime_model_provenance"]["model_sha256"] == sha256(model_path.read_bytes()).hexdigest()
assert runs[0].parameters_json["runtime_model_provenance"]["data_contract_key"] == "geointel.model.pytorch"
assert runs[0].status == "success"
assert jobs[0].status == "success"
@@ -426,6 +612,7 @@ def test_yolo_class_filter_is_case_insensitive_and_persists_canonical_class(tmp_
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path))
_write_model_sidecar(model_path, settings, db=db)
manifest_path = _manifest(tmp_path, tile_count=1)
result = DetectionService.run_detection(
@@ -453,6 +640,7 @@ def test_yolo_run_suppresses_cross_tile_duplicate_detections(tmp_path: Path) ->
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_duplicate_iou_threshold=0.5)
_write_model_sidecar(model_path, settings, db=db)
manifest_path = _manifest(tmp_path, tile_count=2)
result = DetectionService.run_detection(
@@ -12,7 +12,7 @@ from shapely.geometry import Polygon, box
from app.core.errors import AppError
from app.main import app
from app.db.session import get_db
from app.models import AnalysisRun, Dataset, Detection, Metric, QualityCheck, VectorFeature
from app.models import AnalysisRun, Dataset, Detection, Metric, QualityCheck, SourceRegistry, SourceSnapshot, VectorFeature
from app.services.detection_service import DetectionService
@@ -96,11 +96,52 @@ def _source_dataset(project_id, dataset_id):
project_id=project_id,
name="source.tif",
dataset_type="raster",
source="manual",
source_name="manual",
source="test",
source_name="test",
)
def _authoritative_reference(dataset: Dataset) -> Dataset:
"""Model the reference as a fully governed GRB fixture, never test data."""
source_id = uuid4()
snapshot_id = uuid4()
checksum = "a" * 64
source = SourceRegistry(
id=source_id,
source_key="grb",
display_name="GRB QA fixture",
classification="authoritative",
authority_name="Digitaal Vlaanderen",
authority_scope_json={"zone": "Flanders"},
usage_policy_json={"ground_truth_allowed": True, "validation_authority": {"building_validation": "primary"}},
)
snapshot = SourceSnapshot(
id=snapshot_id,
source_registry_id=source_id,
snapshot_key=f"detection-qa-{dataset.id}",
checksum_sha256=checksum,
ingest_status="ingested",
freshness_status="current",
)
dataset.source = "grb"
dataset.source_name = "grb"
dataset.dataset_role = "reference"
dataset.status = "ready"
dataset.checksum_sha256 = checksum
dataset.source_registry_id = source_id
dataset.source_snapshot_id = snapshot_id
dataset.data_contract_key = "geointel.vector.geojson"
dataset.data_contract_version = "1.0.0"
dataset.validation_status = "passed"
dataset.provenance_status = "complete"
dataset.lineage_status = "complete"
dataset.quarantine_status = "not_quarantined"
dataset.source_registry = source
dataset.source_snapshot = snapshot
return dataset
def test_detection_geojson_feature_collection_shape() -> None:
project_id = uuid4()
dataset_id = uuid4()
@@ -201,14 +242,14 @@ def test_detection_qa_persists_quality_check_and_metrics() -> None:
reference_dataset_id = uuid4()
analysis_run_id = uuid4()
detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1))
reference_dataset = Dataset(
reference_dataset = _authoritative_reference(Dataset(
id=reference_dataset_id,
project_id=project_id,
name="reference.geojson",
dataset_type="vector",
source="manual",
source="test",
dataset_role="reference",
)
))
reference_feature = VectorFeature(
id=uuid4(),
dataset_id=reference_dataset_id,
@@ -262,7 +303,7 @@ def test_detection_qa_rejects_non_overlapping_historical_reference_editions() ->
source_dataset.source_metadata = {"product_key": "2020", "supports_detection": False}
source_dataset.valid_from = datetime(2020, 1, 1, tzinfo=UTC)
source_dataset.valid_to = datetime(2020, 12, 31, 23, 59, 59, tzinfo=UTC)
reference_dataset = Dataset(
reference_dataset = _authoritative_reference(Dataset(
id=reference_dataset_id,
project_id=project_id,
name="current-grb.geojson",
@@ -272,7 +313,7 @@ def test_detection_qa_rejects_non_overlapping_historical_reference_editions() ->
dataset_role="reference",
valid_from=datetime(2026, 7, 1, tzinfo=UTC),
valid_to=datetime(2026, 7, 31, 23, 59, 59, tzinfo=UTC),
)
))
db = FakeSession(
objects={
(AnalysisRun, analysis_run_id): AnalysisRun(
@@ -305,14 +346,14 @@ def test_detection_qa_no_match_case_persists_zero_scores() -> None:
reference_dataset_id = uuid4()
analysis_run_id = uuid4()
detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1))
reference_dataset = Dataset(
reference_dataset = _authoritative_reference(Dataset(
id=reference_dataset_id,
project_id=project_id,
name="reference.geojson",
dataset_type="vector",
source="manual",
source="test",
dataset_role="reference",
)
))
reference_feature = VectorFeature(
id=uuid4(),
dataset_id=reference_dataset_id,
@@ -349,14 +390,14 @@ def test_configured_yolo_qa_requires_persisted_tile_manifest_provenance() -> Non
reference_dataset_id = uuid4()
analysis_run_id = uuid4()
detection = _detection(project_id, dataset_id, analysis_run_id)
reference_dataset = Dataset(
reference_dataset = _authoritative_reference(Dataset(
id=reference_dataset_id,
project_id=project_id,
name="reference.geojson",
dataset_type="vector",
source="manual",
source="test",
dataset_role="reference",
)
))
reference_feature = VectorFeature(
id=uuid4(),
dataset_id=reference_dataset_id,
@@ -421,14 +462,14 @@ def test_detection_qa_excludes_references_outside_persisted_tile_coverage(tmp_pa
analysis_run_id = uuid4()
manifest_path = _coverage_manifest(tmp_path, dataset_id, bounds=(0.0, 0.0, 1.0, 1.0))
detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0.1, 0.1, 0.9, 0.9))
reference_dataset = Dataset(
reference_dataset = _authoritative_reference(Dataset(
id=reference_dataset_id,
project_id=project_id,
name="reference.geojson",
dataset_type="vector",
source="manual",
source="test",
dataset_role="reference",
)
))
inside_reference = VectorFeature(
id=uuid4(),
dataset_id=reference_dataset_id,
@@ -489,14 +530,14 @@ def test_detection_qa_reports_box_to_footprint_diagnostic_without_changing_stric
l_shaped_footprint = Polygon(
[(0.0, 0.0), (2.0, 0.0), (2.0, 0.4), (0.4, 0.4), (0.4, 2.0), (0.0, 2.0), (0.0, 0.0)]
)
reference_dataset = Dataset(
reference_dataset = _authoritative_reference(Dataset(
id=reference_dataset_id,
project_id=project_id,
name="reference.geojson",
dataset_type="vector",
source="manual",
source="test",
dataset_role="reference",
)
))
reference_feature = VectorFeature(
id=uuid4(),
dataset_id=reference_dataset_id,
@@ -10,7 +10,18 @@ from shapely.geometry import MultiPolygon, box, mapping
from app.db.session import get_db
from app.main import app
from app.models import AnalysisRun, Dataset, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature
from app.models import (
AnalysisRun,
Dataset,
Job,
Metric,
Project,
QualityCheck,
Segmentation,
SourceRegistry,
SourceSnapshot,
VectorFeature,
)
from app.services.model_registry_service import ModelRegistryService
from app.services.segmentation_service import SegmentationService
@@ -75,7 +86,7 @@ def _project_and_dataset(dataset_type: str = "raster"):
project_id=project_id,
name="source.tif",
dataset_type=dataset_type,
source="user_upload",
source="test",
storage_path="storage/uploads/source.tif",
)
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
@@ -105,6 +116,47 @@ def _segmentation(project_id, dataset_id, analysis_run_id, class_name="vegetatio
)
def _authoritative_reference(dataset: Dataset) -> Dataset:
"""Model segmentation QA references as governed authoritative GRB evidence."""
source_id = uuid4()
snapshot_id = uuid4()
checksum = "a" * 64
source = SourceRegistry(
id=source_id,
source_key="grb",
display_name="GRB segmentation QA fixture",
classification="authoritative",
authority_name="Digitaal Vlaanderen",
authority_scope_json={"zone": "Flanders"},
usage_policy_json={"ground_truth_allowed": True, "validation_authority": {"building_validation": "primary"}},
)
snapshot = SourceSnapshot(
id=snapshot_id,
source_registry_id=source_id,
snapshot_key=f"segmentation-qa-{dataset.id}",
checksum_sha256=checksum,
ingest_status="ingested",
freshness_status="current",
)
dataset.source = "grb"
dataset.source_name = "grb"
dataset.dataset_role = "reference"
dataset.status = "ready"
dataset.checksum_sha256 = checksum
dataset.source_registry_id = source_id
dataset.source_snapshot_id = snapshot_id
dataset.data_contract_key = "geointel.vector.geojson"
dataset.data_contract_version = "1.0.0"
dataset.validation_status = "passed"
dataset.provenance_status = "complete"
dataset.lineage_status = "complete"
dataset.quarantine_status = "not_quarantined"
dataset.source_registry = source
dataset.source_snapshot = snapshot
return dataset
def test_model_registry_returns_segmentation_states() -> None:
models = {model.model_id: model for model in ModelRegistryService.list_model_capabilities(task_type="segmentation")}
@@ -265,14 +317,14 @@ def test_segmentation_qa_persists_quality_check_and_metrics() -> None:
reference_dataset_id = uuid4()
analysis_run_id = uuid4()
segmentation = _segmentation(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1))
reference_dataset = Dataset(
reference_dataset = _authoritative_reference(Dataset(
id=reference_dataset_id,
project_id=project_id,
name="reference.geojson",
dataset_type="vector",
source="manual",
source="test",
dataset_role="reference",
)
))
reference_feature = VectorFeature(
id=uuid4(),
dataset_id=reference_dataset_id,
@@ -282,6 +334,7 @@ def test_segmentation_qa_persists_quality_check_and_metrics() -> None:
db = FakeSession(
objects={
(AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="segmentation", status="success", parameters_json={}),
(Dataset, dataset_id): Dataset(id=dataset_id, project_id=project_id, name="fixture.tif", dataset_type="raster", source="test"),
(Dataset, reference_dataset_id): reference_dataset,
},
query_rows={Segmentation: [segmentation], VectorFeature: [reference_feature]},
@@ -318,14 +371,14 @@ def test_segmentation_qa_no_match_case_persists_zero_scores() -> None:
reference_dataset_id = uuid4()
analysis_run_id = uuid4()
segmentation = _segmentation(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1))
reference_dataset = Dataset(
reference_dataset = _authoritative_reference(Dataset(
id=reference_dataset_id,
project_id=project_id,
name="reference.geojson",
dataset_type="vector",
source="manual",
source="test",
dataset_role="reference",
)
))
reference_feature = VectorFeature(
id=uuid4(),
dataset_id=reference_dataset_id,
@@ -335,6 +388,7 @@ def test_segmentation_qa_no_match_case_persists_zero_scores() -> None:
db = FakeSession(
objects={
(AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="segmentation", status="success", parameters_json={}),
(Dataset, dataset_id): Dataset(id=dataset_id, project_id=project_id, name="fixture.tif", dataset_type="raster", source="test"),
(Dataset, reference_dataset_id): reference_dataset,
},
query_rows={Segmentation: [segmentation], VectorFeature: [reference_feature]},
@@ -0,0 +1,410 @@
from __future__ import annotations
import importlib.util
import hashlib
import json
from pathlib import Path
from types import SimpleNamespace
from uuid import uuid4
ROOT = Path(__file__).resolve().parents[2]
SCRIPT = ROOT / "scripts" / "training_dataset_eligibility.py"
SPEC = importlib.util.spec_from_file_location("training_dataset_eligibility", SCRIPT)
assert SPEC and SPEC.loader
MODULE = importlib.util.module_from_spec(SPEC)
SPEC.loader.exec_module(MODULE)
CHECKSUM = "a" * 64
UNSET = object()
def source_registry(
*,
classification: str = "authoritative",
training_allowed: bool = True,
ground_truth_allowed: bool = True,
allowed_tasks: list[str] | None = None,
building_validation_authority: str = "primary",
) -> SimpleNamespace:
return SimpleNamespace(
id="source-registry-1",
source_key="governed-source",
classification=classification,
freshness_status="current",
ingest_status="ingested",
usage_policy_json={
"training_allowed": training_allowed,
"ground_truth_allowed": ground_truth_allowed,
"allowed_tasks": allowed_tasks or ["building_validation", "building_labels"],
"validation_authority": {
"building_validation": building_validation_authority,
},
},
)
def source_snapshot(*, checksum: str = CHECKSUM) -> SimpleNamespace:
return SimpleNamespace(
id="source-snapshot-1",
snapshot_key="2026-08-01",
checksum_sha256=checksum,
freshness_status="current",
ingest_status="ingested",
)
def governed_dataset(
*,
role: str,
registry: SimpleNamespace | None | object = UNSET,
snapshot: SimpleNamespace | None | object = UNSET,
**overrides: object,
) -> SimpleNamespace:
dataset_type = "raster" if role == "raster" else "vector"
values: dict[str, object] = {
"id": f"dataset-{role}",
"dataset_type": dataset_type,
"dataset_role": "source" if role == "raster" else "reference",
"source": "governed_import",
"source_name": "governed-source",
"checksum_sha256": CHECKSUM,
"data_contract_key": f"{role}-contract",
"data_contract_version": "1.0.0",
"validation_status": "passed",
"provenance_status": "complete",
"lineage_status": "not_applicable",
"quarantine_status": "not_quarantined",
"status": "ready",
"metadata_json": {},
"provenance_metadata": {},
"source_registry": source_registry() if registry is UNSET else registry,
"source_snapshot": source_snapshot() if snapshot is UNSET else snapshot,
}
values.update(overrides)
return SimpleNamespace(**values)
def test_governed_authoritative_pair_is_eligible_for_operational_training() -> None:
raster = governed_dataset(
role="raster",
registry=source_registry(ground_truth_allowed=False),
)
reference = governed_dataset(role="reference")
decision = MODULE.training_pair_evidence(raster=raster, reference=reference)
assert decision["eligible"] is True
assert decision["raster"]["reasons"] == []
assert decision["reference"]["evidence"]["source_ground_truth_allowed"] is True
def test_operational_training_rejects_invalid_quarantined_incomplete_and_untrusted_inputs() -> None:
dataset = governed_dataset(
role="reference",
validation_status="failed",
provenance_status="incomplete",
lineage_status="incomplete",
quarantine_status="quarantined",
source_registry=source_registry(
classification="contextual",
training_allowed=False,
ground_truth_allowed=False,
),
)
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
assert decision.eligible is False
assert set(decision.reasons) >= {
"validation_failed",
"dataset_quarantined",
"provenance_not_complete",
"lineage_not_complete",
"source_not_allowed_for_training",
"reference_source_not_authoritative",
"reference_source_not_ground_truth_allowed",
}
def test_operational_training_rejects_a_due_source_snapshot() -> None:
snapshot = source_snapshot()
snapshot.freshness_status = "due"
dataset = governed_dataset(role="reference", snapshot=snapshot)
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
assert decision.eligible is False
assert "source_snapshot_freshness_not_approved" in decision.reasons
def test_osm_like_context_is_never_accepted_as_building_ground_truth() -> None:
dataset = governed_dataset(
role="reference",
source_name="osm",
source_registry=source_registry(
classification="contextual",
training_allowed=False,
ground_truth_allowed=False,
),
)
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
assert decision.eligible is False
assert "source_not_allowed_for_training" in decision.reasons
assert "reference_source_not_authoritative" in decision.reasons
def test_regional_building_sources_pending_primary_authority_cannot_enter_training_labels() -> None:
for source_key in ("spw_picc", "urbis"):
dataset = governed_dataset(
role="reference",
source_name=source_key,
source_registry=source_registry(
allowed_tasks=["building_validation", "building_labels"],
building_validation_authority="regional_primary_pending_contract",
),
)
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
assert decision.eligible is False
assert "reference_building_validation_not_primary" in decision.reasons
def test_authoritative_source_without_building_validation_task_cannot_be_used_as_a_label_reference() -> None:
dataset = governed_dataset(
role="reference",
source_registry=source_registry(
allowed_tasks=["elevation_validation"],
building_validation_authority="corroborative",
),
)
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
assert decision.eligible is False
assert set(decision.reasons) >= {
"reference_source_not_approved_for_building_validation",
"reference_building_validation_not_primary",
}
def test_dataset_and_snapshot_registry_bindings_cannot_be_forged() -> None:
snapshot = source_snapshot()
snapshot.source_registry_id = "different-registry"
dataset = governed_dataset(
role="reference",
registry=source_registry(),
snapshot=snapshot,
source_registry_id="different-registry",
source_snapshot_id="different-snapshot",
)
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
assert decision.eligible is False
assert set(decision.reasons) >= {
"dataset_source_registry_binding_mismatch",
"dataset_source_snapshot_binding_mismatch",
"source_snapshot_registry_mismatch",
}
def test_fixture_mode_only_relaxes_legacy_provenance_for_explicit_fixtures() -> None:
fixture = governed_dataset(
role="reference",
source="fixture",
source_name="fixture",
validation_status=None,
provenance_status="incomplete",
lineage_status="incomplete",
data_contract_key=None,
data_contract_version=None,
checksum_sha256=None,
source_registry=None,
source_snapshot=None,
metadata_json={"fixture": True},
)
unmarked = governed_dataset(
role="reference",
validation_status=None,
provenance_status="incomplete",
lineage_status="incomplete",
source_registry=None,
source_snapshot=None,
)
assert MODULE.evaluate_dataset_training_eligibility(
fixture,
role="reference",
fixture_mode=True,
).eligible is True
rejected = MODULE.evaluate_dataset_training_eligibility(
unmarked,
role="reference",
fixture_mode=True,
)
assert rejected.eligible is False
assert "fixture_mode_requires_explicit_fixture" in rejected.reasons
def test_fixture_mode_never_allows_failed_validation_or_quarantine() -> None:
fixture = governed_dataset(
role="raster",
source="fixture",
source_name="fixture",
validation_status="failed",
quarantine_status="quarantined",
source_registry=None,
source_snapshot=None,
)
decision = MODULE.evaluate_dataset_training_eligibility(fixture, role="raster", fixture_mode=True)
assert decision.eligible is False
assert set(decision.reasons) >= {"validation_failed", "dataset_quarantined"}
def test_manifest_gate_rejects_missing_or_tampered_pair_decisions() -> None:
raster = governed_dataset(
role="raster",
registry=source_registry(ground_truth_allowed=False),
)
reference = governed_dataset(role="reference")
pair = MODULE.training_pair_evidence(raster=raster, reference=reference)
manifest = {
"samples": [{"sample_slug": "governed", "training_eligibility": pair}],
"training_eligibility": {
"policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION,
"status": "eligible",
"fixture_mode": False,
},
}
assert MODULE.manifest_training_eligibility_failures(manifest) == []
tampered = {
**manifest,
"samples": [{"sample_slug": "governed", "training_eligibility": {**pair, "eligible": False}}],
}
failures = MODULE.manifest_training_eligibility_failures(tampered)
assert "governed:training_pair_not_eligible" in failures
assert MODULE.manifest_training_eligibility_failures({"samples": []}) == [
"manifest_training_eligibility_missing"
]
def test_frozen_manifest_gate_detects_checksum_tampering(tmp_path: Path) -> None:
raster = governed_dataset(
role="raster",
registry=source_registry(ground_truth_allowed=False),
)
reference = governed_dataset(role="reference")
pair = MODULE.training_pair_evidence(raster=raster, reference=reference)
manifest_path = tmp_path / "operator_samples_manifest.json"
manifest_path.write_text(
json.dumps(
{
"immutable": True,
"training_eligibility": {
"policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION,
"status": "eligible",
"fixture_mode": False,
},
"samples": [{"sample_slug": "governed", "training_eligibility": pair}],
}
),
encoding="utf-8",
)
(tmp_path / "corpus-freeze.json").write_text(
json.dumps(
{
"schema_version": 2,
"manifest_sha256": hashlib.sha256(manifest_path.read_bytes()).hexdigest(),
"immutable": True,
"training_eligibility_policy": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION,
"fixture_mode": False,
}
),
encoding="utf-8",
)
assert MODULE.frozen_manifest_training_eligibility_failures(manifest_path) == []
manifest_path.write_text(manifest_path.read_text(encoding="utf-8") + "\n", encoding="utf-8")
assert "corpus_manifest_checksum_mismatch" in MODULE.frozen_manifest_training_eligibility_failures(
manifest_path
)
def test_live_manifest_gate_revokes_a_frozen_pair_when_an_upstream_dataset_is_quarantined() -> None:
raster_id = uuid4()
reference_id = uuid4()
raster_registry = source_registry(ground_truth_allowed=False)
reference_registry = source_registry()
raster_snapshot = source_snapshot()
reference_snapshot = source_snapshot()
raster_snapshot.source_registry_id = raster_registry.id
reference_snapshot.source_registry_id = reference_registry.id
raster = governed_dataset(
role="raster",
id=raster_id,
source_registry=raster_registry,
source_snapshot=raster_snapshot,
source_registry_id=raster_registry.id,
source_snapshot_id=raster_snapshot.id,
)
reference = governed_dataset(
role="reference",
id=reference_id,
source_registry=reference_registry,
source_snapshot=reference_snapshot,
source_registry_id=reference_registry.id,
source_snapshot_id=reference_snapshot.id,
)
pair = MODULE.training_pair_evidence(raster=raster, reference=reference)
manifest = {
"training_eligibility": {
"policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION,
"status": "eligible",
"fixture_mode": False,
},
"samples": [
{
"sample_slug": "governed-aoi",
"raster_dataset_id": str(raster_id),
"reference_dataset_id": str(reference_id),
"training_eligibility": pair,
}
],
}
class DatasetModel:
pass
class Session:
def __init__(self) -> None:
self.closed = False
@staticmethod
def get(model, item_id):
assert model is DatasetModel
return {raster_id: raster, reference_id: reference}.get(item_id)
def close(self) -> None:
self.closed = True
assert MODULE.live_manifest_training_eligibility_failures(
manifest,
session_factory=Session,
dataset_model=DatasetModel,
) == []
raster.quarantine_status = "quarantined"
failures = MODULE.live_manifest_training_eligibility_failures(
manifest,
session_factory=Session,
dataset_model=DatasetModel,
)
assert "governed-aoi:raster_live_revoked:dataset_quarantined" in failures
@@ -0,0 +1,414 @@
from __future__ import annotations
import hashlib
import importlib.util
import json
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[2]
SCRIPTS = ROOT / "scripts"
if str(SCRIPTS) not in sys.path:
sys.path.insert(0, str(SCRIPTS))
SCRIPT = SCRIPTS / "training_release_manifest.py"
SPEC = importlib.util.spec_from_file_location("training_release_manifest", SCRIPT)
assert SPEC and SPEC.loader
MODULE = importlib.util.module_from_spec(SPEC)
sys.modules[SPEC.name] = MODULE
SPEC.loader.exec_module(MODULE)
def sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
@pytest.fixture(autouse=True)
def _synthetic_release_uses_a_static_live_registry_spy(monkeypatch: pytest.MonkeyPatch) -> None:
"""Filesystem unit fixtures cannot resolve a real database, but must ask for it."""
original_assert = MODULE.assert_frozen_manifest_training_eligible
original_failures = MODULE.frozen_manifest_training_eligibility_failures
def static_assert(*args, **kwargs):
assert kwargs.get("verify_live") is True
kwargs["verify_live"] = False
return original_assert(*args, **kwargs)
def static_failures(*args, **kwargs):
assert kwargs.get("verify_live") is True
kwargs["verify_live"] = False
return original_failures(*args, **kwargs)
monkeypatch.setattr(MODULE, "assert_frozen_manifest_training_eligible", static_assert)
monkeypatch.setattr(MODULE, "frozen_manifest_training_eligibility_failures", static_failures)
def write_corpus_manifest(tmp_path: Path, *, fixture_mode: bool = False) -> Path:
policy = "geointel-training-source-eligibility/v1"
def pair(sample_slug: str) -> dict:
raster_id = f"dataset:raster:{sample_slug}"
reference_id = f"dataset:reference:{sample_slug}"
return {
"policy_version": policy,
"eligible": True,
"fixture_mode": fixture_mode,
"raster": {
"eligible": True,
"reasons": [],
"evidence": {
"dataset_id": raster_id,
"checksum_sha256": "a" * 64,
"source_registry_id": "registry:orthophoto",
"source_snapshot_id": "snapshot:orthophoto",
},
},
"reference": {
"eligible": True,
"reasons": [],
"evidence": {
"dataset_id": reference_id,
"checksum_sha256": "b" * 64,
"source_registry_id": "registry:grb",
"source_snapshot_id": "snapshot:grb",
},
},
}
samples = []
for sample_slug, split in (("fixture-train", "train"), ("fixture-val", "val")):
samples.append(
{
"sample_slug": sample_slug,
"split": split,
"raster_dataset_id": f"dataset:raster:{sample_slug}",
"reference_dataset_id": f"dataset:reference:{sample_slug}",
"training_eligibility": pair(sample_slug),
}
)
manifest = tmp_path / "operator_samples_manifest.json"
manifest.write_text(
json.dumps(
{
"immutable": True,
"training_eligibility": {
"policy_version": policy,
"status": "eligible",
"fixture_mode": fixture_mode,
},
"samples": samples,
}
),
encoding="utf-8",
)
(tmp_path / "corpus-freeze.json").write_text(
json.dumps(
{
"schema_version": 2,
"immutable": True,
"fixture_mode": fixture_mode,
"training_eligibility_policy": policy,
"manifest_sha256": sha256(manifest),
}
),
encoding="utf-8",
)
return manifest
def write_yolo_dataset(tmp_path: Path, *, empty_train_label: bool = False) -> Path:
dataset_root = tmp_path / "dataset"
for split, sample_slug in (("train", "fixture-train"), ("val", "fixture-val")):
image = dataset_root / "images" / split / f"{sample_slug}.png"
label = dataset_root / "labels" / split / f"{sample_slug}.txt"
image.parent.mkdir(parents=True, exist_ok=True)
label.parent.mkdir(parents=True, exist_ok=True)
image.write_bytes(f"{split}-image".encode("utf-8"))
label.write_text("" if split == "train" and empty_train_label else "0 0.5 0.5 0.2 0.2\n", encoding="utf-8")
yaml_path = dataset_root / "dataset.yaml"
yaml_path.write_text(
f"path: {dataset_root}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
encoding="utf-8",
)
return yaml_path
def write_accepted_review_audit(tmp_path: Path, corpus_manifest: Path) -> Path:
artifacts = {}
for sample_slug in ("fixture-train", "fixture-val"):
artifact = tmp_path / f"{sample_slug}-contact-sheet.png"
artifact.write_bytes(f"reviewed {sample_slug}".encode("utf-8"))
artifacts[sample_slug] = artifact
decisions = tmp_path / "review-decisions.json"
decisions.write_text(
json.dumps(
{
"decisions": [
{
"sample_slug": sample_slug,
"decision": "accepted",
"reviewer": "reviewer@example.test",
"reviewed_at": "2026-08-01T12:00:00+00:00",
"reviewed_artifact_path": str(artifact.resolve()),
"reviewed_artifact_sha256": sha256(artifact),
}
for sample_slug, artifact in artifacts.items()
]
}
),
encoding="utf-8",
)
evidence = {
"review_decisions_path": str(decisions.resolve()),
"review_decisions_sha256": sha256(decisions),
"required_sample_count": 2,
"accepted_sample_count": 2,
"accepted_sample_slugs": ["fixture-train", "fixture-val"],
"reviewer_ids": ["reviewer@example.test"],
"reviewed_at_by_sample": {
"fixture-train": "2026-08-01T12:00:00+00:00",
"fixture-val": "2026-08-01T12:00:00+00:00",
},
"reviewed_artifact_path_by_sample": {
sample_slug: str(artifact.resolve()) for sample_slug, artifact in artifacts.items()
},
"reviewed_artifact_sha256_by_sample": {
sample_slug: sha256(artifact) for sample_slug, artifact in artifacts.items()
},
}
audit = tmp_path / "belgium-building-corpus-audit.json"
audit.write_text(
json.dumps(
{
"status": "ok",
"manifest_immutable": True,
"spatial_leakage_status": "ok",
"review_complete": True,
"corpus_manifest_path": str(corpus_manifest.resolve()),
"corpus_manifest_sha256": sha256(corpus_manifest),
"human_review_evidence": evidence,
}
),
encoding="utf-8",
)
return audit
def test_valid_operational_release_binds_yaml_assets_corpus_and_human_review(tmp_path: Path) -> None:
corpus_manifest = write_corpus_manifest(tmp_path)
yaml_path = write_yolo_dataset(tmp_path)
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
paths = MODULE.create_training_release_manifest(
train_yaml=yaml_path,
corpus_manifest=corpus_manifest,
review_audit_path=review_audit,
)
assert all(path.is_file() for path in paths.values())
assert MODULE.training_release_failures(
train_yaml=yaml_path,
corpus_manifest=corpus_manifest,
) == []
def test_tile_summary_must_be_an_exact_view_of_the_live_verified_release(tmp_path: Path) -> None:
corpus_manifest = write_corpus_manifest(tmp_path)
yaml_path = write_yolo_dataset(tmp_path)
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
paths = MODULE.create_training_release_manifest(
train_yaml=yaml_path,
corpus_manifest=corpus_manifest,
review_audit_path=review_audit,
)
assets = json.loads(paths["asset_manifest"].read_text(encoding="utf-8"))
summary = yaml_path.parent / "yolo_tile_dataset_summary.json"
summary.write_text(
json.dumps(
{
"dataset_yaml": str(yaml_path.resolve()),
"training_release_manifest": str(paths["release_manifest"].resolve()),
"training_release_manifest_sha256": sha256(paths["release_manifest"]),
"training_asset_manifest": str(paths["asset_manifest"].resolve()),
"source_manifest_sha256": sha256(corpus_manifest),
"tiles": [
{
"split": entry["split"],
"image_path": entry["image_path"],
"label_path": entry["label_path"],
}
for entry in assets["entries"]
],
}
),
encoding="utf-8",
)
MODULE.assert_yolo_summary_bound_to_training_release(
summary_path=summary,
train_yaml=yaml_path,
corpus_manifest=corpus_manifest,
)
payload = json.loads(summary.read_text(encoding="utf-8"))
payload["tiles"] = payload["tiles"][:1]
summary.write_text(json.dumps(payload), encoding="utf-8")
with pytest.raises(MODULE.TrainingReleaseError, match="complete immutable view"):
MODULE.assert_yolo_summary_bound_to_training_release(
summary_path=summary,
train_yaml=yaml_path,
corpus_manifest=corpus_manifest,
)
def test_unbound_or_changed_yaml_is_rejected_before_training(tmp_path: Path) -> None:
corpus_manifest = write_corpus_manifest(tmp_path)
yaml_path = write_yolo_dataset(tmp_path)
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
assert MODULE.training_release_failures(train_yaml=yaml_path) == ["training_release_manifest_missing"]
MODULE.create_training_release_manifest(
train_yaml=yaml_path,
corpus_manifest=corpus_manifest,
review_audit_path=review_audit,
)
yaml_path.write_text(yaml_path.read_text(encoding="utf-8") + "# tampered\n", encoding="utf-8")
assert "training_release_yaml_checksum_mismatch" in MODULE.training_release_failures(
train_yaml=yaml_path,
corpus_manifest=corpus_manifest,
)
def test_changed_label_asset_is_rejected_even_when_yaml_bytes_are_unchanged(tmp_path: Path) -> None:
corpus_manifest = write_corpus_manifest(tmp_path)
yaml_path = write_yolo_dataset(tmp_path)
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
MODULE.create_training_release_manifest(
train_yaml=yaml_path,
corpus_manifest=corpus_manifest,
review_audit_path=review_audit,
)
label = yaml_path.parent / "labels" / "train" / "fixture-train.txt"
label.write_text("0 0.4 0.4 0.2 0.2\n", encoding="utf-8")
failures = MODULE.training_release_failures(
train_yaml=yaml_path,
corpus_manifest=corpus_manifest,
)
assert "training_release_asset_manifest_content_mismatch" in failures
def test_operational_release_requires_complete_accepted_human_review(tmp_path: Path) -> None:
corpus_manifest = write_corpus_manifest(tmp_path)
yaml_path = write_yolo_dataset(tmp_path)
incomplete_audit = tmp_path / "audit.json"
incomplete_audit.write_text(
json.dumps(
{
"status": "needs_human_review",
"manifest_immutable": True,
"spatial_leakage_status": "ok",
"review_complete": False,
}
),
encoding="utf-8",
)
try:
MODULE.create_training_release_manifest(
train_yaml=yaml_path,
corpus_manifest=corpus_manifest,
review_audit_path=incomplete_audit,
)
except MODULE.TrainingReleaseError as exc:
assert "review_complete_not_true" in str(exc)
assert "accepted_human_review_evidence_missing" in str(exc)
else:
raise AssertionError("operational release accepted an incomplete human review")
def test_operational_release_rejects_tampered_accepted_review_artifact(tmp_path: Path) -> None:
corpus_manifest = write_corpus_manifest(tmp_path)
yaml_path = write_yolo_dataset(tmp_path)
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
artifact = tmp_path / "fixture-train-contact-sheet.png"
artifact.write_bytes(b"changed after review")
try:
MODULE.create_training_release_manifest(
train_yaml=yaml_path,
corpus_manifest=corpus_manifest,
review_audit_path=review_audit,
)
except MODULE.TrainingReleaseError as exc:
assert "reviewed_artifact_checksum_mismatch" in str(exc)
else:
raise AssertionError("tampered human-review artifact was accepted")
def test_fixture_relaxation_requires_explicit_fixture_corpus_and_is_not_operational(tmp_path: Path) -> None:
fixture_manifest = write_corpus_manifest(tmp_path, fixture_mode=True)
yaml_path = write_yolo_dataset(tmp_path)
MODULE.create_training_release_manifest(
train_yaml=yaml_path,
corpus_manifest=fixture_manifest,
fixture_mode=True,
)
assert MODULE.training_release_failures(
train_yaml=yaml_path,
corpus_manifest=fixture_manifest,
fixture_mode=True,
) == []
assert "training_release_fixture_mode_mismatch" in MODULE.training_release_failures(
train_yaml=yaml_path,
corpus_manifest=fixture_manifest,
fixture_mode=False,
)
def test_release_contracts_a_reviewed_empty_label_as_explicit_pure_background(tmp_path: Path) -> None:
corpus_manifest = write_corpus_manifest(tmp_path)
yaml_path = write_yolo_dataset(tmp_path, empty_train_label=True)
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
paths = MODULE.create_training_release_manifest(
train_yaml=yaml_path,
corpus_manifest=corpus_manifest,
review_audit_path=review_audit,
)
labels = json.loads(paths["label_contract_manifest"].read_text(encoding="utf-8"))
assert labels["counts"]["pure_background"] == 1
assert MODULE.training_release_failures(train_yaml=yaml_path, corpus_manifest=corpus_manifest) == []
def test_empty_label_without_accepted_sample_review_cannot_become_a_background_negative(tmp_path: Path) -> None:
corpus_manifest = write_corpus_manifest(tmp_path)
yaml_path = write_yolo_dataset(tmp_path, empty_train_label=True)
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
audit_payload = json.loads(review_audit.read_text(encoding="utf-8"))
evidence = audit_payload["human_review_evidence"]
evidence["accepted_sample_slugs"] = ["fixture-val"]
review_audit.write_text(json.dumps(audit_payload), encoding="utf-8")
review = {
"status": "accepted",
"fixture_only": False,
"review_complete": True,
"evidence": evidence,
}
try:
MODULE.build_training_label_contract_manifest(
corpus_manifest_path=corpus_manifest,
asset_manifest=MODULE.build_yolo_asset_manifest(yaml_path),
review=review,
fixture_mode=False,
)
except MODULE.TrainingReleaseError as exc:
assert "Pure-background label sample was not accepted by review" in str(exc)
else:
raise AssertionError("unreviewed empty label was accepted as a background negative")
@@ -1,7 +1,6 @@
from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
from uuid import UUID, uuid4
from app.core.errors import AppError
+2
View File
@@ -128,6 +128,8 @@ COPY scripts/evaluate_belgium_building_candidate.py /app/scripts/evaluate_belgiu
COPY scripts/assess_belgium_building_training_iteration.py /app/scripts/assess_belgium_building_training_iteration.py
COPY scripts/run_belgium_building_training_loop.py /app/scripts/run_belgium_building_training_loop.py
COPY scripts/build_failure_driven_yolo_sampling.py /app/scripts/build_failure_driven_yolo_sampling.py
COPY scripts/training_dataset_eligibility.py /app/scripts/training_dataset_eligibility.py
COPY scripts/training_release_manifest.py /app/scripts/training_release_manifest.py
COPY scripts/build_grayscale_yolo_dataset.py /app/scripts/build_grayscale_yolo_dataset.py
COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py
COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py
+37 -4
View File
@@ -393,15 +393,48 @@ Fields:
- `file`: dataset file.
- `dataset_type`: `vector`, `geojson` (legacy), `raster`.
- `source`: free text, e.g. `user_upload`, `grb`, `osm`.
- `source`: descriptive caller text, e.g. `user_upload`. It is retained as a
claim only and never establishes authority.
- `dataset_role`: `source`, `derived`, or `reference` (default `source`).
- `source_name`: optional source identity, e.g. `manual`, `grb`, `osm`; reference uploads default to `manual` when omitted.
- `source_name`: optional descriptive claim. Public uploads are always bound
to the server-owned `manual` registry entry, including when this field says
`grb`, `osm` or another official name. Reference uploads also default to
`manual`.
- `reference_layer_name`: optional reference layer label, e.g. `buildings`; only retained for reference datasets.
- `area_id`: optional.
Response: `DatasetRead` with extracted metadata if supported.
Response: `DatasetRead` with extracted metadata if supported, plus
`source_registry_id`, `source_snapshot_id`, exact data-contract key/version,
validation report/status, provenance/lineage status, quarantine status and an
idempotent ingest key. A malformed or doubtful artifact is retained as
`status=quarantined`; it is not silently discarded or made ready.
Vector uploads remain stored as original files and are also persisted into `vector_features` as queryable PostGIS state.
Vector uploads remain stored as original files and are also persisted into
`vector_features` as queryable PostGIS state only after their contract passes.
Non-EPSG:4326 vector coordinates are explicitly transformed before canonical
feature persistence; relabelling Lambert coordinates as EPSG:4326 is rejected.
### GET `/api/v1/source-registry`
Lists server-owned source definitions. Optional query parameter
`classification` is one of `authoritative`, `corroborative`, `contextual`,
`derived`, or `experimental`. This endpoint reports policy and snapshot counts;
it does not assert that historical datasets carrying a matching text field are
trusted. It is operator-only because source snapshots can contain
operator-acquisition provenance. A demo session uses its project-scoped
dataset provenance endpoint instead.
### GET `/api/v1/source-registry/{source_key}`
Returns one source definition and its immutable snapshots, including authority
scope, licence/restrictions, expected geometry/attributes, freshness and known
limitations. Unknown source keys return `SOURCE_REGISTRY_ENTRY_NOT_FOUND`.
### GET `/api/v1/projects/{project_id}/datasets/{dataset_id}/provenance`
Returns the dataset's bound source/snapshot, contract report, lineage edges and
quarantine records. A missing binding is visible as incomplete provenance; it
is never backfilled from a display name or caller metadata.
### GET `/api/v1/projects/{project_id}/datasets/orthophoto/products`
+35
View File
@@ -69,6 +69,41 @@ Spatial index required on `geometry`.
- `status text default 'created'`
- `created_at timestamptz`
### Source registry, snapshots and quarantine (Accuracy Phase 2)
Migration `202608010001_source_registry_provenance` adds a server-owned
provenance boundary without deleting or inventing facts for existing rows.
`source_registry` stores one governed source definition per `source_key`:
- classification (`authoritative`, `corroborative`, `contextual`, `derived`,
`experimental`), authority/scope and provider adapter key;
- licence, usage restrictions, expected CRS/units/resolution, temporal and
geographic coverage, expected geometry/attributes and known limitations;
- usage policy, freshness and ingest status.
`source_snapshots` stores immutable acquired editions keyed by
`(source_registry_id, snapshot_key)`, including source version/snapshot/fetch
time, lowercase SHA-256 checksum, CRS/units/resolution, coverage, observed
schema, freshness, ingest status and raw snapshot metadata.
`datasets` and `dataset_versions` receive `source_registry_id`,
`source_snapshot_id`, data-contract key/version, validation report/status,
provenance/lineage status and idempotent `ingest_key`. New governed records
must have a passing report before they become `ready`. Legacy rows are retained
with explicit incomplete/not-validated state; a migration never promotes them
based on a historical `source_name` string.
`dataset_lineage_edges` records parent/child Dataset(+Version), transform name
and version, parameters and input/output hashes. `dataset_quarantines` retains
the rejected dataset/version/snapshot, stage, reason, validation evidence and
artifact location. Quarantine is a stateful preservation record, not a delete.
The database constraints enforce accepted status vocabularies, nonblank ingest
keys, immutable snapshot identity/checksum format, unique ingest idempotency
keys and non-self lineage edges. Service-level contract validation remains
responsible for CRS, topology, bbox, units and source-task semantics.
### vector_features
Used for imported vector datasets and derived vector outputs when feature-level storage is needed. Original files remain source artifacts; this table is the queryable PostGIS state for vector features.
+52 -4
View File
@@ -26,6 +26,10 @@ from app.db.session import SessionLocal # noqa: E402
from app.models import Dataset # noqa: E402
from normalize_belgium_building_labels import normalize # noqa: E402
from training_dataset_eligibility import ( # noqa: E402
TRAINING_ELIGIBILITY_POLICY_VERSION,
training_pair_evidence,
)
REGION_SOURCES = {
"flanders": ({"digitaal_vlaanderen_orthophoto"}, "grb"),
@@ -52,7 +56,13 @@ def _dataset_path(dataset: Dataset) -> Path:
return path
def _validate_pair(sample: dict[str, Any], raster: Dataset, reference: Dataset) -> tuple[str, str]:
def _validate_pair(
sample: dict[str, Any],
raster: Dataset,
reference: Dataset,
*,
fixture_mode: bool,
) -> tuple[str, str, dict[str, Any]]:
region = str(sample.get("region") or "").lower()
if region not in REGION_SOURCES:
raise SystemExit(f"Unsupported region for {sample.get('sample_slug')}: {region}")
@@ -66,7 +76,23 @@ def _validate_pair(sample: dict[str, Any], raster: Dataset, reference: Dataset)
raise SystemExit(f"Unsupported split for {sample['sample_slug']}: {split}")
if raster.status != "ready" or reference.status != "ready":
raise SystemExit(f"Dataset pair is not ready for {sample['sample_slug']}")
return region, expected_reference
eligibility = training_pair_evidence(
raster=raster,
reference=reference,
fixture_mode=fixture_mode,
)
if not eligibility["eligible"]:
reasons = sorted(
{
reason
for role in ("raster", "reference")
for reason in eligibility[role]["reasons"]
}
)
raise SystemExit(
f"Dataset pair is not eligible for training for {sample['sample_slug']}: {', '.join(reasons)}"
)
return region, expected_reference, eligibility
def audit_spatial_leakage(samples: list[dict[str, Any]], buffer_m: float = 64.0) -> dict[str, Any]:
@@ -104,6 +130,14 @@ def main() -> int:
parser.add_argument("--min-label-px", type=float, default=3.0)
parser.add_argument("--merge-touching-roofs", action="store_true")
parser.add_argument("--freeze", action="store_true")
parser.add_argument(
"--fixture-mode",
action="store_true",
help=(
"Allow only explicitly marked fixture datasets with legacy provenance. "
"Never use this mode for an operational corpus."
),
)
args = parser.parse_args()
spec = json.loads(args.spec.read_text(encoding="utf-8-sig"))
@@ -129,7 +163,12 @@ def main() -> int:
reference = db.get(Dataset, UUID(str(sample["reference_dataset_id"])))
if raster is None or reference is None:
raise SystemExit(f"Persisted Dataset pair not found for {slug}")
region, reference_source = _validate_pair(sample, raster, reference)
region, reference_source, eligibility = _validate_pair(
sample,
raster,
reference,
fixture_mode=args.fixture_mode,
)
raster_source = _dataset_path(raster)
reference_source_path = _dataset_path(reference)
sample_dir = pairs_dir / slug
@@ -173,14 +212,20 @@ def main() -> int:
"raster_sha256": sha256(raster_target),
"reference_sha256": sha256(normalized_target),
"label_audit_sha256": sha256(audit_target),
"training_eligibility": eligibility,
"bbox_epsg4326": (raster.source_metadata or {}).get("bbox_epsg4326")
or sample.get("bbox_epsg4326"),
}
)
manifest = {
"schema_version": 1,
"schema_version": 2,
"dataset_version": args.version,
"immutable": bool(args.freeze),
"training_eligibility": {
"policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION,
"status": "eligible",
"fixture_mode": bool(args.fixture_mode),
},
"samples": manifest_samples,
}
manifest_path = output_dir / "operator_samples_manifest.json"
@@ -192,10 +237,13 @@ def main() -> int:
if leakage_audit["status"] != "ok":
raise SystemExit("Spatial split leakage audit failed")
freeze = {
"schema_version": 2,
"dataset_version": args.version,
"manifest_sha256": sha256(manifest_path),
"sample_count": len(manifest_samples),
"immutable": bool(args.freeze),
"training_eligibility_policy": TRAINING_ELIGIBILITY_POLICY_VERSION,
"fixture_mode": bool(args.fixture_mode),
}
(output_dir / "corpus-freeze.json").write_text(json.dumps(freeze, indent=2), encoding="utf-8")
print(json.dumps(freeze))
+113 -4
View File
@@ -4,13 +4,49 @@
from __future__ import annotations
import argparse
import hashlib
import json
import sys
from collections import Counter
from datetime import datetime
from pathlib import Path
from typing import Any
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import frozen_manifest_training_eligibility_failures # noqa: E402
REQUIRED_SPLITS = ("train", "val", "calibration", "test", "background-test")
REGIONS = ("flanders", "wallonia", "brussels")
SHA256_LENGTH = 64
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def valid_review_timestamp(value: Any) -> bool:
if not isinstance(value, str) or not value.strip():
return False
try:
parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
except ValueError:
return False
return parsed.tzinfo is not None
def valid_sha256(value: Any) -> bool:
return (
isinstance(value, str)
and len(value) == SHA256_LENGTH
and all(character in "0123456789abcdefABCDEF" for character in value)
)
def main() -> int:
@@ -19,8 +55,15 @@ def main() -> int:
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--review-decisions", type=Path)
args = parser.parse_args()
manifest = json.loads((args.corpus_dir / "operator_samples_manifest.json").read_text(encoding="utf-8"))
manifest_path = args.corpus_dir / "operator_samples_manifest.json"
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
leakage = json.loads((args.corpus_dir / "spatial-leakage-audit.json").read_text(encoding="utf-8"))
manifest_policy = manifest.get("training_eligibility")
fixture_mode = bool(manifest_policy.get("fixture_mode")) if isinstance(manifest_policy, dict) else False
eligibility_failures = frozen_manifest_training_eligibility_failures(
manifest_path,
fixture_mode=fixture_mode,
)
samples = manifest["samples"]
split_counts = Counter((sample["region"], sample["split"]) for sample in samples)
decision_counts: Counter[str] = Counter()
@@ -28,7 +71,7 @@ def main() -> int:
total_input = 0
total_accepted = 0
temporal_unknown = 0
failures: list[str] = []
failures: list[str] = list(eligibility_failures)
for region in REGIONS:
for split in REQUIRED_SPLITS:
minimum = 4 if split == "train" else 2
@@ -62,8 +105,12 @@ def main() -> int:
failures.append("spatial leakage audit failed")
reviewed = 0
review_complete = False
review_evidence_failures: list[str] = []
accepted_review_evidence: dict[str, dict[str, Any]] = {}
review_decisions_path: Path | None = None
if args.review_decisions and args.review_decisions.is_file():
decisions = json.loads(args.review_decisions.read_text(encoding="utf-8"))
review_decisions_path = args.review_decisions.resolve(strict=False)
decisions = json.loads(review_decisions_path.read_text(encoding="utf-8"))
by_slug = {item["sample_slug"]: item for item in decisions.get("decisions", [])}
for item in review_queue:
decision = by_slug.get(item["sample_slug"])
@@ -71,13 +118,70 @@ def main() -> int:
item["decision"] = decision.get("decision")
item["reviewer"] = decision.get("reviewer")
item["notes"] = decision.get("notes")
if item["decision"] in {"accepted", "rejected"} and item.get("reviewer"):
item["reviewed_at"] = decision.get("reviewed_at")
item["reviewed_artifact_path"] = decision.get("reviewed_artifact_path")
item["reviewed_artifact_sha256"] = decision.get("reviewed_artifact_sha256")
reviewed_artifact = (
Path(item["reviewed_artifact_path"]).expanduser().resolve(strict=False)
if isinstance(item.get("reviewed_artifact_path"), str)
and item["reviewed_artifact_path"].strip()
else None
)
has_evidence = (
item["decision"] == "accepted"
and isinstance(item.get("reviewer"), str)
and bool(item["reviewer"].strip())
and valid_review_timestamp(item.get("reviewed_at"))
and isinstance(item.get("reviewed_artifact_path"), str)
and bool(item["reviewed_artifact_path"].strip())
and valid_sha256(item.get("reviewed_artifact_sha256"))
and reviewed_artifact is not None
and reviewed_artifact.is_file()
and item["reviewed_artifact_sha256"] == sha256(reviewed_artifact)
)
if has_evidence:
reviewed += 1
accepted_review_evidence[str(item["sample_slug"])] = {
"reviewer": item["reviewer"].strip(),
"reviewed_at": item["reviewed_at"],
"reviewed_artifact_path": item["reviewed_artifact_path"],
"reviewed_artifact_sha256": item["reviewed_artifact_sha256"],
}
elif item["decision"] in {"accepted", "rejected"}:
review_evidence_failures.append(
f"{item['sample_slug']}:accepted review lacks reviewer/timestamp/artifact evidence"
)
review_complete = reviewed == len(review_queue) and all(item["decision"] == "accepted" for item in review_queue)
human_review_evidence: dict[str, Any] | None = None
if review_decisions_path is not None:
human_review_evidence = {
"review_decisions_path": str(review_decisions_path),
"review_decisions_sha256": sha256(review_decisions_path),
"required_sample_count": len(review_queue),
"accepted_sample_count": reviewed,
"accepted_sample_slugs": sorted(accepted_review_evidence),
"reviewer_ids": sorted(
{item["reviewer"] for item in accepted_review_evidence.values()}
),
"reviewed_at_by_sample": {
slug: accepted_review_evidence[slug]["reviewed_at"]
for slug in sorted(accepted_review_evidence)
},
"reviewed_artifact_path_by_sample": {
slug: accepted_review_evidence[slug]["reviewed_artifact_path"]
for slug in sorted(accepted_review_evidence)
},
"reviewed_artifact_sha256_by_sample": {
slug: accepted_review_evidence[slug]["reviewed_artifact_sha256"]
for slug in sorted(accepted_review_evidence)
},
}
status = "failed" if failures else ("ok" if review_complete else "needs_human_review")
report = {
"status": status,
"dataset_version": manifest["dataset_version"],
"corpus_manifest_path": str(manifest_path.resolve(strict=False)),
"corpus_manifest_sha256": sha256(manifest_path),
"manifest_immutable": manifest["immutable"],
"sample_count": len(samples),
"split_counts": {f"{region}/{split}": split_counts[(region, split)] for region in REGIONS for split in REQUIRED_SPLITS},
@@ -86,8 +190,13 @@ def main() -> int:
"decision_counts": dict(sorted(decision_counts.items())),
"temporal_unknown_sample_count": temporal_unknown,
"spatial_leakage_status": leakage.get("status"),
"training_eligibility_status": manifest_policy.get("status") if isinstance(manifest_policy, dict) else None,
"training_eligibility_fixture_mode": fixture_mode,
"training_eligibility_failures": eligibility_failures,
"reviewed_sample_count": reviewed,
"review_complete": review_complete,
"human_review_evidence": human_review_evidence,
"review_evidence_failures": sorted(set(review_evidence_failures)),
"failures": failures,
"review_queue": review_queue,
}
@@ -9,7 +9,7 @@ import json
import sys
from collections import Counter
from pathlib import Path
from typing import Any
from typing import Any, Mapping
from PIL import Image
@@ -17,10 +17,21 @@ SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from evaluate_belgium_building_candidate import iou, read_references
from evaluate_belgium_building_candidate import iou, read_references # noqa: E402
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_embedded_training_release,
)
PROTECTED_SPLITS = {"calibration", "test", "background-test"}
PROPOSAL_DATASET_PROVENANCE_NAME = "proposal-dataset-provenance.json"
PROPOSAL_DATASET_EVIDENCE_NAME = "proposal-dataset-evidence.json"
PROPOSAL_DATASET_SCHEMA_VERSION = 1
def sha256(path: Path) -> str:
@@ -31,6 +42,125 @@ def sha256(path: Path) -> str:
return digest.hexdigest()
def _canonical_json_bytes(payload: Mapping[str, Any]) -> bytes:
return json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
def _immutable_payload_sha256(payload: Mapping[str, Any], *, field: str = "manifest_sha256") -> str:
normalized = dict(payload)
normalized.pop(field, None)
return hashlib.sha256(_canonical_json_bytes(normalized)).hexdigest()
def _write_immutable_json(path: Path, payload: Mapping[str, Any]) -> None:
"""Persist one immutable sidecar without silently replacing prior evidence."""
encoded = json.dumps(dict(payload), ensure_ascii=False, indent=2, sort_keys=True) + "\n"
if path.exists():
if path.read_text(encoding="utf-8") != encoded:
raise RuntimeError(f"immutable provenance artifact already exists with different content: {path}")
return
path.write_text(encoded, encoding="utf-8")
def assert_summary_source_manifest_binding(summary: Mapping[str, Any], corpus_manifest_path: Path) -> str:
"""Require the proposal summary to name the exact frozen corpus bytes."""
expected = sha256(corpus_manifest_path)
observed = summary.get("source_manifest_sha256")
if observed != expected:
raise ValueError(
"proposal source summary is not bound to the supplied governed corpus manifest "
f"(expected {expected}, observed {observed!r})"
)
return expected
def load_governed_corpus_manifest(corpus_manifest_path: Path, *, fixture_mode: bool) -> dict[str, Any]:
"""Validate frozen corpus evidence and re-check the live Dataset state.
This must run before importing Ultralytics/PyTorch so a revocation cannot
consume GPU work or create proposal crops.
"""
try:
manifest = assert_frozen_manifest_training_eligible(
corpus_manifest_path,
fixture_mode=fixture_mode,
verify_live=True,
)
except TrainingEligibilityError as exc:
raise ValueError(str(exc)) from exc
if not isinstance(manifest, dict): # pragma: no cover - defensive contract boundary
raise ValueError("governed corpus manifest must be a JSON object")
return manifest
def _source_file_evidence(tile: Mapping[str, Any], *, field: str) -> tuple[Path, str]:
raw_path = tile.get(field)
if not isinstance(raw_path, str) or not raw_path.strip():
raise ValueError(f"proposal source tile has no {field}: {tile.get('sample_slug')!r}")
path = Path(raw_path).expanduser().resolve(strict=False)
if not path.is_file():
raise ValueError(f"proposal source tile {field} is unavailable: {path}")
return path, sha256(path)
def _source_tile_key(tile: Mapping[str, Any]) -> tuple[str, str, str]:
"""Return the canonical source identity used for the pre-inference cache."""
return (
str(tile.get("sample_slug") or ""),
str(Path(str(tile.get("image_path") or "")).expanduser().resolve(strict=False)),
str(Path(str(tile.get("label_path") or "")).expanduser().resolve(strict=False)),
)
def _assert_file_unchanged(path: Path, expected_sha256: str, *, role: str) -> None:
"""Reject a mutable source changing between evidence capture and use."""
if sha256(path) != expected_sha256:
raise RuntimeError(f"proposal {role} changed after evidence capture: {path}")
def _crop_entry(
*,
output_dir: Path,
crop_path: Path,
tile: Mapping[str, Any],
label: str,
proposal_index: int,
proposal_score: float,
source_box_xyxy: tuple[float, float, float, float],
source_image_path: Path,
source_image_sha256: str,
source_label_path: Path,
source_label_sha256: str,
) -> dict[str, Any]:
relative_path = crop_path.resolve(strict=False).relative_to(output_dir.resolve(strict=False)).as_posix()
sample_slug = tile.get("sample_slug")
split = tile.get("split")
if not isinstance(sample_slug, str) or not sample_slug.strip():
raise ValueError("proposal source tile has no sample_slug")
if split not in {"train", "val"}:
raise ValueError(f"proposal crop has unsupported split: {split!r}")
return {
"relative_path": relative_path,
"sha256": sha256(crop_path),
"size_bytes": crop_path.stat().st_size,
"split": split,
"label": label,
"sample_slug": sample_slug,
"proposal_index": proposal_index,
"proposal_score": proposal_score,
"source_box_xyxy": [float(value) for value in source_box_xyxy],
"source_image_path": str(source_image_path),
"source_image_sha256": source_image_sha256,
"source_label_path": str(source_label_path),
"source_label_sha256": source_label_sha256,
}
def classify_proposals(
predictions: list[tuple[tuple[float, float, float, float], float]],
references: list[tuple[float, float, float, float]],
@@ -99,29 +229,74 @@ def main() -> int:
parser.add_argument("--max-negative-per-tile", type=int, default=24)
parser.add_argument("--device", default="cuda:0")
parser.add_argument("--imgsz", type=int, default=640)
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Accept only an explicitly fixture-only corpus manifest; never use for operational training data.",
)
args = parser.parse_args()
if args.output_dir.exists():
parser.error(f"output already exists: {args.output_dir}")
from ultralytics import YOLO
summary = json.loads(args.summary.read_text(encoding="utf-8"))
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
try:
summary = json.loads(args.summary.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise SystemExit(f"proposal source summary is unreadable: {args.summary}") from exc
if not isinstance(summary, dict):
raise SystemExit("proposal source summary must be a JSON object")
try:
source_release = assert_yolo_summary_bound_to_embedded_training_release(
summary_path=args.summary,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
manifest = load_governed_corpus_manifest(args.corpus_manifest, fixture_mode=args.fixture_mode)
corpus_manifest_sha256 = assert_summary_source_manifest_binding(summary, args.corpus_manifest)
except (ValueError, TrainingReleaseError) as exc:
raise SystemExit(str(exc)) from exc
tiles = eligible_tiles(summary, manifest, args.region)
model_path = args.model.expanduser().resolve(strict=False)
if not model_path.is_file():
raise SystemExit(f"proposal model is unavailable: {model_path}")
model_sha256 = sha256(model_path)
# Hash every input image/label before GPU inference. A crop sidecar then
# binds each generated JPEG to the exact source tile rather than its
# filename or an unverified aggregate summary.
source_evidence: dict[tuple[str, str, str], tuple[Path, str, Path, str]] = {}
for tile in tiles:
image_path, image_sha256 = _source_file_evidence(tile, field="image_path")
label_path, label_sha256 = _source_file_evidence(tile, field="label_path")
key = _source_tile_key(tile)
source_evidence[key] = (image_path, image_sha256, label_path, label_sha256)
args.output_dir.mkdir(parents=True)
counts: Counter[str] = Counter()
sample_counts: Counter[str] = Counter()
model = YOLO(str(args.model))
crop_entries: list[dict[str, Any]] = []
# Importing the model is deliberately after the source-manifest and live
# Dataset checks above. A revoked corpus must not start GPU work.
from ultralytics import YOLO
model = YOLO(str(model_path))
for start in range(0, len(tiles), 16):
batch_tiles = tiles[start : start + 16]
batch_sources = [source_evidence[_source_tile_key(tile)] for tile in batch_tiles]
for image_path, image_sha256, label_path, label_sha256 in batch_sources:
_assert_file_unchanged(image_path, image_sha256, role="source image")
_assert_file_unchanged(label_path, label_sha256, role="source label")
results = model.predict(
[tile["image_path"] for tile in batch_tiles], conf=args.confidence,
[str(source[0]) for source in batch_sources], conf=args.confidence,
device=args.device, imgsz=args.imgsz, max_det=1000, iou=0.7, verbose=False,
)
for tile, result in zip(batch_tiles, results, strict=True):
with Image.open(tile["image_path"]) as opened:
image_path, image_sha256, label_path, label_sha256 = source_evidence[_source_tile_key(tile)]
_assert_file_unchanged(image_path, image_sha256, role="source image")
_assert_file_unchanged(label_path, label_sha256, role="source label")
with Image.open(image_path) as opened:
source = opened.convert("RGB")
references = read_references(Path(tile["label_path"]), source.width, source.height)
references = read_references(label_path, source.width, source.height)
proposals = [
(tuple(map(float, box)), float(score))
for box, score in zip(
@@ -139,25 +314,113 @@ def main() -> int:
target_dir = args.output_dir / split / label
target_dir.mkdir(parents=True, exist_ok=True)
name = f"{tile['sample_slug']}__{Path(tile['image_path']).stem}__{proposal_index:04d}.jpg"
crop_square(source, box, args.crop_scale).save(target_dir / name, quality=92)
crop_path = target_dir / name
if crop_path.exists():
raise RuntimeError(f"proposal crop identity collision: {crop_path}")
crop_square(source, box, args.crop_scale).save(crop_path, quality=92)
crop_entries.append(
_crop_entry(
output_dir=args.output_dir,
crop_path=crop_path,
tile=tile,
label=label,
proposal_index=proposal_index,
proposal_score=score,
source_box_xyxy=box,
source_image_path=image_path,
source_image_sha256=image_sha256,
source_label_path=label_path,
source_label_sha256=label_sha256,
)
)
counts[f"{split}/{label}"] += 1
sample_counts[tile["sample_slug"]] += 1
for split in ("train", "val"):
for label in ("positive", "negative"):
if counts[f"{split}/{label}"] == 0:
raise RuntimeError(f"empty proposal class: {split}/{label}")
crop_entries.sort(key=lambda item: str(item["relative_path"]))
if len({str(item["relative_path"]) for item in crop_entries}) != len(crop_entries):
raise RuntimeError("proposal crop manifest contains duplicate relative paths")
source_summary_sha256 = sha256(args.summary)
corpus_freeze_path = args.corpus_manifest.parent / "corpus-freeze.json"
if not corpus_freeze_path.is_file(): # guarded above; retain a local invariant for provenance output
raise RuntimeError(f"governed corpus freeze is unavailable: {corpus_freeze_path}")
provenance: dict[str, Any] = {
"schema_version": PROPOSAL_DATASET_SCHEMA_VERSION,
"status": "ok",
"immutable": True,
"dataset_kind": "building_proposal_classifier_crops",
"fixture_mode": bool(args.fixture_mode),
"governed_corpus_live_recheck": True,
"source": {
"corpus_manifest": {
"path": str(args.corpus_manifest.expanduser().resolve(strict=False)),
"sha256": corpus_manifest_sha256,
},
"corpus_freeze": {
"path": str(corpus_freeze_path.resolve(strict=False)),
"sha256": sha256(corpus_freeze_path),
},
"summary": {
"path": str(args.summary.expanduser().resolve(strict=False)),
"sha256": source_summary_sha256,
"source_manifest_sha256": summary["source_manifest_sha256"],
"training_release_manifest": summary["training_release_manifest"],
"training_release_manifest_sha256": summary["training_release_manifest_sha256"],
"training_asset_manifest": summary["training_asset_manifest"],
},
"training_release": {
"dataset_yaml_path": source_release["dataset_yaml"]["path"],
"dataset_yaml_sha256": source_release["dataset_yaml"]["sha256"],
"corpus_manifest_sha256": source_release["corpus"]["manifest_sha256"],
},
"proposal_model": {
"path": str(model_path),
"sha256": model_sha256,
},
},
"parameters": {
"region": args.region,
"confidence": args.confidence,
"match_iou": args.match_iou,
"crop_scale": args.crop_scale,
"max_positive_per_tile": args.max_positive_per_tile,
"max_negative_per_tile": args.max_negative_per_tile,
"device": args.device,
"imgsz": args.imgsz,
},
"counts": dict(sorted(counts.items())),
"sample_counts": dict(sorted(sample_counts.items())),
"tile_count": len(tiles),
"crop_count": len(crop_entries),
"crops_sha256": hashlib.sha256(_canonical_json_bytes({"crops": crop_entries})).hexdigest(),
"crops": crop_entries,
}
provenance["manifest_sha256"] = _immutable_payload_sha256(provenance)
provenance_path = args.output_dir / PROPOSAL_DATASET_PROVENANCE_NAME
_write_immutable_json(provenance_path, provenance)
evidence = {
"schema_version": 1, "status": "ok", "model": str(args.model),
"model_sha256": sha256(args.model), "summary": str(args.summary),
"summary_sha256": sha256(args.summary), "corpus_manifest": str(args.corpus_manifest),
"corpus_manifest_sha256": sha256(args.corpus_manifest), "region": args.region,
"schema_version": PROPOSAL_DATASET_SCHEMA_VERSION,
"status": "ok",
"model": str(model_path),
"model_sha256": model_sha256,
"summary": str(args.summary),
"summary_sha256": source_summary_sha256,
"corpus_manifest": str(args.corpus_manifest),
"corpus_manifest_sha256": corpus_manifest_sha256,
"region": args.region,
"confidence": args.confidence, "match_iou": args.match_iou, "crop_scale": args.crop_scale,
"counts": dict(sorted(counts.items())), "sample_counts": dict(sorted(sample_counts.items())),
"protected_samples_in_training": [], "tile_count": len(tiles),
"fixture_mode": bool(args.fixture_mode),
"proposal_dataset_provenance": str(provenance_path),
"proposal_dataset_provenance_sha256": sha256(provenance_path),
"proposal_dataset_manifest_sha256": provenance["manifest_sha256"],
}
(args.output_dir / "proposal-dataset-evidence.json").write_text(
json.dumps(evidence, indent=2), encoding="utf-8"
)
_write_immutable_json(args.output_dir / PROPOSAL_DATASET_EVIDENCE_NAME, evidence)
print(json.dumps(evidence, indent=2))
return 0
+67 -4
View File
@@ -8,10 +8,25 @@ import hashlib
import json
import math
import re
import sys
from collections import Counter
from pathlib import Path
from typing import Any
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_embedded_training_release,
create_training_release_manifest,
)
PRECISION_NEGATIVE_CONTEXTS = {
"coastal-urban": {"port-hard-negative", "dunes-negative"},
@@ -80,6 +95,17 @@ def build_sampling(
) -> tuple[list[str], dict[str, Any]]:
if assessment.get("status") != "continue_training_loop":
raise ValueError("Failure-driven sampling requires a failed assessment")
protected_feedback = [
role
for role in ("test", "background")
if assessment.get(role) is not None
]
if protected_feedback:
raise ValueError(
"Failure-driven sampling is prohibited after protected "
+ "/".join(protected_feedback)
+ " evidence was opened"
)
if min(
positive_repeat,
negative_repeat,
@@ -97,9 +123,9 @@ def build_sampling(
samples = {item["sample_slug"]: item for item in manifest["samples"]}
gates = assessment["gates"]
evaluation = assessment.get("test") or assessment.get("calibration")
evaluation = assessment.get("calibration")
if not evaluation or "regions" not in evaluation:
raise ValueError("Assessment has no regional calibration or test evidence")
raise ValueError("Assessment has no regional calibration evidence")
regions = evaluation["regions"]
weak_recall_regions = {
region
@@ -246,7 +272,7 @@ def build_sampling(
"schema_version": 1,
"status": "ok",
"strategy": "failed-region-positive-and-hard-negative-repeat",
"failure_evidence_source": "test" if assessment.get("test") else "calibration",
"failure_evidence_source": "calibration",
"weak_recall_regions": sorted(weak_recall_regions),
"weak_precision_regions": sorted(weak_precision_regions),
"recall_dominant_regions": sorted(recall_dominant_regions),
@@ -290,6 +316,11 @@ def main() -> int:
parser.add_argument("--corpus-manifest", type=Path, required=True)
parser.add_argument("--assessment", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument(
"--review-audit",
type=Path,
help="Passed corpus audit containing accepted human-review evidence for the frozen corpus.",
)
parser.add_argument("--positive-repeat", type=int, default=3)
parser.add_argument("--negative-repeat", type=int, default=4)
parser.add_argument("--precision-positive-repeat", type=int, default=1)
@@ -299,8 +330,26 @@ def main() -> int:
parser.add_argument("--sampling-round", type=int)
parser.add_argument("--precision-guard-band", type=float, default=0.03)
parser.add_argument("--recall-guard-band", type=float, default=0.03)
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Accept only an explicitly fixture-only corpus manifest; never use for operational sampling.",
)
args = parser.parse_args()
try:
assert_frozen_manifest_training_eligible(
args.corpus_manifest,
fixture_mode=args.fixture_mode,
verify_live=True,
)
source_release = assert_yolo_summary_bound_to_embedded_training_release(
summary_path=args.summary,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except (TrainingEligibilityError, TrainingReleaseError) as exc:
raise SystemExit(str(exc)) from exc
summary = json.loads(args.summary.read_text(encoding="utf-8"))
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
assessment = json.loads(args.assessment.read_text(encoding="utf-8"))
@@ -325,7 +374,7 @@ def main() -> int:
args.output_dir.mkdir(parents=True, exist_ok=True)
train_list = args.output_dir / "train-failure-driven.txt"
train_list.write_text("\n".join(paths) + "\n", encoding="utf-8")
source_yaml = args.summary.parent / "dataset.yaml"
source_yaml = Path(str(source_release["dataset_yaml"]["path"]))
val_source = dataset_validation_source(source_yaml)
dataset_yaml = args.output_dir / "dataset.yaml"
dataset_yaml.write_text(
@@ -335,6 +384,15 @@ def main() -> int:
"names:\n 0: building\n",
encoding="utf-8",
)
try:
release_paths = create_training_release_manifest(
train_yaml=dataset_yaml,
corpus_manifest=args.corpus_manifest,
review_audit_path=args.review_audit,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
metadata.update(
{
"summary": str(args.summary),
@@ -346,6 +404,11 @@ def main() -> int:
"source_dataset_yaml": str(source_yaml),
"train_list": str(train_list),
"dataset_yaml": str(dataset_yaml),
"training_release_manifest": str(release_paths["release_manifest"]),
"training_release_manifest_sha256": file_sha256(release_paths["release_manifest"]),
"training_release_freeze": str(release_paths["release_freeze"]),
"training_asset_manifest": str(release_paths["asset_manifest"]),
"fixture_mode": bool(args.fixture_mode),
}
)
output = args.output_dir / "failure-driven-sampling.json"
+48
View File
@@ -7,11 +7,24 @@ import argparse
import hashlib
import json
import shutil
import sys
from pathlib import Path
from PIL import Image
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_training_release,
file_sha256,
training_release_paths,
)
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
@@ -23,9 +36,25 @@ def sha256(path: Path) -> str:
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--summary", type=Path, required=True)
parser.add_argument("--train-yaml", type=Path, required=True)
parser.add_argument("--corpus-manifest", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Accept only an explicitly fixture-only release; never creates a production-ready derived dataset.",
)
parser.add_argument("--force", action="store_true")
args = parser.parse_args()
try:
release = assert_yolo_summary_bound_to_training_release(
summary_path=args.summary,
train_yaml=args.train_yaml,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
if args.output_dir.exists():
if not args.force:
raise SystemExit(f"Output exists: {args.output_dir}")
@@ -54,6 +83,16 @@ def main() -> int:
f"path: {args.output_dir}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
encoding="utf-8",
)
# The source release binds the original image bytes. These transformed
# bytes cannot inherit it, so remove its operational release pointers and
# retain them only as explicitly non-trainable parent evidence below.
for field_name in (
"training_release_manifest",
"training_release_manifest_sha256",
"training_release_freeze",
"training_asset_manifest",
):
summary.pop(field_name, None)
summary.update(
{
"output_dir": str(args.output_dir),
@@ -71,10 +110,19 @@ def main() -> int:
"preprocessing": "luminance_rgb_replicated",
"source_summary": str(args.summary),
"source_summary_sha256": sha256(args.summary),
"source_training_release": str(training_release_paths(args.train_yaml)["release_manifest"]),
"source_training_release_sha256": file_sha256(
training_release_paths(args.train_yaml)["release_manifest"]
),
"source_corpus_manifest_sha256": release["corpus"]["manifest_sha256"],
"converted_tile_count": converted,
"output_summary": str(output_summary),
"output_summary_sha256": sha256(output_summary),
"dataset_yaml": str(dataset_yaml),
"training_eligible": False,
"training_eligibility_reason": (
"Derived image bytes require a new governed corpus, validation report and immutable training release."
),
}
(args.output_dir / "grayscale-dataset-evidence.json").write_text(
json.dumps(evidence, indent=2), encoding="utf-8"
+52
View File
@@ -6,10 +6,25 @@ from __future__ import annotations
import argparse
import hashlib
import json
import sys
from collections import Counter
from pathlib import Path
from typing import Any
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_embedded_training_release,
create_training_release_manifest,
)
PROTECTED_SPLITS = {"calibration", "test", "background-test"}
@@ -138,11 +153,34 @@ def main() -> int:
parser.add_argument("--priority-context", action="append", default=[])
parser.add_argument("--priority-repeat", type=int, default=2)
parser.add_argument("--negative-repeat", type=int, default=2)
parser.add_argument(
"--review-audit",
type=Path,
help="Accepted corpus-audit evidence; mandatory for an operational training release.",
)
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Accept only an explicitly fixture-only corpus manifest; never use for operational training data.",
)
args = parser.parse_args()
if args.priority_repeat < 1 or args.negative_repeat < 1:
parser.error("repeat counts must be positive")
summary = json.loads(args.summary.read_text(encoding="utf-8"))
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
try:
assert_frozen_manifest_training_eligible(
args.corpus_manifest,
fixture_mode=args.fixture_mode,
verify_live=True,
)
assert_yolo_summary_bound_to_embedded_training_release(
summary_path=args.summary,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except (TrainingEligibilityError, TrainingReleaseError) as exc:
raise SystemExit(str(exc)) from exc
train, val, evidence = build(
summary=summary,
manifest=manifest,
@@ -161,6 +199,15 @@ def main() -> int:
f"path: /\ntrain: {train_path}\nval: {val_path}\nnames:\n 0: building\n",
encoding="utf-8",
)
try:
release_paths = create_training_release_manifest(
train_yaml=yaml_path,
corpus_manifest=args.corpus_manifest,
review_audit_path=args.review_audit,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
evidence.update({
"source_summary": str(args.summary),
"source_summary_sha256": sha256(args.summary),
@@ -169,6 +216,11 @@ def main() -> int:
"train_sha256": sha256(train_path),
"validation_sha256": sha256(val_path),
"dataset_yaml": str(yaml_path),
"training_release_manifest": str(release_paths["release_manifest"]),
"training_release_manifest_sha256": sha256(release_paths["release_manifest"]),
"training_release_freeze": str(release_paths["release_freeze"]),
"training_asset_manifest": str(release_paths["asset_manifest"]),
"fixture_mode": bool(args.fixture_mode),
})
encoded_evidence = json.dumps(evidence, indent=2)
(args.output_dir / "regional-dataset-evidence.json").write_text(encoded_evidence, encoding="utf-8")
@@ -6,8 +6,23 @@ from __future__ import annotations
import argparse
import hashlib
import json
import sys
from pathlib import Path
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_embedded_training_release,
create_training_release_manifest,
)
def sha256(path: Path) -> str:
digest = hashlib.sha256()
@@ -31,12 +46,35 @@ def main() -> int:
default=0,
help="Include each train tile outside the expert region this many times.",
)
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Accept only an explicitly fixture-only corpus manifest; never use for operational training data.",
)
parser.add_argument(
"--review-audit",
type=Path,
help="Accepted corpus-audit evidence; mandatory for an operational training release.",
)
args = parser.parse_args()
if args.positive_repeat < 1 or args.negative_repeat < 1 or args.other_region_repeat < 0:
raise SystemExit("regional repeats must be positive and other-region repeat non-negative")
summary = json.loads(args.summary.read_text(encoding="utf-8"))
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
try:
assert_frozen_manifest_training_eligible(
args.corpus_manifest,
fixture_mode=args.fixture_mode,
verify_live=True,
)
assert_yolo_summary_bound_to_embedded_training_release(
summary_path=args.summary,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except (TrainingEligibilityError, TrainingReleaseError) as exc:
raise SystemExit(str(exc)) from exc
samples = {item["sample_slug"]: item for item in manifest["samples"]}
paths: list[str] = []
selected_samples: set[str] = set()
@@ -69,6 +107,15 @@ def main() -> int:
f"val: {args.summary.parent / 'images' / 'val'}\nnames:\n 0: building\n",
encoding="utf-8",
)
try:
release_paths = create_training_release_manifest(
train_yaml=dataset_yaml,
corpus_manifest=args.corpus_manifest,
review_audit_path=args.review_audit,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
evidence = {
"schema_version": 1,
"status": "ok",
@@ -88,6 +135,11 @@ def main() -> int:
"protected_samples_in_training": [],
"train_list": str(train_list),
"dataset_yaml": str(dataset_yaml),
"training_release_manifest": str(release_paths["release_manifest"]),
"training_release_manifest_sha256": sha256(release_paths["release_manifest"]),
"training_release_freeze": str(release_paths["release_freeze"]),
"training_asset_manifest": str(release_paths["asset_manifest"]),
"fixture_mode": bool(args.fixture_mode),
}
evidence_path = args.output_dir / "regional-expert-dataset.json"
evidence_path.write_text(json.dumps(evidence, indent=2), encoding="utf-8")
+47 -1
View File
@@ -9,6 +9,7 @@ new provider data.
from __future__ import annotations
import argparse
import hashlib
import json
import os
import shutil
@@ -16,6 +17,19 @@ import sys
from pathlib import Path
from typing import Any, Iterable
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
create_training_release_manifest,
)
DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/operator_samples_manifest.json")
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-dataset")
@@ -24,6 +38,14 @@ Transformer: Any = None
Image: Any = None
def file_sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Export operator real-data samples to a YOLO detection dataset.",
@@ -45,6 +67,12 @@ def parse_args() -> argparse.Namespace:
default=os.environ.get("OPERATOR_YOLO_VAL_SAMPLES", "turnhout"),
help="Comma/space separated sample slugs assigned to validation. Defaults to turnhout.",
)
parser.add_argument(
"--review-audit",
type=Path,
required=True,
help="Passed corpus audit with accepted human-review evidence for this frozen source manifest.",
)
parser.add_argument(
"--force",
action="store_true",
@@ -218,12 +246,16 @@ def ensure_yolo_directories(output_dir: Path) -> None:
def main() -> int:
args = parse_args()
manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
try:
assert_frozen_manifest_training_eligible(args.manifest_path, verify_live=True)
except TrainingEligibilityError as exc:
raise SystemExit(str(exc)) from exc
ensure_dependencies()
if args.force and args.output_dir.exists():
shutil.rmtree(args.output_dir)
args.output_dir.mkdir(parents=True, exist_ok=True)
ensure_yolo_directories(args.output_dir)
manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
samples = manifest.get("samples") or []
if not samples:
raise SystemExit("Operator sample manifest contains no samples")
@@ -234,12 +266,26 @@ def main() -> int:
if not any(item["split"] == "val" for item in exported):
raise SystemExit("YOLO dataset export produced no validation samples")
dataset_yaml = write_dataset_yaml(args.output_dir)
try:
release_paths = create_training_release_manifest(
train_yaml=dataset_yaml,
corpus_manifest=args.manifest_path,
review_audit_path=args.review_audit,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
summary = {
"status": "ok",
"dataset_yaml": str(dataset_yaml),
"training_release_manifest": str(release_paths["release_manifest"]),
"training_release_manifest_sha256": file_sha256(release_paths["release_manifest"]),
"training_release_freeze": str(release_paths["release_freeze"]),
"training_asset_manifest": str(release_paths["asset_manifest"]),
"output_dir": str(args.output_dir),
"class_names": ["building"],
"sample_count": len(exported),
"source_manifest": str(args.manifest_path),
"source_manifest_sha256": file_sha256(args.manifest_path),
"train_sample_count": sum(1 for item in exported if item["split"] == "train"),
"val_sample_count": sum(1 for item in exported if item["split"] == "val"),
"label_count": sum(item["label_count"] for item in exported),
+46 -1
View File
@@ -18,6 +18,19 @@ import sys
from pathlib import Path
from typing import Any, Iterable
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
create_training_release_manifest,
)
DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/operator_samples_manifest.json")
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-tile-dataset")
@@ -45,6 +58,14 @@ Transformer: Any = None
Image: Any = None
def file_sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
@dataclass(frozen=True)
class TileWindow:
row_off: int
@@ -87,6 +108,12 @@ def parse_args() -> argparse.Namespace:
default=os.environ.get("OPERATOR_YOLO_REFERENCE_SOURCE", DEFAULT_REFERENCE_SOURCE),
help="Required source_name in reference GeoJSON features.",
)
parser.add_argument(
"--review-audit",
type=Path,
required=True,
help="Passed corpus audit with accepted human-review evidence for this frozen source manifest.",
)
parser.add_argument(
"--reference-layer",
default=os.environ.get("OPERATOR_YOLO_REFERENCE_LAYER", DEFAULT_REFERENCE_LAYER),
@@ -626,13 +653,17 @@ def main() -> int:
):
if not value or any(character not in "abcdefghijklmnopqrstuvwxyz0123456789_-" for character in value):
raise SystemExit(f"YOLO {label} must be a non-empty canonical slug")
manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
try:
assert_frozen_manifest_training_eligible(args.manifest_path, verify_live=True)
except TrainingEligibilityError as exc:
raise SystemExit(str(exc)) from exc
ensure_dependencies()
if args.force and args.output_dir.exists():
shutil.rmtree(args.output_dir)
args.output_dir.mkdir(parents=True, exist_ok=True)
ensure_yolo_directories(args.output_dir)
manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
manifest_samples = manifest.get("samples") or []
if not manifest_samples:
raise SystemExit("Operator sample manifest contains no samples")
@@ -672,6 +703,14 @@ def main() -> int:
if not args.allow_empty_validation and not any(tile["split"] == "val" for tile in kept_tiles):
raise SystemExit("YOLO tile dataset export produced no validation tiles")
dataset_yaml = write_dataset_yaml(args.output_dir, class_name)
try:
release_paths = create_training_release_manifest(
train_yaml=dataset_yaml,
corpus_manifest=args.manifest_path,
review_audit_path=args.review_audit,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
positive_tiles = [tile for tile in kept_tiles if not tile["is_negative"]]
negative_tiles = [tile for tile in kept_tiles if tile["is_negative"]]
skipped_negative_tiles = [tile for tile in exported_tiles if not tile["kept"] and tile["is_negative"]]
@@ -684,6 +723,10 @@ def main() -> int:
summary = {
"status": "ok",
"dataset_yaml": str(dataset_yaml),
"training_release_manifest": str(release_paths["release_manifest"]),
"training_release_manifest_sha256": file_sha256(release_paths["release_manifest"]),
"training_release_freeze": str(release_paths["release_freeze"]),
"training_asset_manifest": str(release_paths["asset_manifest"]),
"output_dir": str(args.output_dir),
"class_names": [class_name],
"reference_source": reference_source,
@@ -697,6 +740,8 @@ def main() -> int:
"drop_low_variance_negatives": args.drop_low_variance_negatives,
"blank_range_threshold": args.blank_range_threshold,
"source_manifest_sample_count": len(manifest_samples),
"source_manifest": str(args.manifest_path),
"source_manifest_sha256": file_sha256(args.manifest_path),
"source_sample_count": len(samples),
"selected_sample_slugs": sorted(
str(sample.get("sample_slug") or "").strip().lower() for sample in samples
@@ -16,6 +16,12 @@ from dataclasses import dataclass, replace
from pathlib import Path
from typing import Any
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import TRAINING_ELIGIBILITY_POLICY_VERSION # noqa: E402
WMS_URL = "https://geo.api.vlaanderen.be/omwrgbmrvl/wms"
GRB_GBG_URL = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items"
@@ -803,6 +809,14 @@ def main() -> int:
manifest = {
"schema_version": 2,
"description": "GeoIntel operator real-data samples for configured-YOLO QA validation.",
"training_eligibility": {
"policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION,
"status": "not_eligible",
"reason": (
"Direct provider downloads are QA-only until re-ingested through the governed "
"dataset source registry, contract validator and provenance snapshot flow."
),
},
"output_dir": str(output_dir),
"sample_width": args.width,
"sample_height": args.height,
+1 -1
View File
@@ -626,7 +626,7 @@ def provision_dataset(
from app.services.dataset_service import DatasetService
partition_checksums = {
summary["nis_code"]: summary["sha256"]
summary["filename"]: summary["sha256"]
for summary in manifest["partitions"]
}
metadata_json = {
+1 -2
View File
@@ -43,7 +43,6 @@ from provision_regional_grb_buildings import (
list_paginated_items,
next_page_url,
observed_at,
response_data,
reusable_manifest,
safe_slug,
sha256_file,
@@ -758,7 +757,7 @@ def provision_dataset(
"source_urls": manifest["grb_source_urls"],
"artifact_sha256": manifest["artifact_sha256"],
"artifact_size_bytes": manifest["artifact_size_bytes"],
"partition_checksums": {summary["nis_code"]: summary["sha256"] for summary in manifest["partitions"]},
"partition_checksums": {summary["filename"]: summary["sha256"] for summary in manifest["partitions"]},
"partition_assignment_rule": manifest["partition_assignment_rule"],
"reference_truncated": False,
}
+50 -1
View File
@@ -10,8 +10,20 @@ import json
import math
import os
import shutil
import sys
from pathlib import Path
from typing import Any
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_training_release,
file_sha256,
training_release_paths,
)
def sha256(path: Path) -> str:
@@ -97,6 +109,8 @@ def link_or_copy(source: Path, target: Path) -> None:
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--summary", type=Path, required=True)
parser.add_argument("--train-yaml", type=Path, required=True)
parser.add_argument("--corpus-manifest", type=Path, required=True)
parser.add_argument("--model", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--device", default="cuda:0")
@@ -110,7 +124,21 @@ def main() -> int:
parser.add_argument("--max-prompts-per-pass", type=int, default=96)
parser.add_argument("--fallback-policy", choices=("retain", "drop"), default="retain")
parser.add_argument("--force", action="store_true")
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Accept only an explicitly fixture-only source release; refined labels remain non-trainable.",
)
args = parser.parse_args()
try:
source_release = assert_yolo_summary_bound_to_training_release(
summary_path=args.summary,
train_yaml=args.train_yaml,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
if args.output_dir.exists():
if not args.force:
raise SystemExit(f"Output exists: {args.output_dir}")
@@ -194,6 +222,16 @@ def main() -> int:
print(f"{index}/{len(summary['tiles'])} {tile['sample_slug']}: {len(source_boxes)}", flush=True)
output_summary = dict(summary)
# SAM changes label bytes and semantics. It therefore cannot inherit the
# source release; keep the parent evidence separately and require a new
# governed corpus/review/release before any training consumer can use it.
for field_name in (
"training_release_manifest",
"training_release_manifest_sha256",
"training_release_freeze",
"training_asset_manifest",
):
output_summary.pop(field_name, None)
output_summary.update(
{
"output_dir": str(args.output_dir),
@@ -207,6 +245,10 @@ def main() -> int:
"label_count": refined_count if args.fallback_policy == "drop" else summary["label_count"],
"positive_tile_count": sum(not tile["is_negative"] for tile in output_tiles),
"negative_tile_count": sum(tile["is_negative"] for tile in output_tiles),
"training_eligible": False,
"training_eligibility_reason": (
"SAM-refined labels require a new governed corpus, validation report, human review and immutable training release."
),
}
)
summary_path = args.output_dir / "yolo_tile_dataset_summary.json"
@@ -220,6 +262,11 @@ def main() -> int:
"status": "ok",
"source_summary": str(args.summary),
"source_summary_sha256": sha256(args.summary),
"source_training_release": str(training_release_paths(args.train_yaml)["release_manifest"]),
"source_training_release_sha256": file_sha256(
training_release_paths(args.train_yaml)["release_manifest"]
),
"source_corpus_manifest_sha256": source_release["corpus"]["manifest_sha256"],
"sam_model": str(args.model),
"sam_model_sha256": sha256(args.model),
"device": args.device,
@@ -236,6 +283,8 @@ def main() -> int:
"fallback_label_count": fallback_count,
"dropped_fallback_label_count": fallback_count if args.fallback_policy == "drop" else 0,
"fallback_reason_counts": reason_counts,
"training_eligible": False,
"fixture_mode": bool(args.fixture_mode),
}
(args.output_dir / "sam-refinement.json").write_text(json.dumps(evidence, indent=2), encoding="utf-8")
print(json.dumps(evidence, indent=2))
@@ -7,6 +7,7 @@ import argparse
import hashlib
import json
import shutil
import sys
from collections import Counter
from pathlib import Path
from typing import Any
@@ -15,6 +16,19 @@ from pyproj import Transformer
from shapely.geometry import box
from shapely.ops import transform as shapely_transform
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_embedded_training_release,
)
def sha256(path: Path) -> str:
digest = hashlib.sha256()
@@ -73,6 +87,11 @@ def main() -> int:
parser.add_argument("--internal-val-samples", required=True)
parser.add_argument("--version", required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Accept only an explicitly fixture-only source corpus; never use for operational holdout rotation.",
)
args = parser.parse_args()
role_slugs = {
@@ -86,6 +105,20 @@ def main() -> int:
raise SystemExit("rotated holdout lists overlap")
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
try:
assert_frozen_manifest_training_eligible(
args.corpus_manifest,
fixture_mode=args.fixture_mode,
verify_live=True,
)
for path in args.source_summary:
assert_yolo_summary_bound_to_embedded_training_release(
summary_path=path,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except (TrainingEligibilityError, TrainingReleaseError) as exc:
raise SystemExit(str(exc)) from exc
source_samples = {item["sample_slug"]: item for item in manifest["samples"]}
missing = sorted(all_holdouts - source_samples.keys())
if missing:
@@ -126,6 +159,18 @@ def main() -> int:
args.output_dir.mkdir(parents=True, exist_ok=True)
manifest_path = args.output_dir / "operator_samples_manifest.json"
write_json(manifest_path, rotated_manifest)
write_json(
args.output_dir / "corpus-freeze.json",
{
"schema_version": 2,
"dataset_version": rotated_manifest["dataset_version"],
"manifest_sha256": sha256(manifest_path),
"sample_count": len(rotated_manifest["samples"]),
"immutable": True,
"training_eligibility_policy": rotated_manifest["training_eligibility"]["policy_version"],
"fixture_mode": bool(args.fixture_mode),
},
)
leakage = audit_spatial_leakage(rotated_manifest["samples"])
write_json(args.output_dir / "spatial-leakage-audit.json", leakage)
if leakage["status"] != "ok":
@@ -193,6 +238,11 @@ def main() -> int:
"fit_samples_in_internal_validation": sorted(
{tile["sample_slug"] for tile in train_tiles} & {tile["sample_slug"] for tile in internal_val_tiles}
),
"fixture_mode": bool(args.fixture_mode),
"training_eligible": False,
"training_eligibility_reason": (
"A rotated split needs a new human review, label contract validation and immutable training release."
),
}
if evidence["protected_samples_in_training"]:
raise SystemExit("protected samples leaked into rotated training lists")
@@ -0,0 +1,133 @@
#!/usr/bin/env python3
"""Emit reproducible local evidence for the Phase-2 data foundation.
The collector does not connect to PostgreSQL, source providers, a GPU or any
training corpus. It inventories the versioned source-policy and contract
definitions that are present in this checkout. Runtime migration/application
evidence is recorded separately because it needs an explicitly chosen target
database and must never be inferred from this static report.
"""
from __future__ import annotations
import argparse
import json
import subprocess
import sys
from collections import Counter
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parents[1]
BACKEND = ROOT / "backend"
if str(BACKEND) not in sys.path:
sys.path.insert(0, str(BACKEND))
from app.services.data_contract_validation import build_default_data_contract_registry # noqa: E402
from app.services.source_registry_service import SERVER_OWNED_SOURCE_DEFINITIONS # noqa: E402
def _git_value(*args: str) -> str | None:
try:
return subprocess.check_output(
["git", *args],
cwd=ROOT,
text=True,
stderr=subprocess.DEVNULL,
).strip() or None
except (OSError, subprocess.CalledProcessError):
return None
def collect() -> dict[str, Any]:
definitions = list(SERVER_OWNED_SOURCE_DEFINITIONS.values())
contracts = build_default_data_contract_registry().registered_contracts()
classification_counts = Counter(item.classification for item in definitions)
source_items = [
{
"source_key": item.source_key,
"classification": item.classification,
"authority_name": item.authority_name,
"authority_scope": item.authority_scope,
"provider_adapter_key": item.provider_adapter_key,
"default_crs": item.default_crs,
"default_units": item.default_units,
"freshness_status": item.freshness_status,
"ingest_status": item.ingest_status,
"ground_truth_allowed": bool((item.usage_policy or {}).get("ground_truth_allowed")),
"training_allowed": bool((item.usage_policy or {}).get("training_allowed")),
}
for item in sorted(definitions, key=lambda definition: definition.source_key)
]
contract_items = [
{
"key": item.key,
"version": item.version,
"kind": item.kind.value,
"fingerprint_sha256": item.fingerprint(),
"canonical_storage_crs": item.canonical_storage_crs,
"accepted_source_crs": sorted(item.accepted_source_crs),
"required_metadata_fields": list(item.required_metadata_fields),
"requires_source_registry": item.lineage_rules.require_source_registry,
"requires_source_snapshot": item.lineage_rules.require_source_snapshot,
"requires_upstream_assets": item.lineage_rules.require_upstream_assets,
}
for item in sorted(contracts, key=lambda contract: (contract.kind.value, contract.key, contract.version))
]
return {
"schema_version": 1,
"program": "GeoIntel Accuracy Improvement Program",
"phase": "P2",
"collected_at": datetime.now(UTC).isoformat(),
"repository": {
"branch": _git_value("branch", "--show-current"),
"head": _git_value("rev-parse", "HEAD"),
"dirty": bool(_git_value("status", "--porcelain")),
},
"scope": "Belgium and the Belgian North Sea",
"migration_revision": "202608010001",
"source_registry": {
"definition_count": len(source_items),
"classification_counts": dict(sorted(classification_counts.items())),
"required_building_policy": {
"grb_primary_building_validation": SERVER_OWNED_SOURCE_DEFINITIONS["grb"].usage_policy[
"validation_authority"
].get("building_validation"),
"buildings_register_classification": SERVER_OWNED_SOURCE_DEFINITIONS[
"digitaal_vlaanderen_buildings_addresses_register"
].classification,
"sentinel_2_classification": SERVER_OWNED_SOURCE_DEFINITIONS["sentinel_2"].classification,
"dhmv_classification": SERVER_OWNED_SOURCE_DEFINITIONS["digitaal_vlaanderen_dhmv"].classification,
"osm_ground_truth_allowed": SERVER_OWNED_SOURCE_DEFINITIONS["osm"].usage_policy[
"ground_truth_allowed"
],
},
"definitions": source_items,
},
"data_contracts": contract_items,
"claim_boundary": (
"Static policy/contract inventory only. It does not attest that a database migration ran, "
"that legacy rows are complete, or that a model/corpus is release-ready."
),
}
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument(
"--output",
type=Path,
default=ROOT / "artifacts" / "evidence" / "accuracy" / "P2" / "source-contract-inventory.json",
)
args = parser.parse_args()
result = collect()
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
print(json.dumps({"output": str(args.output), "source_count": result["source_registry"]["definition_count"]}))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+172 -5
View File
@@ -13,6 +13,21 @@ from datetime import UTC, datetime
from pathlib import Path
from typing import Any
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_training_release_eligible,
human_review_audit_failures,
training_release_paths,
)
def sha256(path: Path) -> str:
digest = hashlib.sha256()
@@ -32,11 +47,14 @@ def write_json(path: Path, value: dict[str, Any]) -> None:
def dataset_audit_failures(
audit: dict[str, Any],
train_quality_audit: dict[str, Any],
*,
fixture_mode: bool = False,
) -> list[str]:
"""Return automated corpus blockers while leaving final human review deferred."""
"""Return fail-closed corpus blockers, including human review in normal mode."""
failures = [str(item) for item in audit.get("failures") or []]
status = audit.get("status")
if status not in {"ok", "needs_human_review"}:
permitted_statuses = {"ok", "needs_human_review"} if fixture_mode else {"ok"}
if status not in permitted_statuses:
failures.append(f"unsupported audit status: {status}")
if audit.get("manifest_immutable") is not True:
failures.append("corpus manifest is not immutable")
@@ -50,9 +68,56 @@ def dataset_audit_failures(
failures.append("train tile quality audit contains missing label files")
if int(train_quality_audit.get("low_variance_positive_tile_count", -1)) != 0:
failures.append("dataset contains blank/low-variance positive tiles")
if not fixture_mode:
failures.extend(human_review_audit_failures(audit))
return failures
def verify_training_inputs(
*,
train_yaml: Path,
corpus_manifest: Path,
fixture_mode: bool,
) -> dict[str, Any]:
"""Re-check every immutable input before initial, retry or resume training."""
try:
assert_frozen_manifest_training_eligible(
corpus_manifest,
fixture_mode=fixture_mode,
verify_live=True,
)
return assert_training_release_eligible(
train_yaml=train_yaml,
corpus_manifest=corpus_manifest,
fixture_mode=fixture_mode,
)
except (TrainingEligibilityError, TrainingReleaseError) as exc:
raise TrainingReleaseError(str(exc)) from exc
def assert_dataset_audit_bound_to_release(
*,
release: dict[str, Any],
dataset_audit: Path,
fixture_mode: bool,
) -> None:
"""Do not let a caller swap the reviewed corpus audit after release sealing."""
if fixture_mode:
return
review = release.get("human_review")
if not isinstance(review, dict):
raise TrainingReleaseError("Training release has no human-review audit binding")
review_audit_path = review.get("audit_path")
if not isinstance(review_audit_path, str) or not review_audit_path:
raise TrainingReleaseError("Training release human-review audit path is missing")
if Path(review_audit_path).resolve(strict=False) != dataset_audit.resolve(strict=False):
raise TrainingReleaseError(
"--dataset-audit does not match the immutable training-release human-review audit"
)
def select_calibration_threshold(report: dict[str, Any]) -> dict[str, Any]:
"""Choose a threshold without consulting test or background evidence."""
eligible = [item for item in report["sweeps"] if item["pure_empty_false_positives"] == 0]
@@ -111,6 +176,16 @@ def rejected_candidate_score(assessment: dict[str, Any]) -> tuple[float, ...]:
return tuple(normalized)
def protected_feedback_roles(assessment: dict[str, Any]) -> list[str]:
"""Return protected evidence roles that make iterative retraining illegal."""
return [
role
for role in ("test", "background")
if assessment.get(role) is not None
]
def training_command(
yolo: str,
*,
@@ -183,17 +258,23 @@ def failure_sampling_command(
corpus_manifest: Path,
assessment: Path,
output_dir: Path,
review_audit: Path,
sampling_round: int = 0,
fixture_mode: bool = False,
) -> list[str]:
return [
command = [
sys.executable,
str(scripts_dir / "build_failure_driven_yolo_sampling.py"),
"--summary", str(train_summary),
"--corpus-manifest", str(corpus_manifest),
"--assessment", str(assessment),
"--output-dir", str(output_dir),
"--review-audit", str(review_audit),
"--sampling-round", str(sampling_round),
]
if fixture_mode:
command.append("--fixture-mode")
return command
def resumable_training_command(yolo: str, checkpoint: Path) -> list[str]:
@@ -259,6 +340,14 @@ def main() -> int:
parser.add_argument("--min-region-recall", type=float, default=0.4)
parser.add_argument("--max-pure-empty-fp", type=int, default=0)
parser.add_argument("--dry-run", action="store_true")
parser.add_argument(
"--fixture-mode",
action="store_true",
help=(
"Accept an explicitly fixture-only corpus manifest. "
"This mode is prohibited for operational training."
),
)
parser.add_argument(
"--evaluate-initial-model",
action="store_true",
@@ -267,9 +356,29 @@ def main() -> int:
args = parser.parse_args()
if args.iterations < 1:
raise SystemExit("--iterations must be positive")
try:
initial_release = verify_training_inputs(
train_yaml=args.train_yaml,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
try:
assert_dataset_audit_bound_to_release(
release=initial_release,
dataset_audit=args.dataset_audit,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise SystemExit(str(exc)) from exc
dataset_audit = json.loads(args.dataset_audit.read_text(encoding="utf-8"))
train_quality_audit = json.loads(args.train_quality_audit.read_text(encoding="utf-8"))
audit_failures = dataset_audit_failures(dataset_audit, train_quality_audit)
audit_failures = dataset_audit_failures(
dataset_audit,
train_quality_audit,
fixture_mode=args.fixture_mode,
)
if audit_failures:
raise SystemExit(f"Dataset audit is not eligible for training: {audit_failures}")
@@ -285,6 +394,13 @@ def main() -> int:
"train_quality_audit": str(args.train_quality_audit),
"train_quality_audit_sha256": sha256(args.train_quality_audit),
"corpus_manifest": str(args.corpus_manifest),
"corpus_manifest_sha256": sha256(args.corpus_manifest),
"initial_training_release": str(training_release_paths(args.train_yaml)["release_manifest"]),
"initial_training_release_sha256": sha256(
training_release_paths(args.train_yaml)["release_manifest"]
),
"initial_training_release_contract": initial_release["contract_version"],
"fixture_mode": bool(args.fixture_mode),
"iterations": [],
}
if state_path.is_file():
@@ -304,6 +420,26 @@ def main() -> int:
evaluate_existing = args.evaluate_initial_model and offset == 0 and not state["iterations"]
partial_checkpoint = train_run / "weights" / "last.pt"
resume_partial = not evaluate_existing and partial_checkpoint.is_file()
try:
release = verify_training_inputs(
train_yaml=train_yaml,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise RuntimeError(
f"Training inputs changed before {name}; refusing initial/retry/resume execution: {exc}"
) from exc
try:
assert_dataset_audit_bound_to_release(
release=release,
dataset_audit=args.dataset_audit,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise RuntimeError(
f"Training audit changed before {name}; refusing initial/retry/resume execution: {exc}"
) from exc
command = None if evaluate_existing else (
resumable_training_command(args.yolo, partial_checkpoint)
if resume_partial else training_command(
@@ -454,11 +590,27 @@ def main() -> int:
"candidate_sha256": sha256(candidate),
"training_skipped_for_existing_checkpoint": evaluate_existing,
"training_resumed_from_partial_checkpoint": resume_partial,
"training_release": str(training_release_paths(train_yaml)["release_manifest"]),
"training_release_sha256": sha256(training_release_paths(train_yaml)["release_manifest"]),
"training_release_asset_manifest_sha256": release["asset_manifest"]["sha256"],
"assessment": str(assessment),
"status": decision["status"],
"failures": decision["failures"],
}
state["iterations"].append(record)
protected_feedback = protected_feedback_roles(decision)
if decision["status"] != "training_complete" and protected_feedback:
record["protected_feedback_blocked"] = protected_feedback
record["retraining_prohibited"] = True
state["status"] = "protected_evaluation_rejected"
state["stopped_at"] = datetime.now(UTC).isoformat()
state["stop_reason"] = (
"Protected test/background evidence was opened for a rejected candidate; "
"its results cannot generate another training YAML."
)
write_json(state_path, state)
print(json.dumps(state, indent=2))
return 3
score = rejected_candidate_score(decision) if decision["status"] != "training_complete" else ()
incumbent_score = tuple(state.get("incumbent_rejected_score", ()))
if not incumbent_score or score > incumbent_score:
@@ -482,14 +634,29 @@ def main() -> int:
corpus_manifest=args.corpus_manifest,
assessment=assessment,
output_dir=sampling_dir,
review_audit=args.dataset_audit,
sampling_round=index,
fixture_mode=args.fixture_mode,
),
iteration_dir / "failure-driven-sampling.log",
)
sampling_evidence = sampling_dir / "failure-driven-sampling.json"
next_train_yaml = sampling_dir / "dataset.yaml"
if not sampling_evidence.is_file() or not next_train_yaml.is_file():
next_release_paths = training_release_paths(next_train_yaml)
if not sampling_evidence.is_file() or not next_train_yaml.is_file() or not all(
path.is_file() for path in next_release_paths.values()
):
raise RuntimeError("Failure-driven sampling produced incomplete evidence")
try:
verify_training_inputs(
train_yaml=next_train_yaml,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
raise RuntimeError(
f"Failure-driven sampling produced an unbound training release: {exc}"
) from exc
record["failure_driven_sampling"] = str(sampling_evidence)
record["failure_driven_sampling_sha256"] = sha256(sampling_evidence)
record["next_train_yaml"] = str(next_train_yaml)
+48 -5
View File
@@ -3,6 +3,8 @@ from __future__ import annotations
import argparse
import json
import sys
from datetime import UTC, datetime
from hashlib import sha256
from pathlib import Path
from uuid import uuid4
@@ -12,9 +14,10 @@ BACKEND_ROOT = ROOT / "backend"
if str(BACKEND_ROOT) not in sys.path:
sys.path.insert(0, str(BACKEND_ROOT))
from app.models import Area, Dataset, Metric, QualityCheck # noqa: E402
from app.models import Dataset, Metric, QualityCheck, SourceRegistry, SourceSnapshot # noqa: E402
from app.services.qa_service import QaService # noqa: E402
from app.services.quality_service import QualityService # noqa: E402
from app.services.source_registry_service import SourceRegistryService # noqa: E402
class BenchmarkSession:
@@ -55,18 +58,58 @@ def _load_manifest() -> dict:
def _dataset(dataset_id, project_id, name: str, path: Path, *, role: str) -> Dataset:
# The benchmark is still synthetic evidence, but it intentionally models
# the same complete Phase 2 provenance binding required at the QA service
# boundary. The candidate is a derived fixture; the reference simulates
# a governed GRB building snapshot. No legacy fixture exception is used.
source_key = "grb" if role == "reference" else "derived"
source = SourceRegistry(
id=uuid4(),
**SourceRegistryService.definition_for(source_key).as_model_values(),
)
checksum = sha256(path.read_bytes()).hexdigest()
snapshot = SourceSnapshot(
id=uuid4(),
source_registry_id=source.id,
snapshot_key=f"golden-qa:{source_key}:{checksum}",
checksum_sha256=checksum,
fetched_at=datetime.now(UTC),
crs="EPSG:4326",
units=source.default_units,
spatial_resolution_json={"status": "fixture"},
temporal_coverage_json={"status": "fixture"},
geographic_coverage_json={"scope": "golden-qa-fixture"},
observed_schema_json={"dataset_type": "vector", "fixture_mode": True},
freshness_status="current",
ingest_status="ingested",
known_limitations_json=["Synthetic golden benchmark fixture; not production evidence."],
snapshot_metadata_json={"fixture_mode": True, "benchmark": "golden-qa"},
)
return Dataset(
id=dataset_id,
project_id=project_id,
name=name,
dataset_type="vector",
source="golden_fixture",
dataset_role=role,
source_name="fixture",
source="governed golden benchmark fixture",
dataset_role="reference" if role == "reference" else "derived",
source_name=source.source_key,
reference_layer_name="buildings" if role == "reference" else None,
storage_path=str(path),
crs="EPSG:4326",
metadata_json={"crs_assumed": False},
checksum_sha256=checksum,
source_registry_id=source.id,
source_snapshot_id=snapshot.id,
source_registry=source,
source_snapshot=snapshot,
data_contract_key="geointel.vector.geojson",
data_contract_version="1.0.0",
validation_status="passed",
provenance_status="complete",
lineage_status="not_applicable",
quarantine_status="not_quarantined",
metadata_json={"crs_assumed": False, "fixture_mode": True},
source_metadata={"fixture_mode": True, "source_registry_key": source.source_key},
provenance_metadata={"fixture_mode": True, "source_snapshot_id": str(snapshot.id)},
status="ready",
)
+4
View File
@@ -109,6 +109,10 @@ ${PYTHON_BIN} -m py_compile scripts/audit_source_freshness.py
${PYTHON_BIN} -m py_compile scripts/manage_grb_refresh.py
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py
${PYTHON_BIN} -m py_compile scripts/training_dataset_eligibility.py
${PYTHON_BIN} -m py_compile scripts/training_release_manifest.py
${PYTHON_BIN} -m py_compile scripts/run_belgium_building_training_loop.py
${PYTHON_BIN} -m py_compile scripts/build_failure_driven_yolo_sampling.py
${PYTHON_BIN} -m py_compile scripts/audit_operator_yolo_dataset_quality.py
${PYTHON_BIN} -m py_compile scripts/build_detection_model_promotion_report.py
${PYTHON_BIN} -m py_compile scripts/build_mol_operational_benchmark_report.py
@@ -6,10 +6,20 @@ from __future__ import annotations
import argparse
import json
import subprocess
import sys
import time
from datetime import UTC, datetime
from pathlib import Path
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_training_release_eligible,
)
def container_running(container: str) -> bool:
result = subprocess.run(
@@ -44,11 +54,33 @@ def load_completion_command(path: Path) -> list[str]:
return command
def verify_training_release(
*,
train_yaml: Path,
corpus_manifest: Path,
fixture_mode: bool,
) -> None:
"""Re-check the exact release before each detached resume command."""
assert_training_release_eligible(
train_yaml=train_yaml,
corpus_manifest=corpus_manifest,
fixture_mode=fixture_mode,
)
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--container", required=True)
parser.add_argument("--host-run-dir", type=Path, required=True)
parser.add_argument("--container-checkpoint", required=True)
parser.add_argument("--train-yaml", type=Path, required=True)
parser.add_argument("--corpus-manifest", type=Path, required=True)
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Only valid for an explicitly fixture-only frozen corpus release.",
)
parser.add_argument("--run-marker", required=True)
parser.add_argument("--yolo", default="/opt/geointel/venv/bin/yolo")
parser.add_argument("--poll-seconds", type=int, default=30)
@@ -101,6 +133,17 @@ def main() -> int:
state["status"] = "resume_budget_exhausted"
write_state(state_path, state)
return 3
try:
verify_training_release(
train_yaml=args.train_yaml,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except TrainingReleaseError as exc:
state["status"] = "training_release_verification_failed"
state["training_release_error"] = str(exc)
write_state(state_path, state)
return 6
result = subprocess.run(
["docker", "exec", "-d", args.container, args.yolo, "train",
f"resume={args.container_checkpoint}", "device=0"],
+302 -7
View File
@@ -4,8 +4,248 @@
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
import sys
from collections import Counter
from pathlib import Path, PurePosixPath
from typing import Any, Mapping
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_embedded_training_release,
)
PROPOSAL_DATASET_PROVENANCE_NAME = "proposal-dataset-provenance.json"
PROPOSAL_DATASET_SCHEMA_VERSION = 1
_CROP_IMAGE_SUFFIXES = {".bmp", ".jpeg", ".jpg", ".png", ".tif", ".tiff", ".webp"}
class ProposalDatasetProvenanceError(ValueError):
"""Raised when proposal crops no longer match their governed provenance."""
def sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def _canonical_json_bytes(payload: Mapping[str, Any]) -> bytes:
return json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
def _payload_sha256(payload: Mapping[str, Any]) -> str:
normalized = dict(payload)
normalized.pop("manifest_sha256", None)
return hashlib.sha256(_canonical_json_bytes(normalized)).hexdigest()
def _is_sha256(value: Any) -> bool:
return isinstance(value, str) and len(value) == 64 and all(character in "0123456789abcdef" for character in value)
def _require_mapping(value: Any, *, field: str) -> Mapping[str, Any]:
if not isinstance(value, Mapping):
raise ProposalDatasetProvenanceError(f"proposal dataset provenance {field} must be an object")
return value
def _require_sha256(value: Any, *, field: str) -> str:
if not _is_sha256(value):
raise ProposalDatasetProvenanceError(f"proposal dataset provenance {field} must be a lowercase SHA-256")
return str(value)
def _require_nonempty_path(value: Any, *, field: str) -> str:
if not isinstance(value, str) or not value.strip():
raise ProposalDatasetProvenanceError(f"proposal dataset provenance {field} must be a non-empty path")
return value
def _safe_relative_crop_path(value: Any, *, dataset_root: Path) -> tuple[str, Path]:
if not isinstance(value, str) or not value or "\\" in value:
raise ProposalDatasetProvenanceError("proposal crop relative_path must be a non-empty POSIX relative path")
relative = PurePosixPath(value)
if relative.is_absolute() or ".." in relative.parts or "." in relative.parts:
raise ProposalDatasetProvenanceError(f"proposal crop has an unsafe relative path: {value!r}")
path = (dataset_root / Path(*relative.parts)).resolve(strict=False)
try:
path.relative_to(dataset_root)
except ValueError as exc: # pragma: no cover - resolved-path defence
raise ProposalDatasetProvenanceError(f"proposal crop escapes dataset directory: {value!r}") from exc
return relative.as_posix(), path
def load_governed_corpus_manifest(corpus_manifest_path: Path, *, fixture_mode: bool) -> dict[str, Any]:
"""Assert frozen evidence and current live Dataset eligibility before PyTorch."""
try:
manifest = assert_frozen_manifest_training_eligible(
corpus_manifest_path,
fixture_mode=fixture_mode,
verify_live=True,
)
except TrainingEligibilityError as exc:
raise ProposalDatasetProvenanceError(str(exc)) from exc
if not isinstance(manifest, dict): # pragma: no cover - defensive contract boundary
raise ProposalDatasetProvenanceError("governed corpus manifest must be a JSON object")
return manifest
def validate_proposal_dataset_provenance(
dataset_dir: Path,
corpus_manifest_path: Path,
*,
fixture_mode: bool,
) -> dict[str, Any]:
"""Fail closed unless every trainable crop still matches its source evidence.
ImageFolder discovers files from the directory tree. Merely verifying a
manifest is insufficient: any unbound image placed under train/ or val/
would otherwise enter PyTorch training. This validator compares the
complete tree with the immutable crop manifest and re-binds it to the
supplied frozen corpus and summary.
"""
root = dataset_dir.expanduser().resolve(strict=False)
if not root.is_dir():
raise ProposalDatasetProvenanceError(f"proposal dataset directory is unavailable: {root}")
manifest_path = root / PROPOSAL_DATASET_PROVENANCE_NAME
try:
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise ProposalDatasetProvenanceError(f"proposal dataset provenance is unreadable: {manifest_path}") from exc
if not isinstance(payload, dict):
raise ProposalDatasetProvenanceError("proposal dataset provenance must be a JSON object")
if payload.get("schema_version") != PROPOSAL_DATASET_SCHEMA_VERSION:
raise ProposalDatasetProvenanceError("proposal dataset provenance schema_version is unsupported")
if payload.get("status") != "ok" or payload.get("immutable") is not True:
raise ProposalDatasetProvenanceError("proposal dataset provenance is not an immutable successful release")
if payload.get("governed_corpus_live_recheck") is not True:
raise ProposalDatasetProvenanceError("proposal dataset provenance lacks the governed live-source recheck")
if bool(payload.get("fixture_mode")) != fixture_mode:
raise ProposalDatasetProvenanceError("proposal dataset provenance fixture-mode binding mismatches this invocation")
if payload.get("manifest_sha256") != _payload_sha256(payload):
raise ProposalDatasetProvenanceError("proposal dataset provenance manifest checksum mismatches its content")
expected_corpus_sha256 = sha256(corpus_manifest_path)
source = _require_mapping(payload.get("source"), field="source")
recorded_corpus = _require_mapping(source.get("corpus_manifest"), field="source.corpus_manifest")
if _require_sha256(recorded_corpus.get("sha256"), field="source.corpus_manifest.sha256") != expected_corpus_sha256:
raise ProposalDatasetProvenanceError("proposal crops are not bound to the supplied governed corpus manifest")
corpus_freeze_path = corpus_manifest_path.parent / "corpus-freeze.json"
recorded_freeze = _require_mapping(source.get("corpus_freeze"), field="source.corpus_freeze")
if not corpus_freeze_path.is_file() or _require_sha256(
recorded_freeze.get("sha256"), field="source.corpus_freeze.sha256"
) != sha256(corpus_freeze_path):
raise ProposalDatasetProvenanceError("proposal crops are not bound to the current frozen corpus sidecar")
summary = _require_mapping(source.get("summary"), field="source.summary")
summary_path_value = _require_nonempty_path(summary.get("path"), field="source.summary.path")
summary_path = Path(summary_path_value).expanduser().resolve(strict=False)
if not summary_path.is_file():
raise ProposalDatasetProvenanceError(f"proposal source summary is unavailable: {summary_path}")
if _require_sha256(summary.get("sha256"), field="source.summary.sha256") != sha256(summary_path):
raise ProposalDatasetProvenanceError("proposal source summary checksum no longer matches the crop provenance")
try:
summary_payload = json.loads(summary_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as exc:
raise ProposalDatasetProvenanceError(f"proposal source summary is unreadable: {summary_path}") from exc
if not isinstance(summary_payload, Mapping) or summary_payload.get("source_manifest_sha256") != expected_corpus_sha256:
raise ProposalDatasetProvenanceError("proposal source summary is not bound to the supplied governed corpus manifest")
if summary.get("source_manifest_sha256") != expected_corpus_sha256:
raise ProposalDatasetProvenanceError("proposal crop provenance summary binding is invalid")
try:
source_release = assert_yolo_summary_bound_to_embedded_training_release(
summary_path=summary_path,
corpus_manifest=corpus_manifest_path,
fixture_mode=fixture_mode,
)
except TrainingReleaseError as exc:
raise ProposalDatasetProvenanceError(
"proposal source summary has no eligible immutable training release"
) from exc
recorded_release = _require_mapping(source.get("training_release"), field="source.training_release")
if (
recorded_release.get("dataset_yaml_path") != source_release["dataset_yaml"]["path"]
or recorded_release.get("dataset_yaml_sha256") != source_release["dataset_yaml"]["sha256"]
or recorded_release.get("corpus_manifest_sha256") != source_release["corpus"]["manifest_sha256"]
):
raise ProposalDatasetProvenanceError("proposal crop provenance release binding is invalid")
proposal_model = _require_mapping(source.get("proposal_model"), field="source.proposal_model")
_require_nonempty_path(proposal_model.get("path"), field="source.proposal_model.path")
_require_sha256(proposal_model.get("sha256"), field="source.proposal_model.sha256")
entries = payload.get("crops")
if not isinstance(entries, list) or not entries:
raise ProposalDatasetProvenanceError("proposal dataset provenance has no crop entries")
expected_paths: set[str] = set()
calculated_counts: Counter[str] = Counter()
normalized_entries: list[dict[str, Any]] = []
for entry in entries:
item = _require_mapping(entry, field="crops[]")
relative_path, crop_path = _safe_relative_crop_path(item.get("relative_path"), dataset_root=root)
if relative_path in expected_paths:
raise ProposalDatasetProvenanceError(f"proposal crop provenance has duplicate path: {relative_path}")
expected_paths.add(relative_path)
split = item.get("split")
label = item.get("label")
if split not in {"train", "val"} or label not in {"negative", "positive"}:
raise ProposalDatasetProvenanceError(f"proposal crop has invalid split/class: {relative_path}")
if not isinstance(item.get("sample_slug"), str) or not item["sample_slug"].strip():
raise ProposalDatasetProvenanceError(f"proposal crop has no sample identity: {relative_path}")
if not crop_path.is_file():
raise ProposalDatasetProvenanceError(f"proposal crop is missing: {relative_path}")
if _require_sha256(item.get("sha256"), field=f"crops[{relative_path}].sha256") != sha256(crop_path):
raise ProposalDatasetProvenanceError(f"proposal crop checksum mismatches provenance: {relative_path}")
if item.get("size_bytes") != crop_path.stat().st_size:
raise ProposalDatasetProvenanceError(f"proposal crop byte size mismatches provenance: {relative_path}")
_require_nonempty_path(item.get("source_image_path"), field=f"crops[{relative_path}].source_image_path")
_require_sha256(item.get("source_image_sha256"), field=f"crops[{relative_path}].source_image_sha256")
_require_nonempty_path(item.get("source_label_path"), field=f"crops[{relative_path}].source_label_path")
_require_sha256(item.get("source_label_sha256"), field=f"crops[{relative_path}].source_label_sha256")
calculated_counts[f"{split}/{label}"] += 1
normalized_entries.append(dict(item))
canonical_entries = sorted(normalized_entries, key=lambda item: str(item["relative_path"]))
expected_crops_sha256 = hashlib.sha256(_canonical_json_bytes({"crops": canonical_entries})).hexdigest()
if payload.get("crops_sha256") != expected_crops_sha256:
raise ProposalDatasetProvenanceError("proposal crop collection checksum mismatches provenance")
if payload.get("crop_count") != len(entries):
raise ProposalDatasetProvenanceError("proposal crop count mismatches provenance")
if payload.get("counts") != dict(sorted(calculated_counts.items())):
raise ProposalDatasetProvenanceError("proposal crop class counts mismatch provenance")
if any(calculated_counts[f"{split}/{label}"] < 1 for split in ("train", "val") for label in ("negative", "positive")):
raise ProposalDatasetProvenanceError("proposal dataset has an empty train/validation class")
actual_paths = {
file_path.relative_to(root).as_posix()
for file_path in root.rglob("*")
if file_path.is_file() and file_path.suffix.lower() in _CROP_IMAGE_SUFFIXES
}
if actual_paths != expected_paths:
unexpected = sorted(actual_paths - expected_paths)
missing = sorted(expected_paths - actual_paths)
raise ProposalDatasetProvenanceError(
"proposal dataset files are not exactly the immutable crop manifest "
f"(unexpected={unexpected}, missing={missing})"
)
return payload
def binary_metrics(scores: list[float], labels: list[int], threshold: float = 0.5) -> dict[str, float | int]:
@@ -21,16 +261,44 @@ def binary_metrics(scores: list[float], labels: list[int], threshold: float = 0.
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--dataset-dir", type=Path, required=True)
parser.add_argument(
"--corpus-manifest",
type=Path,
required=True,
help="The exact frozen governed corpus manifest that produced the proposal crops.",
)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--epochs", type=int, default=12)
parser.add_argument("--batch", type=int, default=64)
parser.add_argument("--lr", type=float, default=1e-4)
parser.add_argument("--device", default="cuda:0")
parser.add_argument("--export-existing-best", action="store_true")
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Permit only an explicitly fixture-only corpus; operational runs always resolve live dataset evidence.",
)
args = parser.parse_args()
if args.output_dir.exists() and not args.export_existing_best:
parser.error(f"output already exists: {args.output_dir}")
# All governed-source checks run before importing torch/torchvision or
# touching CUDA. A source revocation therefore cannot start PyTorch work.
try:
governed_corpus = load_governed_corpus_manifest(
args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
proposal_provenance = validate_proposal_dataset_provenance(
args.dataset_dir,
args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except ProposalDatasetProvenanceError as exc:
raise SystemExit(str(exc)) from exc
proposal_provenance_path = args.dataset_dir / PROPOSAL_DATASET_PROVENANCE_NAME
proposal_provenance_sha256 = sha256(proposal_provenance_path)
import torch
from torch import nn
from torch.utils.data import DataLoader
@@ -54,7 +322,18 @@ def main() -> int:
model.load_state_dict(torch.load(state_path, map_location=device))
model.eval()
torch.jit.script(model).save(str(args.output_dir / "proposal-classifier.torchscript.pt"))
print(json.dumps({"status": "exported_existing_best", "model": str(state_path)}))
print(
json.dumps(
{
"status": "exported_existing_best",
"model": str(state_path),
"proposal_dataset_provenance": str(proposal_provenance_path),
"proposal_dataset_provenance_sha256": proposal_provenance_sha256,
"governed_corpus_manifest_sha256": sha256(args.corpus_manifest),
"governed_corpus_live_recheck": True,
}
)
)
return 0
train_loader = DataLoader(train_ds, batch_size=args.batch, shuffle=True, num_workers=0)
val_loader = DataLoader(val_ds, batch_size=args.batch, shuffle=False, num_workers=0)
@@ -93,11 +372,27 @@ def main() -> int:
model.load_state_dict(torch.load(args.output_dir / "best-state.pt", map_location=device))
model.eval()
scripted = torch.jit.script(model)
scripted.save(str(args.output_dir / "proposal-classifier.torchscript.pt"))
report = {"schema_version": 1, "status": "ok", "classes": train_ds.class_to_idx,
"train_count": len(train_ds), "validation_count": len(val_ds),
"best_validation_f1": best_f1, "history": history,
"model": str(args.output_dir / "proposal-classifier.torchscript.pt")}
model_path = args.output_dir / "proposal-classifier.torchscript.pt"
scripted.save(str(model_path))
report = {
"schema_version": 1,
"status": "ok",
"classes": train_ds.class_to_idx,
"train_count": len(train_ds),
"validation_count": len(val_ds),
"best_validation_f1": best_f1,
"history": history,
"model": str(model_path),
"model_sha256": sha256(model_path),
"proposal_dataset_provenance": str(proposal_provenance_path),
"proposal_dataset_provenance_sha256": proposal_provenance_sha256,
"proposal_dataset_manifest_sha256": proposal_provenance["manifest_sha256"],
"corpus_manifest": str(args.corpus_manifest),
"corpus_manifest_sha256": sha256(args.corpus_manifest),
"fixture_mode": bool(args.fixture_mode),
"governed_corpus_live_recheck": True,
"governed_corpus_sample_count": len(governed_corpus.get("samples", [])),
}
(args.output_dir / "training-report.json").write_text(json.dumps(report, indent=2), encoding="utf-8")
return 0
+54 -2
View File
@@ -8,9 +8,23 @@ import hashlib
import json
import random
import shutil
import sys
from pathlib import Path
from typing import Any, TypeVar
SCRIPT_DIR = Path(__file__).resolve().parent
if str(SCRIPT_DIR) not in sys.path:
sys.path.insert(0, str(SCRIPT_DIR))
from training_dataset_eligibility import ( # noqa: E402
TrainingEligibilityError,
assert_frozen_manifest_training_eligible,
)
from training_release_manifest import ( # noqa: E402
TrainingReleaseError,
assert_yolo_summary_bound_to_training_release,
)
T = TypeVar("T")
@@ -111,6 +125,18 @@ def choose_threshold(probabilities: list[float], labels: list[int]) -> tuple[flo
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--summary", type=Path, required=True)
parser.add_argument(
"--corpus-manifest",
type=Path,
required=True,
help="Immutable governed corpus manifest that produced the tile summary.",
)
parser.add_argument(
"--train-yaml",
type=Path,
required=True,
help="YOLO dataset YAML with an eligible immutable GeoIntel training release.",
)
parser.add_argument("--proposal-model", type=Path, required=True)
parser.add_argument("--output-dir", type=Path, required=True)
parser.add_argument("--device", default="cuda:0")
@@ -124,7 +150,31 @@ def main() -> int:
parser.add_argument("--proposal-chunk-size", type=int, default=16)
parser.add_argument("--seed", type=int, default=20260727)
parser.add_argument("--force", action="store_true")
parser.add_argument(
"--fixture-mode",
action="store_true",
help="Accept only an explicitly fixture-only corpus manifest; never use for operational training.",
)
args = parser.parse_args()
try:
assert_frozen_manifest_training_eligible(
args.corpus_manifest,
fixture_mode=args.fixture_mode,
verify_live=True,
)
assert_yolo_summary_bound_to_training_release(
summary_path=args.summary,
train_yaml=args.train_yaml,
corpus_manifest=args.corpus_manifest,
fixture_mode=args.fixture_mode,
)
except (TrainingEligibilityError, TrainingReleaseError) as exc:
raise SystemExit(str(exc)) from exc
summary = json.loads(args.summary.read_text(encoding="utf-8"))
if summary.get("source_manifest_sha256") != sha256(args.corpus_manifest):
raise SystemExit(
"Proposal-filter tile summary is not bound to the supplied governed corpus manifest"
)
if args.output_dir.exists():
if not args.force:
raise SystemExit(f"Output exists: {args.output_dir}")
@@ -133,12 +183,11 @@ def main() -> int:
import torch
from torch import nn
from torch.utils.data import DataLoader
from torchvision import datasets, transforms
from torchvision import datasets
from torchvision.models import ResNet18_Weights, resnet18
from ultralytics import YOLO
torch.manual_seed(args.seed)
summary = json.loads(args.summary.read_text(encoding="utf-8"))
split_tiles = {
split: [tile for tile in summary["tiles"] if tile.get("kept", True) and tile["split"] == split]
for split in ("train", "val")
@@ -214,6 +263,9 @@ def main() -> int:
evidence = {
"schema_version": 1, "status": "ok", "architecture": "resnet18_binary_proposal_filter",
"summary": str(args.summary), "summary_sha256": sha256(args.summary),
"corpus_manifest": str(args.corpus_manifest),
"corpus_manifest_sha256": sha256(args.corpus_manifest),
"fixture_mode": bool(args.fixture_mode),
"proposal_model": str(args.proposal_model), "proposal_model_sha256": sha256(args.proposal_model),
"device": args.device, "proposal_confidence": args.proposal_confidence, "crop_scale": args.crop_scale,
"proposal_chunk_size": args.proposal_chunk_size,
+7
View File
@@ -53,6 +53,8 @@ if [[ -z "${PYTHON_BIN:-}" ]]; then
fi
fi
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
DATASET_YAML="${OPERATOR_YOLO_DATASET_DIR%/}/dataset.yaml"
SUMMARY_PATH="${TRAIN_OUTPUT_DIR%/}/${TRAIN_RUN_NAME}/training_summary.json"
export DATASET_YAML
@@ -79,6 +81,11 @@ if [[ ! -f "${YOLO_BASE_MODEL_PATH}" ]]; then
exit 1
fi
# A filename is not a training identity. Verify the immutable YAML/corpus/
# asset/review release before importing Ultralytics or allocating CUDA work.
"${PYTHON_BIN}" "${SCRIPT_DIR}/training_release_manifest.py" verify \
--train-yaml "${DATASET_YAML}"
mkdir -p "${TRAIN_OUTPUT_DIR}" "$(dirname "${TRAIN_MODEL_OUTPUT_PATH}")"
"${PYTHON_BIN}" - <<'PY'
+545
View File
@@ -0,0 +1,545 @@
"""Fail-closed provenance gate for Belgian building-training inputs.
The database data-contract validator decides whether a dataset can be stored.
This module is intentionally a second, independent gate at the point where
persisted datasets become irreversible training pairs. It has no database
side effects, making the decision reproducible in a corpus manifest and easy
to re-check before CUDA training starts.
Legacy data may only pass through this module in explicit fixture mode. That
mode is deliberately limited to records marked as fixtures and must never be
used for an operational corpus.
"""
from __future__ import annotations
import hashlib
import json
import re
from collections.abc import Mapping
from pathlib import Path
from typing import Any, Callable, Literal
from uuid import UUID
TRAINING_ELIGIBILITY_POLICY_VERSION = "geointel-training-source-eligibility/v1"
DatasetRole = Literal["raster", "reference"]
_SHA256_RE = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE)
_ALLOWED_SOURCE_CLASSIFICATIONS = {
"authoritative",
"corroborative",
"contextual",
"derived",
}
_FIXTURE_SOURCES = {"fixture", "test", "test_fixture", "test-fixture"}
class TrainingEligibilityError(ValueError):
"""Raised when persisted inputs cannot be used in an operational corpus."""
class TrainingEligibilityResult:
"""Stable, JSON-serializable decision for one persisted dataset."""
__slots__ = ("role", "eligible", "fixture_mode", "reasons", "evidence")
def __init__(
self,
*,
role: DatasetRole,
eligible: bool,
fixture_mode: bool,
reasons: tuple[str, ...],
evidence: dict[str, Any],
) -> None:
self.role = role
self.eligible = eligible
self.fixture_mode = fixture_mode
self.reasons = reasons
self.evidence = evidence
def as_dict(self) -> dict[str, Any]:
return {
"policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION,
"role": self.role,
"eligible": self.eligible,
"fixture_mode": self.fixture_mode,
"reasons": list(self.reasons),
"evidence": self.evidence,
}
def _value(record: Any, field: str, default: Any = None) -> Any:
if isinstance(record, Mapping):
return record.get(field, default)
return getattr(record, field, default)
def _normalise_status(value: Any) -> str:
return str(value or "").strip().lower()
def _normalise_mapping(value: Any) -> dict[str, Any]:
return dict(value) if isinstance(value, Mapping) else {}
def _source_identity(dataset: Any) -> tuple[str, str]:
return (
_normalise_status(_value(dataset, "source")),
_normalise_status(_value(dataset, "source_name")),
)
def _is_explicit_fixture(dataset: Any, registry: Any) -> bool:
source, source_name = _source_identity(dataset)
metadata = _normalise_mapping(_value(dataset, "metadata_json"))
provenance = _normalise_mapping(_value(dataset, "provenance_metadata"))
usage_policy = _normalise_mapping(_value(registry, "usage_policy_json"))
return bool(
source in _FIXTURE_SOURCES
or source_name in _FIXTURE_SOURCES
or metadata.get("fixture") is True
or provenance.get("fixture") is True
or usage_policy.get("fixture_only") is True
)
def _dataset_evidence(dataset: Any, registry: Any, snapshot: Any) -> dict[str, Any]:
usage_policy = _normalise_mapping(_value(registry, "usage_policy_json"))
allowed_tasks = usage_policy.get("allowed_tasks")
validation_authority = _normalise_mapping(usage_policy.get("validation_authority"))
return {
"dataset_id": str(_value(dataset, "id") or ""),
"dataset_type": _normalise_status(_value(dataset, "dataset_type")),
"dataset_role": _normalise_status(_value(dataset, "dataset_role")),
"source": _normalise_status(_value(dataset, "source")),
"source_name": _normalise_status(_value(dataset, "source_name")),
"checksum_sha256": _value(dataset, "checksum_sha256"),
"data_contract_key": _value(dataset, "data_contract_key"),
"data_contract_version": _value(dataset, "data_contract_version"),
"validation_status": _normalise_status(_value(dataset, "validation_status")),
"provenance_status": _normalise_status(_value(dataset, "provenance_status")),
"lineage_status": _normalise_status(_value(dataset, "lineage_status")),
"quarantine_status": _normalise_status(_value(dataset, "quarantine_status")),
"dataset_status": _normalise_status(_value(dataset, "status")),
"dataset_source_registry_id": str(_value(dataset, "source_registry_id") or ""),
"dataset_source_snapshot_id": str(_value(dataset, "source_snapshot_id") or ""),
"source_registry_id": str(_value(registry, "id") or ""),
"source_key": _normalise_status(_value(registry, "source_key")),
"source_classification": _normalise_status(_value(registry, "classification")),
"source_freshness_status": _normalise_status(_value(registry, "freshness_status")),
"source_ingest_status": _normalise_status(_value(registry, "ingest_status")),
"source_training_allowed": usage_policy.get("training_allowed"),
"source_ground_truth_allowed": usage_policy.get("ground_truth_allowed"),
"source_allowed_tasks": list(allowed_tasks) if isinstance(allowed_tasks, list) else [],
"source_validation_authority": validation_authority,
"source_building_validation_authority": validation_authority.get("building_validation"),
"source_snapshot_id": str(_value(snapshot, "id") or _value(dataset, "source_snapshot_id") or ""),
"snapshot_source_registry_id": str(_value(snapshot, "source_registry_id") or ""),
"snapshot_key": _value(snapshot, "snapshot_key"),
"snapshot_checksum_sha256": _value(snapshot, "checksum_sha256"),
"snapshot_freshness_status": _normalise_status(_value(snapshot, "freshness_status")),
"snapshot_ingest_status": _normalise_status(_value(snapshot, "ingest_status")),
}
def evaluate_dataset_training_eligibility(
dataset: Any,
*,
role: DatasetRole,
fixture_mode: bool = False,
) -> TrainingEligibilityResult:
"""Evaluate a source Dataset without mutating it.
Operational calls require server-attested source registry and snapshot
relationships. ``fixture_mode`` can relax only legacy provenance fields,
and only for an explicitly marked fixture. A failed validation or a
quarantine is never relaxed.
"""
registry = _value(dataset, "source_registry")
snapshot = _value(dataset, "source_snapshot")
evidence = _dataset_evidence(dataset, registry, snapshot)
reasons: list[str] = []
dataset_type = evidence["dataset_type"]
dataset_role = evidence["dataset_role"]
if role == "raster" and dataset_type != "raster":
reasons.append("dataset_type_not_raster")
if role == "reference":
if dataset_type != "vector":
reasons.append("dataset_type_not_vector")
if dataset_role != "reference":
reasons.append("dataset_role_not_reference")
if evidence["dataset_status"] != "ready":
reasons.append("dataset_not_ready")
if evidence["quarantine_status"] == "quarantined":
reasons.append("dataset_quarantined")
explicit_fixture = _is_explicit_fixture(dataset, registry)
if fixture_mode and not explicit_fixture:
reasons.append("fixture_mode_requires_explicit_fixture")
validation_status = evidence["validation_status"]
if validation_status == "failed":
reasons.append("validation_failed")
elif validation_status != "passed" and not (fixture_mode and explicit_fixture):
reasons.append("validation_not_passed")
if not fixture_mode:
if evidence["provenance_status"] != "complete":
reasons.append("provenance_not_complete")
if evidence["lineage_status"] not in {"complete", "not_applicable"}:
reasons.append("lineage_not_complete")
if not evidence["data_contract_key"] or not evidence["data_contract_version"]:
reasons.append("data_contract_not_versioned")
checksum = str(evidence["checksum_sha256"] or "")
if not _SHA256_RE.fullmatch(checksum):
reasons.append("dataset_checksum_invalid")
if registry is None:
reasons.append("source_registry_missing")
if snapshot is None:
reasons.append("source_snapshot_missing")
if registry is not None:
if (
evidence["dataset_source_registry_id"]
and evidence["dataset_source_registry_id"] != evidence["source_registry_id"]
):
reasons.append("dataset_source_registry_binding_mismatch")
classification = evidence["source_classification"]
if classification not in _ALLOWED_SOURCE_CLASSIFICATIONS:
reasons.append("source_classification_not_allowed")
if evidence["source_training_allowed"] is not True:
reasons.append("source_not_allowed_for_training")
if evidence["source_ingest_status"] == "quarantined":
reasons.append("source_registry_quarantined")
if snapshot is not None:
if (
evidence["dataset_source_snapshot_id"]
and evidence["dataset_source_snapshot_id"] != evidence["source_snapshot_id"]
):
reasons.append("dataset_source_snapshot_binding_mismatch")
if (
registry is not None
and evidence["snapshot_source_registry_id"]
and evidence["snapshot_source_registry_id"] != evidence["source_registry_id"]
):
reasons.append("source_snapshot_registry_mismatch")
if evidence["snapshot_ingest_status"] != "ingested":
reasons.append("source_snapshot_not_ingested")
if evidence["snapshot_freshness_status"] not in {"current", "not_applicable"}:
reasons.append("source_snapshot_freshness_not_approved")
snapshot_checksum = str(evidence["snapshot_checksum_sha256"] or "")
if not _SHA256_RE.fullmatch(snapshot_checksum):
reasons.append("source_snapshot_checksum_invalid")
elif snapshot_checksum.lower() != checksum.lower():
reasons.append("source_snapshot_checksum_mismatch")
if role == "reference":
if evidence["source_classification"] != "authoritative":
reasons.append("reference_source_not_authoritative")
if evidence["source_ground_truth_allowed"] is not True:
reasons.append("reference_source_not_ground_truth_allowed")
if "building_validation" not in evidence["source_allowed_tasks"]:
reasons.append("reference_source_not_approved_for_building_validation")
if evidence["source_building_validation_authority"] != "primary":
reasons.append("reference_building_validation_not_primary")
return TrainingEligibilityResult(
role=role,
eligible=not reasons,
fixture_mode=fixture_mode,
reasons=tuple(sorted(set(reasons))),
evidence=evidence,
)
def assert_dataset_training_eligible(
dataset: Any,
*,
role: DatasetRole,
fixture_mode: bool = False,
sample_slug: str | None = None,
) -> TrainingEligibilityResult:
result = evaluate_dataset_training_eligibility(dataset, role=role, fixture_mode=fixture_mode)
if result.eligible:
return result
label = f" for sample {sample_slug!r}" if sample_slug else ""
raise TrainingEligibilityError(
f"{role} dataset is not eligible for training{label}: {', '.join(result.reasons)}"
)
def training_pair_evidence(
*,
raster: Any,
reference: Any,
fixture_mode: bool = False,
) -> dict[str, Any]:
"""Return manifest-ready evidence for one raster/reference pair."""
raster_result = evaluate_dataset_training_eligibility(
raster,
role="raster",
fixture_mode=fixture_mode,
)
reference_result = evaluate_dataset_training_eligibility(
reference,
role="reference",
fixture_mode=fixture_mode,
)
return {
"policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION,
"eligible": raster_result.eligible and reference_result.eligible,
"fixture_mode": fixture_mode,
"raster": raster_result.as_dict(),
"reference": reference_result.as_dict(),
}
def manifest_training_eligibility_failures(
manifest: Mapping[str, Any],
*,
fixture_mode: bool = False,
) -> list[str]:
"""Re-check immutable eligibility evidence before an actual training run."""
failures: list[str] = []
eligibility = manifest.get("training_eligibility")
if not isinstance(eligibility, Mapping):
return ["manifest_training_eligibility_missing"]
if eligibility.get("policy_version") != TRAINING_ELIGIBILITY_POLICY_VERSION:
failures.append("manifest_training_eligibility_policy_invalid")
if eligibility.get("status") != "eligible":
failures.append("manifest_training_eligibility_not_eligible")
manifest_fixture_mode = eligibility.get("fixture_mode") is True
if manifest_fixture_mode != fixture_mode:
failures.append("manifest_fixture_mode_mismatch")
samples = manifest.get("samples")
if not isinstance(samples, list) or not samples:
failures.append("manifest_samples_missing")
return failures
for sample in samples:
if not isinstance(sample, Mapping):
failures.append("manifest_sample_invalid")
continue
slug = str(sample.get("sample_slug") or "<unknown>")
pair = sample.get("training_eligibility")
if not isinstance(pair, Mapping):
failures.append(f"{slug}:training_eligibility_missing")
continue
if pair.get("policy_version") != TRAINING_ELIGIBILITY_POLICY_VERSION:
failures.append(f"{slug}:training_eligibility_policy_invalid")
if pair.get("fixture_mode") is not fixture_mode:
failures.append(f"{slug}:fixture_mode_mismatch")
if pair.get("eligible") is not True:
failures.append(f"{slug}:training_pair_not_eligible")
for role in ("raster", "reference"):
decision = pair.get(role)
if not isinstance(decision, Mapping):
failures.append(f"{slug}:{role}_eligibility_missing")
continue
if decision.get("eligible") is not True:
failures.append(f"{slug}:{role}_not_eligible")
reasons = decision.get("reasons")
if isinstance(reasons, list) and reasons:
failures.append(f"{slug}:{role}_has_rejection_reasons")
return sorted(set(failures))
def _default_live_dataset_access() -> tuple[Callable[[], Any], type[Any]]:
"""Load the database dependencies only for an operational live re-check.
This module is also imported by pure filesystem tooling and fixture tests.
Keeping the import lazy means those paths do not accidentally open a
database connection, while a normal release/verify invocation still fails
closed if the live registry cannot be checked.
"""
import sys
repo_root = Path(__file__).resolve().parents[1]
app_root = repo_root / "backend"
if str(app_root) not in sys.path:
sys.path.insert(0, str(app_root))
from app.db.session import SessionLocal
from app.models import Dataset
return SessionLocal, Dataset
def live_manifest_training_eligibility_failures(
manifest: Mapping[str, Any],
*,
fixture_mode: bool = False,
session_factory: Callable[[], Any] | None = None,
dataset_model: type[Any] | None = None,
) -> list[str]:
"""Re-evaluate every frozen corpus parent against the live database.
A frozen manifest proves what was eligible when it was created, not what
remains eligible now. Operational releases therefore resolve the exact
raster/reference Dataset ids again immediately before seal, retry, resume
and CUDA training. A later quarantine, failed contract, snapshot change
or missing record is a revocation and cannot be masked by the old manifest.
Fixture-only corpora deliberately have no operational authority and are
never used for a production release; their isolated tests may skip this
live database boundary.
"""
if fixture_mode:
return []
failures: list[str] = []
samples = manifest.get("samples")
if not isinstance(samples, list) or not samples:
return ["live_manifest_samples_missing"]
if session_factory is None or dataset_model is None:
try:
default_factory, default_model = _default_live_dataset_access()
except Exception:
return ["live_training_dataset_access_unavailable"]
session_factory = session_factory or default_factory
dataset_model = dataset_model or default_model
try:
db = session_factory()
except Exception:
return ["live_training_dataset_access_unavailable"]
try:
for sample in samples:
if not isinstance(sample, Mapping):
failures.append("live_manifest_sample_invalid")
continue
slug = str(sample.get("sample_slug") or "<unknown>")
recorded_pair = sample.get("training_eligibility")
for role, field_name in (("raster", "raster_dataset_id"), ("reference", "reference_dataset_id")):
raw_id = sample.get(field_name)
if not isinstance(raw_id, str) or not raw_id.strip():
failures.append(f"{slug}:{role}_dataset_id_missing_for_live_check")
continue
try:
dataset_id = UUID(raw_id)
except (TypeError, ValueError, AttributeError):
failures.append(f"{slug}:{role}_dataset_id_invalid_for_live_check")
continue
try:
dataset = db.get(dataset_model, dataset_id)
except Exception:
failures.append(f"{slug}:{role}_live_lookup_failed")
continue
if dataset is None:
failures.append(f"{slug}:{role}_live_dataset_missing")
continue
result = evaluate_dataset_training_eligibility(
dataset,
role=role, # type: ignore[arg-type]
fixture_mode=False,
)
if not result.eligible:
for reason in result.reasons:
failures.append(f"{slug}:{role}_live_revoked:{reason}")
if isinstance(recorded_pair, Mapping):
recorded_role = recorded_pair.get(role)
if isinstance(recorded_role, Mapping):
recorded_evidence = recorded_role.get("evidence")
if isinstance(recorded_evidence, Mapping) and str(recorded_evidence.get("dataset_id") or "") != raw_id:
failures.append(f"{slug}:{role}_manifest_dataset_binding_mismatch")
finally:
close = getattr(db, "close", None)
if callable(close):
close()
return sorted(set(failures))
def assert_manifest_training_eligible(
manifest: Mapping[str, Any],
*,
fixture_mode: bool = False,
) -> None:
failures = manifest_training_eligibility_failures(manifest, fixture_mode=fixture_mode)
if failures:
raise TrainingEligibilityError(
"Corpus manifest is not eligible for training: " + ", ".join(failures)
)
def _file_sha256(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as stream:
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def frozen_manifest_training_eligibility_failures(
manifest_path: Path,
*,
fixture_mode: bool = False,
verify_live: bool = False,
session_factory: Callable[[], Any] | None = None,
dataset_model: type[Any] | None = None,
) -> list[str]:
"""Verify an immutable manifest and its source-eligibility decision together."""
failures: list[str] = []
try:
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return ["corpus_manifest_unreadable"]
if not isinstance(manifest, Mapping):
return ["corpus_manifest_invalid"]
failures.extend(manifest_training_eligibility_failures(manifest, fixture_mode=fixture_mode))
freeze_path = manifest_path.parent / "corpus-freeze.json"
try:
freeze = json.loads(freeze_path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return sorted(set(failures + ["corpus_freeze_missing_or_invalid"]))
if not isinstance(freeze, Mapping):
return sorted(set(failures + ["corpus_freeze_missing_or_invalid"]))
if freeze.get("immutable") is not True:
failures.append("corpus_freeze_not_immutable")
if freeze.get("manifest_sha256") != _file_sha256(manifest_path):
failures.append("corpus_manifest_checksum_mismatch")
if freeze.get("training_eligibility_policy") != TRAINING_ELIGIBILITY_POLICY_VERSION:
failures.append("corpus_freeze_policy_invalid")
if bool(freeze.get("fixture_mode")) != fixture_mode:
failures.append("corpus_freeze_fixture_mode_mismatch")
if verify_live:
failures.extend(
live_manifest_training_eligibility_failures(
manifest,
fixture_mode=fixture_mode,
session_factory=session_factory,
dataset_model=dataset_model,
)
)
return sorted(set(failures))
def assert_frozen_manifest_training_eligible(
manifest_path: Path,
*,
fixture_mode: bool = False,
verify_live: bool = False,
session_factory: Callable[[], Any] | None = None,
dataset_model: type[Any] | None = None,
) -> dict[str, Any]:
"""Load only a frozen, policy-valid corpus manifest for a training entrypoint."""
failures = frozen_manifest_training_eligibility_failures(
manifest_path,
fixture_mode=fixture_mode,
verify_live=verify_live,
session_factory=session_factory,
dataset_model=dataset_model,
)
if failures:
raise TrainingEligibilityError(
"Corpus manifest is not eligible for training: " + ", ".join(failures)
)
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
assert isinstance(payload, dict)
return payload
File diff suppressed because it is too large Load Diff
+738
View File
@@ -0,0 +1,738 @@
#!/usr/bin/env python3
"""Exercise Phase-2 PostgreSQL migration guards on a disposable database.
This is intentionally *not* a general migration runner. It refuses every
database whose name does not start with ``geointel_phase2_`` so it cannot be
pointed accidentally at a developer, staging or production database. The
script upgrades the complete Alembic chain, proves the new trigger guards with
real DML, and optionally downgrades again.
"""
from __future__ import annotations
import argparse
from contextlib import contextmanager
from datetime import UTC, datetime
import json
import os
from pathlib import Path
import subprocess
import sys
from typing import Any, Iterator
from uuid import UUID, uuid4
from sqlalchemy import create_engine, text
from sqlalchemy.engine import Connection, Engine, make_url
from sqlalchemy.exc import IntegrityError
ROOT = Path(__file__).resolve().parents[1]
BACKEND = ROOT / "backend"
SAFE_DATABASE_PREFIX = "geointel_phase2_"
def _json_dump(path: Path, value: dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(
json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8"
)
def _require_disposable_database(database_url: str) -> None:
parsed = make_url(database_url)
database_name = str(parsed.database or "")
if not database_name.startswith(SAFE_DATABASE_PREFIX):
raise SystemExit(
"Refusing migration guard test: database name must start with "
f"{SAFE_DATABASE_PREFIX!r}, received {database_name!r}."
)
def _run_alembic(database_url: str, *arguments: str) -> str:
environment = dict(os.environ)
environment["DATABASE_URL"] = database_url
completed = subprocess.run(
[sys.executable, "-m", "alembic", *arguments],
cwd=BACKEND,
env=environment,
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
check=False,
)
if completed.returncode != 0:
raise RuntimeError(
f"Alembic {' '.join(arguments)} failed with exit code {completed.returncode}:\n{completed.stdout}"
)
return completed.stdout
@contextmanager
def _transaction(engine: Engine) -> Iterator[Connection]:
with engine.begin() as connection:
yield connection
def _insert_project(connection: Connection) -> UUID:
project_id = uuid4()
connection.execute(
text("INSERT INTO projects (id, name) VALUES (:id, :name)"),
{"id": project_id, "name": "Phase 2 migration guard fixture"},
)
return project_id
def _source_id(connection: Connection, source_key: str) -> UUID:
value = connection.execute(
text("SELECT id FROM source_registry WHERE source_key = :source_key"),
{"source_key": source_key},
).scalar_one()
return UUID(str(value))
def _insert_snapshot(
connection: Connection, source_registry_id: UUID, *, key: str, checksum: str
) -> UUID:
snapshot_id = uuid4()
connection.execute(
text(
"""
INSERT INTO source_snapshots (
id, source_registry_id, snapshot_key, checksum_sha256,
crs, units, freshness_status, ingest_status
) VALUES (
:id, :source_registry_id, :snapshot_key, :checksum_sha256,
'EPSG:4326', 'metres', 'current', 'ingested'
)
"""
),
{
"id": snapshot_id,
"source_registry_id": source_registry_id,
"snapshot_key": key,
"checksum_sha256": checksum,
},
)
return snapshot_id
def _accepted_report(
*, contract_key: str = "geointel.vector.geojson", contract_version: str = "1.0.0"
) -> str:
"""Produce a structurally complete persisted validation report fixture."""
return json.dumps(
{
"asset_id": "phase2-guard-fixture",
"data_contract_key": contract_key,
"data_contract_version": contract_version,
"contract_fingerprint_sha256": "d" * 64,
"validation_status": "passed",
"provenance_status": "complete",
"lineage_status": "complete",
"quarantine_status": "not_quarantined",
"validation_scope": ["contract"],
"report_sha256": "e" * 64,
}
)
def _insert_dataset(
connection: Connection,
*,
project_id: UUID,
source_registry_id: UUID,
source_snapshot_id: UUID,
suffix: str,
checksum_sha256: str,
) -> UUID:
dataset_id = uuid4()
connection.execute(
text(
"""
INSERT INTO datasets (
id, project_id, name, dataset_type, source, source_name,
status, source_registry_id, source_snapshot_id,
data_contract_key, data_contract_version, validation_report_json,
validation_status, provenance_status, lineage_status, quarantine_status,
checksum_sha256
) VALUES (
:id, :project_id, :name, 'vector', 'governed', 'grb',
'ready', :source_registry_id, :source_snapshot_id,
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
'passed', 'complete', 'complete', 'not_quarantined', :checksum_sha256
)
"""
),
{
"id": dataset_id,
"project_id": project_id,
"name": f"phase2-{suffix}.geojson",
"source_registry_id": source_registry_id,
"source_snapshot_id": source_snapshot_id,
"validation_report_json": _accepted_report(),
"checksum_sha256": checksum_sha256,
},
)
return dataset_id
def _assert_constraint_rejection(
engine: Engine, statement: str, parameters: dict[str, Any], *, label: str
) -> None:
try:
with _transaction(engine) as connection:
connection.execute(text(statement), parameters)
except IntegrityError as exc:
sqlstate = getattr(getattr(exc, "orig", None), "sqlstate", None)
if sqlstate == "23514":
return
raise AssertionError(
f"{label} failed with unexpected SQLSTATE {sqlstate!r}"
) from exc
raise AssertionError(f"{label} was accepted unexpectedly")
def verify(database_url: str, *, verify_downgrade: bool) -> dict[str, Any]:
_require_disposable_database(database_url)
started_at = datetime.now(UTC).isoformat()
upgrade_output = _run_alembic(database_url, "upgrade", "head")
engine = create_engine(database_url)
try:
edge_a = uuid4()
edge_b = uuid4()
with _transaction(engine) as connection:
project_id = _insert_project(connection)
grb_id = _source_id(connection, "grb")
osm_id = _source_id(connection, "osm")
grb_snapshot_id = _insert_snapshot(
connection, grb_id, key="phase2-grb", checksum="a" * 64
)
osm_snapshot_id = _insert_snapshot(
connection, osm_id, key="phase2-osm", checksum="b" * 64
)
derived_b_snapshot_id = _insert_snapshot(
connection, grb_id, key="phase2-derived-b", checksum="c" * 64
)
derived_c_snapshot_id = _insert_snapshot(
connection, grb_id, key="phase2-derived-c", checksum="d" * 64
)
mutation_snapshot_id = _insert_snapshot(
connection, grb_id, key="phase2-mutation", checksum="e" * 64
)
dataset_a = _insert_dataset(
connection,
project_id=project_id,
source_registry_id=grb_id,
source_snapshot_id=grb_snapshot_id,
suffix="a",
checksum_sha256="a" * 64,
)
dataset_b = _insert_dataset(
connection,
project_id=project_id,
source_registry_id=grb_id,
source_snapshot_id=derived_b_snapshot_id,
suffix="b",
checksum_sha256="c" * 64,
)
dataset_c = _insert_dataset(
connection,
project_id=project_id,
source_registry_id=grb_id,
source_snapshot_id=derived_c_snapshot_id,
suffix="c",
checksum_sha256="d" * 64,
)
shared_dataset = _insert_dataset(
connection,
project_id=project_id,
source_registry_id=grb_id,
source_snapshot_id=grb_snapshot_id,
suffix="shared",
checksum_sha256="a" * 64,
)
mutation_dataset = _insert_dataset(
connection,
project_id=project_id,
source_registry_id=grb_id,
source_snapshot_id=mutation_snapshot_id,
suffix="mutation",
checksum_sha256="e" * 64,
)
version_id = uuid4()
connection.execute(
text(
"""
INSERT INTO dataset_versions (
id, dataset_id, version, source_registry_id, source_snapshot_id,
data_contract_key, data_contract_version, validation_report_json,
validation_status, provenance_status, lineage_status, checksum_sha256
) VALUES (
:id, :dataset_id, 1, :source_registry_id, :source_snapshot_id,
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
'passed', 'complete', 'complete', :checksum_sha256
)
"""
),
{
"id": version_id,
"dataset_id": dataset_a,
"source_registry_id": grb_id,
"source_snapshot_id": grb_snapshot_id,
"validation_report_json": _accepted_report(),
"checksum_sha256": "a" * 64,
},
)
connection.execute(
text(
"""
INSERT INTO dataset_lineage_edges (
id, parent_dataset_id, child_dataset_id, relation_type, transformation_name
) VALUES
(:edge_a, :dataset_a, :dataset_b, 'derived_from', 'clip'),
(:edge_b, :dataset_b, :dataset_c, 'derived_from', 'buffer')
"""
),
{
"edge_a": edge_a,
"edge_b": edge_b,
"dataset_a": dataset_a,
"dataset_b": dataset_b,
"dataset_c": dataset_c,
},
)
_assert_constraint_rejection(
engine,
"""
INSERT INTO datasets (
id, project_id, name, dataset_type, source, source_name, status,
source_registry_id, source_snapshot_id, validation_status,
data_contract_key, data_contract_version, validation_report_json,
provenance_status, lineage_status, quarantine_status, checksum_sha256
) VALUES (
:id, :project_id, 'mismatched.geojson', 'vector', 'governed', 'grb', 'ready',
:source_registry_id, :source_snapshot_id, 'passed',
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
'complete', 'complete', 'not_quarantined', :checksum_sha256
)
""",
{
"id": uuid4(),
"project_id": project_id,
"source_registry_id": grb_id,
"source_snapshot_id": osm_snapshot_id,
"validation_report_json": _accepted_report(),
"checksum_sha256": "b" * 64,
},
label="snapshot-registry mismatch guard",
)
_assert_constraint_rejection(
engine,
"""
INSERT INTO datasets (
id, project_id, name, dataset_type, source, source_name, status,
source_registry_id, source_snapshot_id, validation_status,
provenance_status, lineage_status, quarantine_status, checksum_sha256
) VALUES (
:id, :project_id, 'unreported.geojson', 'vector', 'governed', 'grb', 'ready',
:source_registry_id, :source_snapshot_id, 'passed', 'complete', 'complete', 'not_quarantined', :checksum_sha256
)
""",
{
"id": uuid4(),
"project_id": project_id,
"source_registry_id": grb_id,
"source_snapshot_id": grb_snapshot_id,
"checksum_sha256": "a" * 64,
},
label="passed contract report guard",
)
_assert_constraint_rejection(
engine,
"""
INSERT INTO datasets (
id, project_id, name, dataset_type, source, source_name, status,
source_registry_id, source_snapshot_id, validation_status,
data_contract_key, data_contract_version, validation_report_json,
provenance_status, lineage_status, quarantine_status
) VALUES (
:id, :project_id, 'missing-checksum.geojson', 'vector', 'governed', 'grb', 'ready',
:source_registry_id, :source_snapshot_id, 'passed',
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
'complete', 'complete', 'not_quarantined'
)
""",
{
"id": uuid4(),
"project_id": project_id,
"source_registry_id": grb_id,
"source_snapshot_id": grb_snapshot_id,
"validation_report_json": _accepted_report(),
},
label="passed dataset checksum required guard",
)
_assert_constraint_rejection(
engine,
"""
INSERT INTO datasets (
id, project_id, name, dataset_type, source, source_name, status,
source_registry_id, source_snapshot_id, validation_status,
data_contract_key, data_contract_version, validation_report_json,
provenance_status, lineage_status, quarantine_status, checksum_sha256
) VALUES (
:id, :project_id, 'checksum-mismatch.geojson', 'vector', 'governed', 'grb', 'ready',
:source_registry_id, :source_snapshot_id, 'passed',
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
'complete', 'complete', 'not_quarantined', :checksum_sha256
)
""",
{
"id": uuid4(),
"project_id": project_id,
"source_registry_id": grb_id,
"source_snapshot_id": grb_snapshot_id,
"validation_report_json": _accepted_report(),
"checksum_sha256": "f" * 64,
},
label="passed dataset snapshot checksum binding guard",
)
_assert_constraint_rejection(
engine,
"""
INSERT INTO dataset_versions (
id, dataset_id, version, source_registry_id, source_snapshot_id,
data_contract_key, data_contract_version, validation_report_json,
validation_status, provenance_status, lineage_status
) VALUES (
:id, :dataset_id, 2, :source_registry_id, :source_snapshot_id,
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
'passed', 'complete', 'complete'
)
""",
{
"id": uuid4(),
"dataset_id": dataset_a,
"source_registry_id": grb_id,
"source_snapshot_id": grb_snapshot_id,
"validation_report_json": _accepted_report(),
},
label="passed dataset version checksum required guard",
)
_assert_constraint_rejection(
engine,
"""
INSERT INTO dataset_versions (
id, dataset_id, version, source_registry_id, source_snapshot_id,
data_contract_key, data_contract_version, validation_report_json,
validation_status, provenance_status, lineage_status, checksum_sha256
) VALUES (
:id, :dataset_id, 2, :source_registry_id, :source_snapshot_id,
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
'passed', 'complete', 'complete', :checksum_sha256
)
""",
{
"id": uuid4(),
"dataset_id": dataset_a,
"source_registry_id": grb_id,
"source_snapshot_id": grb_snapshot_id,
"validation_report_json": _accepted_report(),
"checksum_sha256": "f" * 64,
},
label="passed dataset version snapshot checksum binding guard",
)
_assert_constraint_rejection(
engine,
"UPDATE datasets SET data_contract_version = '9.9.9' WHERE id = :dataset_id",
{"dataset_id": dataset_b},
label="accepted contract evidence immutability guard",
)
_assert_constraint_rejection(
engine,
"UPDATE dataset_versions SET data_contract_version = '9.9.9' WHERE id = :dataset_version_id",
{"dataset_version_id": version_id},
label="accepted dataset-version contract evidence immutability guard",
)
_assert_constraint_rejection(
engine,
"UPDATE datasets SET storage_path = '/tampered/asset.geojson' WHERE id = :dataset_id",
{"dataset_id": mutation_dataset},
label="accepted dataset artifact immutability guard",
)
_assert_constraint_rejection(
engine,
"UPDATE datasets SET observed_at = CURRENT_TIMESTAMP WHERE id = :dataset_id",
{"dataset_id": mutation_dataset},
label="accepted dataset temporal evidence immutability guard",
)
_assert_constraint_rejection(
engine,
"UPDATE dataset_versions SET storage_path = '/tampered/version.geojson' WHERE id = :dataset_version_id",
{"dataset_version_id": version_id},
label="accepted dataset-version artifact immutability guard",
)
_assert_constraint_rejection(
engine,
"""
UPDATE datasets
SET validation_status = 'failed', checksum_sha256 = 'f' || repeat('0', 63)
WHERE id = :dataset_id
""",
{"dataset_id": mutation_dataset},
label="accepted dataset requires invalidation before artifact replacement",
)
with _transaction(engine) as connection:
connection.execute(
text(
"UPDATE datasets SET validation_status = 'failed' WHERE id = :dataset_id"
),
{"dataset_id": mutation_dataset},
)
connection.execute(
text(
"UPDATE datasets SET storage_path = '/replacement/asset.geojson' WHERE id = :dataset_id"
),
{"dataset_id": mutation_dataset},
)
_assert_constraint_rejection(
engine,
"UPDATE source_snapshots SET source_registry_id = :registry_id WHERE id = :snapshot_id",
{"registry_id": osm_id, "snapshot_id": grb_snapshot_id},
label="immutable snapshot registry guard",
)
_assert_constraint_rejection(
engine,
"UPDATE source_registry SET display_name = 'tampered' WHERE id = :registry_id",
{"registry_id": grb_id},
label="server-owned source registry guard",
)
_assert_constraint_rejection(
engine,
"UPDATE source_snapshots SET checksum_sha256 = :checksum_sha256 WHERE id = :snapshot_id",
{"checksum_sha256": "c" * 64, "snapshot_id": grb_snapshot_id},
label="immutable snapshot evidence guard",
)
_assert_constraint_rejection(
engine,
"""
INSERT INTO dataset_lineage_edges (
id, parent_dataset_id, child_dataset_id, relation_type, transformation_name
) VALUES (:id, :parent_dataset_id, :child_dataset_id, 'derived_from', 'cycle')
""",
{
"id": uuid4(),
"parent_dataset_id": dataset_c,
"child_dataset_id": dataset_a,
},
label="lineage cycle guard",
)
_assert_constraint_rejection(
engine,
"UPDATE dataset_lineage_edges SET transformation_name = 'tampered' WHERE id = :id",
{"id": edge_a},
label="lineage edge immutability update guard",
)
_assert_constraint_rejection(
engine,
"DELETE FROM dataset_lineage_edges WHERE id = :id",
{"id": edge_b},
label="lineage edge immutability delete guard",
)
with _transaction(engine) as connection:
connection.execute(
text(
"""
INSERT INTO dataset_quarantines (
id, dataset_version_id, stage, reason_code, status
) VALUES (:id, :dataset_version_id, 'integration_test', 'CHECKSUM_MISMATCH', 'quarantined')
"""
),
{"id": uuid4(), "dataset_version_id": version_id},
)
dataset_state = (
connection.execute(
text(
"""
SELECT status, quarantine_status, validation_status, provenance_status, lineage_status
FROM datasets WHERE id = :dataset_id
"""
),
{"dataset_id": dataset_a},
)
.mappings()
.one()
)
version_state = (
connection.execute(
text(
"""
SELECT validation_status, provenance_status, lineage_status
FROM dataset_versions WHERE id = :dataset_version_id
"""
),
{"dataset_version_id": version_id},
)
.mappings()
.one()
)
snapshot_state = connection.execute(
text(
"SELECT ingest_status FROM source_snapshots WHERE id = :snapshot_id"
),
{"snapshot_id": grb_snapshot_id},
).scalar_one()
child_dataset_state = (
connection.execute(
text(
"""
SELECT status, quarantine_status, validation_status, provenance_status, lineage_status
FROM datasets WHERE id = :dataset_id
"""
),
{"dataset_id": dataset_b},
)
.mappings()
.one()
)
grandchild_dataset_state = (
connection.execute(
text(
"""
SELECT status, quarantine_status, validation_status, provenance_status, lineage_status
FROM datasets WHERE id = :dataset_id
"""
),
{"dataset_id": dataset_c},
)
.mappings()
.one()
)
shared_dataset_state = (
connection.execute(
text(
"""
SELECT status, quarantine_status, validation_status, provenance_status, lineage_status
FROM datasets WHERE id = :dataset_id
"""
),
{"dataset_id": shared_dataset},
)
.mappings()
.one()
)
expected_dataset_state = {
"status": "quarantined",
"quarantine_status": "quarantined",
"validation_status": "failed",
"provenance_status": "incomplete",
"lineage_status": "incomplete",
}
if dict(dataset_state) != expected_dataset_state:
raise AssertionError(
f"quarantine parent propagation mismatch: {dict(dataset_state)}"
)
expected_version_state = {
"validation_status": "failed",
"provenance_status": "incomplete",
"lineage_status": "incomplete",
}
if dict(version_state) != expected_version_state:
raise AssertionError(
f"quarantine version propagation mismatch: {dict(version_state)}"
)
if snapshot_state != "quarantined":
raise AssertionError(
f"quarantine snapshot propagation mismatch: {snapshot_state}"
)
if dict(child_dataset_state) != expected_dataset_state:
raise AssertionError(
"quarantine transitive-child propagation mismatch: "
f"{dict(child_dataset_state)}"
)
if dict(grandchild_dataset_state) != expected_dataset_state:
raise AssertionError(
"quarantine transitive-grandchild propagation mismatch: "
f"{dict(grandchild_dataset_state)}"
)
if dict(shared_dataset_state) != expected_dataset_state:
raise AssertionError(
"quarantine shared-snapshot propagation mismatch: "
f"{dict(shared_dataset_state)}"
)
finally:
engine.dispose()
downgrade_output = (
_run_alembic(database_url, "downgrade", "base")
if verify_downgrade
else "not requested"
)
return {
"schema_version": 1,
"phase": "P2",
"started_at": started_at,
"completed_at": datetime.now(UTC).isoformat(),
"migration_revision": "202608010001",
"database_name": str(make_url(database_url).database),
"result": "passed",
"guards": {
"snapshot_registry_pairing": "passed",
"snapshot_registry_immutable": "passed",
"source_registry_immutable": "passed",
"snapshot_evidence_immutable": "passed",
"lineage_cycle": "passed",
"lineage_edge_immutable": "passed",
"version_quarantine_propagation": "passed",
"snapshot_quarantine_fanout": "passed",
"transitive_lineage_quarantine": "passed",
"passed_contract_report_required": "passed",
"passed_dataset_checksum_required_and_snapshot_bound": "passed",
"passed_dataset_version_checksum_required_and_snapshot_bound": "passed",
"accepted_contract_evidence_immutable": "passed",
"accepted_dataset_version_contract_evidence_immutable": "passed",
"accepted_artifact_and_temporal_evidence_immutable": "passed",
},
"upgrade_output_tail": upgrade_output.splitlines()[-8:],
"downgrade_output_tail": downgrade_output.splitlines()[-8:],
}
def main() -> int:
parser = argparse.ArgumentParser()
parser.add_argument("--database-url", required=True)
parser.add_argument(
"--output",
type=Path,
default=ROOT
/ "artifacts"
/ "evidence"
/ "accuracy"
/ "P2"
/ "postgres-migration-guards.json",
)
parser.add_argument("--skip-downgrade", action="store_true")
args = parser.parse_args()
try:
result = verify(args.database_url, verify_downgrade=not args.skip_downgrade)
except Exception as exc:
result = {
"schema_version": 1,
"phase": "P2",
"completed_at": datetime.now(UTC).isoformat(),
"migration_revision": "202608010001",
"result": "failed",
"error_type": type(exc).__name__,
"error": str(exc),
}
_json_dump(args.output, result)
print(json.dumps(result, indent=2))
return 1
_json_dump(args.output, result)
print(json.dumps(result, indent=2))
return 0
if __name__ == "__main__":
raise SystemExit(main())
+9
View File
@@ -0,0 +1,9 @@
[
{
"class_id": 2,
"x_center": 0.95,
"y_center": 0.5,
"width": 0.2,
"height": -0.1
}
]
+9
View File
@@ -0,0 +1,9 @@
[
{
"class_id": 0,
"x_center": 0.5,
"y_center": 0.5,
"width": 0.2,
"height": 0.3
}
]
@@ -0,0 +1,24 @@
{
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {
"native_id": "GRB-GBG-001",
"feature_status": "active"
},
"geometry": {
"type": "Polygon",
"coordinates": [
[
[4.8470, 51.1550],
[4.8480, 51.1550],
[4.8480, 51.1560],
[4.8470, 51.1560],
[4.8470, 51.1550]
]
]
}
}
]
}
@@ -0,0 +1,24 @@
{
"type": "FeatureCollection",
"features": [
{
"type": "Feature",
"properties": {
"native_id": "GRB-GBG-002",
"feature_status": "active"
},
"geometry": {
"type": "Polygon",
"coordinates": [
[
[193277.5, 205708.3],
[193377.5, 205708.3],
[193377.5, 205808.3],
[193277.5, 205808.3],
[193277.5, 205708.3]
]
]
}
}
]
}
+249
View File
@@ -1,9 +1,14 @@
from __future__ import annotations
import hashlib
import importlib.util
import json
import sys
import types
from pathlib import Path
import pytest
from PIL import Image
ROOT = Path(__file__).resolve().parents[1]
@@ -22,6 +27,87 @@ miner = load("build_building_proposal_classifier_dataset")
trainer = load("train_building_proposal_classifier")
def _sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def _write_json(path: Path, payload: dict) -> None:
path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8")
def _governed_proposal_crop_fixture(tmp_path: Path) -> tuple[Path, Path]:
"""Create a complete tiny crop release without importing Torch or YOLO."""
corpus_manifest = tmp_path / "corpus-manifest.json"
_write_json(corpus_manifest, {"samples": []})
corpus_sha256 = _sha256(corpus_manifest)
corpus_freeze = tmp_path / "corpus-freeze.json"
_write_json(corpus_freeze, {"immutable": True})
summary = tmp_path / "source-summary.json"
_write_json(summary, {"source_manifest_sha256": corpus_sha256, "tiles": []})
dataset_dir = tmp_path / "proposal-crops"
entries: list[dict] = []
for split in ("train", "val"):
for label, colour in (("negative", (0, 0, 0)), ("positive", (255, 255, 255))):
crop_path = dataset_dir / split / label / f"{split}-{label}.jpg"
crop_path.parent.mkdir(parents=True, exist_ok=True)
Image.new("RGB", (2, 2), colour).save(crop_path)
relative_path = crop_path.relative_to(dataset_dir).as_posix()
entries.append(
{
"relative_path": relative_path,
"sha256": _sha256(crop_path),
"size_bytes": crop_path.stat().st_size,
"split": split,
"label": label,
"sample_slug": f"{split}-{label}",
"proposal_index": 0,
"proposal_score": 0.8,
"source_box_xyxy": [0.0, 0.0, 1.0, 1.0],
"source_image_path": str(tmp_path / "source-image.tif"),
"source_image_sha256": "a" * 64,
"source_label_path": str(tmp_path / "source-label.txt"),
"source_label_sha256": "b" * 64,
}
)
entries.sort(key=lambda item: item["relative_path"])
counts = {f"{split}/{label}": 1 for split in ("train", "val") for label in ("negative", "positive")}
payload: dict = {
"schema_version": 1,
"status": "ok",
"immutable": True,
"dataset_kind": "building_proposal_classifier_crops",
"fixture_mode": False,
"governed_corpus_live_recheck": True,
"source": {
"corpus_manifest": {"path": str(corpus_manifest), "sha256": corpus_sha256},
"corpus_freeze": {"path": str(corpus_freeze), "sha256": _sha256(corpus_freeze)},
"summary": {
"path": str(summary),
"sha256": _sha256(summary),
"source_manifest_sha256": corpus_sha256,
},
"training_release": {
"dataset_yaml_path": "fixture-dataset.yaml",
"dataset_yaml_sha256": "d" * 64,
"corpus_manifest_sha256": corpus_sha256,
},
"proposal_model": {"path": str(tmp_path / "proposal.pt"), "sha256": "c" * 64},
},
"parameters": {"crop_scale": 1.4},
"counts": counts,
"sample_counts": {item["sample_slug"]: 1 for item in entries},
"tile_count": 2,
"crop_count": len(entries),
"crops_sha256": hashlib.sha256(trainer._canonical_json_bytes({"crops": entries})).hexdigest(),
"crops": entries,
}
payload["manifest_sha256"] = trainer._payload_sha256(payload)
_write_json(dataset_dir / trainer.PROPOSAL_DATASET_PROVENANCE_NAME, payload)
return dataset_dir, corpus_manifest
def test_classify_proposals_consumes_reference_once() -> None:
reference = [(0.0, 0.0, 10.0, 10.0)]
proposals = [((0.0, 0.0, 10.0, 10.0), 0.9), ((0.0, 0.0, 10.0, 10.0), 0.8)]
@@ -38,3 +124,166 @@ def test_eligible_tiles_rejects_protected_manifest_split() -> None:
def test_binary_metrics() -> None:
result = trainer.binary_metrics([0.9, 0.8, 0.2, 0.1], [1, 0, 1, 0])
assert result == {"tp": 1, "fp": 1, "fn": 1, "precision": 0.5, "recall": 0.5, "f1": 0.5}
def test_builder_rechecks_frozen_corpus_against_live_governed_state(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
observed: dict[str, object] = {}
def fake_assert(path: Path, **kwargs: object) -> dict:
observed["path"] = path
observed.update(kwargs)
return {"samples": []}
monkeypatch.setattr(miner, "assert_frozen_manifest_training_eligible", fake_assert)
manifest_path = tmp_path / "corpus.json"
manifest_path.write_text("{}", encoding="utf-8")
assert miner.load_governed_corpus_manifest(manifest_path, fixture_mode=False) == {"samples": []}
assert observed == {"path": manifest_path, "fixture_mode": False, "verify_live": True}
def test_builder_rejects_summary_not_bound_to_exact_corpus_manifest(tmp_path: Path) -> None:
manifest_path = tmp_path / "corpus.json"
manifest_path.write_text('{"samples": []}', encoding="utf-8")
with pytest.raises(ValueError, match="not bound"):
miner.assert_summary_source_manifest_binding({"source_manifest_sha256": "0" * 64}, manifest_path)
def test_builder_emits_immutable_checksum_bound_crop_provenance(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
class FakeTensor:
def __init__(self, values: list[list[float]] | list[float]) -> None:
self._values = values
def cpu(self) -> "FakeTensor":
return self
def tolist(self) -> list[list[float]] | list[float]:
return self._values
class FakeBoxes:
xyxy = FakeTensor([[40.0, 40.0, 60.0, 60.0], [0.0, 0.0, 10.0, 10.0]])
conf = FakeTensor([0.9, 0.8])
class FakeResult:
boxes = FakeBoxes()
class FakeYOLO:
def __init__(self, model: str) -> None:
self.model = model
def predict(self, sources: list[str], **_kwargs: object) -> list[FakeResult]:
return [FakeResult() for _source in sources]
corpus_manifest = tmp_path / "corpus-manifest.json"
manifest = {
"samples": [
{"sample_slug": "train-a", "split": "train", "region": "flanders"},
{"sample_slug": "val-b", "split": "val", "region": "flanders"},
]
}
_write_json(corpus_manifest, manifest)
_write_json(tmp_path / "corpus-freeze.json", {"immutable": True})
tiles: list[dict[str, object]] = []
for sample_slug, split in (("train-a", "train"), ("val-b", "val")):
image_path = tmp_path / f"{sample_slug}.png"
Image.new("RGB", (100, 100), (50, 100, 150)).save(image_path)
label_path = tmp_path / f"{sample_slug}.txt"
label_path.write_text("0 0.5 0.5 0.2 0.2\n", encoding="utf-8")
tiles.append(
{
"sample_slug": sample_slug,
"split": split,
"kept": True,
"image_path": str(image_path),
"label_path": str(label_path),
}
)
summary_path = tmp_path / "source-summary.json"
_write_json(
summary_path,
{
"source_manifest_sha256": _sha256(corpus_manifest),
"training_release_manifest": "fixture-training-release.json",
"training_release_manifest_sha256": "a" * 64,
"training_asset_manifest": "fixture-training-assets.json",
"tiles": tiles,
},
)
model_path = tmp_path / "proposal-model.pt"
model_path.write_bytes(b"proposal-model")
output_dir = tmp_path / "proposal-crops"
monkeypatch.setattr(miner, "load_governed_corpus_manifest", lambda *_args, **_kwargs: manifest)
monkeypatch.setattr(
miner,
"assert_yolo_summary_bound_to_embedded_training_release",
lambda **_kwargs: {
"dataset_yaml": {"path": "fixture-dataset.yaml", "sha256": "d" * 64},
"corpus": {"manifest_sha256": _sha256(corpus_manifest)},
},
)
monkeypatch.setitem(sys.modules, "ultralytics", types.SimpleNamespace(YOLO=FakeYOLO))
monkeypatch.setattr(
miner.sys,
"argv",
[
"build_building_proposal_classifier_dataset.py",
"--model",
str(model_path),
"--summary",
str(summary_path),
"--corpus-manifest",
str(corpus_manifest),
"--output-dir",
str(output_dir),
],
)
assert miner.main() == 0
provenance_path = output_dir / miner.PROPOSAL_DATASET_PROVENANCE_NAME
provenance = json.loads(provenance_path.read_text(encoding="utf-8"))
assert provenance["immutable"] is True
assert provenance["crop_count"] == 4
assert provenance["manifest_sha256"] == miner._immutable_payload_sha256(provenance)
assert all(_sha256(output_dir / item["relative_path"]) == item["sha256"] for item in provenance["crops"])
with pytest.raises(RuntimeError, match="immutable provenance"):
miner._write_immutable_json(provenance_path, {"different": True})
def test_trainer_validates_every_crop_and_source_binding_before_torch(
monkeypatch: pytest.MonkeyPatch,
tmp_path: Path,
) -> None:
dataset_dir, corpus_manifest = _governed_proposal_crop_fixture(tmp_path)
monkeypatch.setattr(
trainer,
"assert_yolo_summary_bound_to_embedded_training_release",
lambda **_kwargs: {
"dataset_yaml": {"path": "fixture-dataset.yaml", "sha256": "d" * 64},
"corpus": {"manifest_sha256": _sha256(corpus_manifest)},
},
)
payload = trainer.validate_proposal_dataset_provenance(dataset_dir, corpus_manifest, fixture_mode=False)
assert payload["crop_count"] == 4
crop = dataset_dir / "train" / "positive" / "train-positive.jpg"
crop.write_bytes(b"tampered")
with pytest.raises(trainer.ProposalDatasetProvenanceError, match="checksum"):
trainer.validate_proposal_dataset_provenance(dataset_dir, corpus_manifest, fixture_mode=False)
def test_trainer_rechecks_live_governed_corpus_before_pytorch(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
observed: dict[str, object] = {}
def fake_assert(path: Path, **kwargs: object) -> dict:
observed["path"] = path
observed.update(kwargs)
return {"samples": []}
monkeypatch.setattr(trainer, "assert_frozen_manifest_training_eligible", fake_assert)
manifest_path = tmp_path / "corpus.json"
manifest_path.write_text("{}", encoding="utf-8")
assert trainer.load_governed_corpus_manifest(manifest_path, fixture_mode=False) == {"samples": []}
assert observed == {"path": manifest_path, "fixture_mode": False, "verify_live": True}