feat(provenance): govern source snapshots and data inputs
This commit is contained in:
@@ -25,6 +25,8 @@ build/
|
|||||||
/artifacts/evidence/accuracy/*
|
/artifacts/evidence/accuracy/*
|
||||||
!/artifacts/evidence/accuracy/P1/
|
!/artifacts/evidence/accuracy/P1/
|
||||||
!/artifacts/evidence/accuracy/P1/**
|
!/artifacts/evidence/accuracy/P1/**
|
||||||
|
!/artifacts/evidence/accuracy/P2/
|
||||||
|
!/artifacts/evidence/accuracy/P2/**
|
||||||
/.cache/
|
/.cache/
|
||||||
/datasets/raw/*
|
/datasets/raw/*
|
||||||
/datasets/processed/*
|
/datasets/processed/*
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -1 +1,15 @@
|
|||||||
__all__ = ["analysis", "areas", "assistant", "auth", "datasets", "health", "projects", "exports", "jobs", "external", "qa", "temporal"]
|
__all__ = [
|
||||||
|
"analysis",
|
||||||
|
"areas",
|
||||||
|
"assistant",
|
||||||
|
"auth",
|
||||||
|
"datasets",
|
||||||
|
"exports",
|
||||||
|
"external",
|
||||||
|
"health",
|
||||||
|
"jobs",
|
||||||
|
"projects",
|
||||||
|
"qa",
|
||||||
|
"source_registry",
|
||||||
|
"temporal",
|
||||||
|
]
|
||||||
|
|||||||
@@ -0,0 +1,119 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends
|
||||||
|
from sqlalchemy import func, or_
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.db.session import get_db
|
||||||
|
from app.models import Dataset, DatasetLineageEdge, DatasetQuarantine, Project, SourceRegistry, SourceSnapshot
|
||||||
|
from app.schemas import (
|
||||||
|
DatasetLineageEdgeRead,
|
||||||
|
DatasetProvenanceRead,
|
||||||
|
DatasetQuarantineRead,
|
||||||
|
Envelope,
|
||||||
|
ItemList,
|
||||||
|
SourceRegistryDetailRead,
|
||||||
|
SourceRegistryRead,
|
||||||
|
SourceSnapshotRead,
|
||||||
|
)
|
||||||
|
from app.utils.response import envelope
|
||||||
|
|
||||||
|
|
||||||
|
router = APIRouter(tags=["source-registry"])
|
||||||
|
|
||||||
|
|
||||||
|
def _source_read(source: SourceRegistry, *, snapshot_count: int = 0) -> SourceRegistryRead:
|
||||||
|
return SourceRegistryRead.model_validate(source).model_copy(update={"snapshot_count": int(snapshot_count)})
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/source-registry", response_model=Envelope[ItemList[SourceRegistryRead]])
|
||||||
|
def list_source_registry(
|
||||||
|
classification: str | None = None,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
query = (
|
||||||
|
db.query(SourceRegistry, func.count(SourceSnapshot.id).label("snapshot_count"))
|
||||||
|
.outerjoin(SourceSnapshot, SourceSnapshot.source_registry_id == SourceRegistry.id)
|
||||||
|
)
|
||||||
|
if classification:
|
||||||
|
query = query.filter(SourceRegistry.classification == classification.strip().lower())
|
||||||
|
rows = (
|
||||||
|
query.group_by(SourceRegistry.id)
|
||||||
|
.order_by(SourceRegistry.classification.asc(), SourceRegistry.display_name.asc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
items = [_source_read(source, snapshot_count=count) for source, count in rows]
|
||||||
|
return envelope({"items": items, "total": len(items)})
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/source-registry/{source_key}", response_model=Envelope[SourceRegistryDetailRead])
|
||||||
|
def get_source_registry_entry(source_key: str, db: Session = Depends(get_db)) -> dict:
|
||||||
|
normalized_key = source_key.strip().lower()
|
||||||
|
source = db.query(SourceRegistry).filter(SourceRegistry.source_key == normalized_key).one_or_none()
|
||||||
|
if source is None:
|
||||||
|
raise AppError(code="SOURCE_REGISTRY_ENTRY_NOT_FOUND", message="Source registry entry was not found", status_code=404)
|
||||||
|
snapshots = (
|
||||||
|
db.query(SourceSnapshot)
|
||||||
|
.filter(SourceSnapshot.source_registry_id == source.id)
|
||||||
|
.order_by(SourceSnapshot.fetched_at.desc(), SourceSnapshot.created_at.desc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
detail = SourceRegistryDetailRead(
|
||||||
|
source=_source_read(source, snapshot_count=len(snapshots)),
|
||||||
|
snapshots=[SourceSnapshotRead.model_validate(snapshot) for snapshot in snapshots],
|
||||||
|
)
|
||||||
|
return envelope(detail)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get(
|
||||||
|
"/projects/{project_id}/datasets/{dataset_id}/provenance",
|
||||||
|
response_model=Envelope[DatasetProvenanceRead],
|
||||||
|
)
|
||||||
|
def get_dataset_provenance(
|
||||||
|
project_id: UUID,
|
||||||
|
dataset_id: UUID,
|
||||||
|
db: Session = Depends(get_db),
|
||||||
|
) -> dict:
|
||||||
|
if db.get(Project, project_id) is None:
|
||||||
|
raise AppError(code="PROJECT_NOT_FOUND", message="Project not found", status_code=404)
|
||||||
|
dataset = db.get(Dataset, dataset_id)
|
||||||
|
if dataset is None or dataset.project_id != project_id:
|
||||||
|
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
||||||
|
|
||||||
|
source = db.get(SourceRegistry, dataset.source_registry_id) if dataset.source_registry_id else None
|
||||||
|
snapshot = db.get(SourceSnapshot, dataset.source_snapshot_id) if dataset.source_snapshot_id else None
|
||||||
|
lineage = (
|
||||||
|
db.query(DatasetLineageEdge)
|
||||||
|
.filter(
|
||||||
|
or_(
|
||||||
|
DatasetLineageEdge.parent_dataset_id == dataset.id,
|
||||||
|
DatasetLineageEdge.child_dataset_id == dataset.id,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.order_by(DatasetLineageEdge.created_at.asc(), DatasetLineageEdge.id.asc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
quarantines = (
|
||||||
|
db.query(DatasetQuarantine)
|
||||||
|
.filter(DatasetQuarantine.dataset_id == dataset.id)
|
||||||
|
.order_by(DatasetQuarantine.created_at.desc(), DatasetQuarantine.id.desc())
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
result = DatasetProvenanceRead(
|
||||||
|
dataset_id=dataset.id,
|
||||||
|
source=_source_read(source) if source else None,
|
||||||
|
snapshot=SourceSnapshotRead.model_validate(snapshot) if snapshot else None,
|
||||||
|
data_contract_key=dataset.data_contract_key,
|
||||||
|
data_contract_version=dataset.data_contract_version,
|
||||||
|
validation_status=dataset.validation_status,
|
||||||
|
validation_report_json=dataset.validation_report_json,
|
||||||
|
provenance_status=dataset.provenance_status,
|
||||||
|
lineage_status=dataset.lineage_status,
|
||||||
|
quarantine_status=dataset.quarantine_status,
|
||||||
|
lineage=[DatasetLineageEdgeRead.model_validate(item) for item in lineage],
|
||||||
|
quarantines=[DatasetQuarantineRead.model_validate(item) for item in quarantines],
|
||||||
|
)
|
||||||
|
return envelope(result)
|
||||||
+2
-1
@@ -12,7 +12,7 @@ from fastapi.exceptions import RequestValidationError
|
|||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
from fastapi.responses import JSONResponse
|
from fastapi.responses import JSONResponse
|
||||||
|
|
||||||
from app.api.routes import analysis, aoi_operations, areas, assistant, auth, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation, selection_partitions, temporal
|
from app.api.routes import analysis, aoi_operations, areas, assistant, auth, datasets, demo, detection, exports, external, health, jobs, projects, qa, quality_checks, segmentation, selection_partitions, source_registry, temporal
|
||||||
from app.core.config import get_settings
|
from app.core.config import get_settings
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.core.logging import configure_logging
|
from app.core.logging import configure_logging
|
||||||
@@ -103,6 +103,7 @@ def create_app() -> FastAPI:
|
|||||||
app.include_router(quality_checks.router, prefix=settings.api_prefix)
|
app.include_router(quality_checks.router, prefix=settings.api_prefix)
|
||||||
app.include_router(exports.router, prefix=settings.api_prefix)
|
app.include_router(exports.router, prefix=settings.api_prefix)
|
||||||
app.include_router(external.router, prefix=settings.api_prefix)
|
app.include_router(external.router, prefix=settings.api_prefix)
|
||||||
|
app.include_router(source_registry.router, prefix=settings.api_prefix)
|
||||||
app.include_router(demo.router, prefix=settings.api_prefix)
|
app.include_router(demo.router, prefix=settings.api_prefix)
|
||||||
app.include_router(qa.router, prefix=settings.api_prefix)
|
app.include_router(qa.router, prefix=settings.api_prefix)
|
||||||
app.include_router(detection.router, prefix=settings.api_prefix)
|
app.include_router(detection.router, prefix=settings.api_prefix)
|
||||||
|
|||||||
@@ -1,4 +1,24 @@
|
|||||||
from .entities import AoiOperation, AoiOperationPartition, AnalysisRun, Area, Dataset, DatasetVersion, Detection, DetectionReview, Export, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature
|
from .entities import (
|
||||||
|
AoiOperation,
|
||||||
|
AoiOperationPartition,
|
||||||
|
AnalysisRun,
|
||||||
|
Area,
|
||||||
|
Dataset,
|
||||||
|
DatasetLineageEdge,
|
||||||
|
DatasetQuarantine,
|
||||||
|
DatasetVersion,
|
||||||
|
Detection,
|
||||||
|
DetectionReview,
|
||||||
|
Export,
|
||||||
|
Job,
|
||||||
|
Metric,
|
||||||
|
Project,
|
||||||
|
QualityCheck,
|
||||||
|
Segmentation,
|
||||||
|
SourceRegistry,
|
||||||
|
SourceSnapshot,
|
||||||
|
VectorFeature,
|
||||||
|
)
|
||||||
|
|
||||||
__all__ = [
|
__all__ = [
|
||||||
"AnalysisRun",
|
"AnalysisRun",
|
||||||
@@ -6,6 +26,8 @@ __all__ = [
|
|||||||
"AoiOperationPartition",
|
"AoiOperationPartition",
|
||||||
"Area",
|
"Area",
|
||||||
"Dataset",
|
"Dataset",
|
||||||
|
"DatasetLineageEdge",
|
||||||
|
"DatasetQuarantine",
|
||||||
"DatasetVersion",
|
"DatasetVersion",
|
||||||
"Detection",
|
"Detection",
|
||||||
"DetectionReview",
|
"DetectionReview",
|
||||||
@@ -15,5 +37,7 @@ __all__ = [
|
|||||||
"Project",
|
"Project",
|
||||||
"QualityCheck",
|
"QualityCheck",
|
||||||
"Segmentation",
|
"Segmentation",
|
||||||
|
"SourceRegistry",
|
||||||
|
"SourceSnapshot",
|
||||||
"VectorFeature",
|
"VectorFeature",
|
||||||
]
|
]
|
||||||
|
|||||||
@@ -12,6 +12,37 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship
|
|||||||
from app.db.base import Base
|
from app.db.base import Base
|
||||||
|
|
||||||
|
|
||||||
|
SOURCE_CLASSIFICATIONS = (
|
||||||
|
"authoritative",
|
||||||
|
"corroborative",
|
||||||
|
"contextual",
|
||||||
|
"derived",
|
||||||
|
"experimental",
|
||||||
|
)
|
||||||
|
SOURCE_FRESHNESS_STATUSES = (
|
||||||
|
"unknown",
|
||||||
|
"current",
|
||||||
|
"due",
|
||||||
|
"stale",
|
||||||
|
"not_applicable",
|
||||||
|
"review_required",
|
||||||
|
)
|
||||||
|
SOURCE_INGEST_STATUSES = (
|
||||||
|
"registered",
|
||||||
|
"configured",
|
||||||
|
"not_configured",
|
||||||
|
"available",
|
||||||
|
"ingested",
|
||||||
|
"failed",
|
||||||
|
"quarantined",
|
||||||
|
"legacy_unverified",
|
||||||
|
)
|
||||||
|
PROVENANCE_STATUSES = ("complete", "incomplete", "not_applicable")
|
||||||
|
LINEAGE_STATUSES = ("complete", "incomplete", "not_applicable")
|
||||||
|
VALIDATION_STATUSES = ("not_validated", "passed", "failed")
|
||||||
|
QUARANTINE_STATUSES = ("not_quarantined", "quarantined")
|
||||||
|
|
||||||
|
|
||||||
class Project(Base):
|
class Project(Base):
|
||||||
__tablename__ = "projects"
|
__tablename__ = "projects"
|
||||||
|
|
||||||
@@ -42,6 +73,123 @@ class Area(Base):
|
|||||||
project: Mapped[Project] = relationship("Project", back_populates="areas")
|
project: Mapped[Project] = relationship("Project", back_populates="areas")
|
||||||
|
|
||||||
|
|
||||||
|
class SourceRegistry(Base):
|
||||||
|
"""Server-owned source identity and authority contract.
|
||||||
|
|
||||||
|
Dataset metadata remains descriptive until a governed importer binds a
|
||||||
|
dataset to both this registry entry and an immutable SourceSnapshot.
|
||||||
|
"""
|
||||||
|
|
||||||
|
__tablename__ = "source_registry"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("source_key", name="uq_source_registry_source_key"),
|
||||||
|
CheckConstraint(
|
||||||
|
"classification IN ('authoritative', 'corroborative', 'contextual', 'derived', 'experimental')",
|
||||||
|
name="ck_source_registry_classification",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')",
|
||||||
|
name="ck_source_registry_freshness_status",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', "
|
||||||
|
"'failed', 'quarantined', 'legacy_unverified')",
|
||||||
|
name="ck_source_registry_ingest_status",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||||
|
source_key: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||||
|
display_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
classification: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||||
|
authority_name: Mapped[str] = mapped_column(String(255), nullable=False, default="unknown", server_default="unknown")
|
||||||
|
authority_scope_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
provider_adapter_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||||
|
source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
license_name: Mapped[str] = mapped_column(String(255), nullable=False, default="unknown", server_default="unknown")
|
||||||
|
license_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
usage_restrictions: Mapped[str] = mapped_column(Text, nullable=False, default="unknown", server_default="unknown")
|
||||||
|
default_crs: Mapped[str] = mapped_column(String(64), nullable=False, default="unknown", server_default="unknown")
|
||||||
|
default_units: Mapped[str] = mapped_column(String(120), nullable=False, default="unknown", server_default="unknown")
|
||||||
|
spatial_resolution_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
temporal_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
geographic_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
expected_geometry_types_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
|
||||||
|
expected_attributes_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
usage_policy_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
freshness_status: Mapped[str] = mapped_column(
|
||||||
|
String(32), nullable=False, default="unknown", server_default="unknown"
|
||||||
|
)
|
||||||
|
ingest_status: Mapped[str] = mapped_column(
|
||||||
|
String(32), nullable=False, default="registered", server_default="registered"
|
||||||
|
)
|
||||||
|
known_limitations_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
|
||||||
|
registry_metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now())
|
||||||
|
|
||||||
|
snapshots: Mapped[list["SourceSnapshot"]] = relationship(
|
||||||
|
"SourceSnapshot", back_populates="source_registry", cascade="all, delete-orphan"
|
||||||
|
)
|
||||||
|
datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="source_registry")
|
||||||
|
dataset_versions: Mapped[list["DatasetVersion"]] = relationship("DatasetVersion", back_populates="source_registry")
|
||||||
|
|
||||||
|
|
||||||
|
class SourceSnapshot(Base):
|
||||||
|
"""Immutable source-version evidence recorded by governed ingestion."""
|
||||||
|
|
||||||
|
__tablename__ = "source_snapshots"
|
||||||
|
__table_args__ = (
|
||||||
|
UniqueConstraint("source_registry_id", "snapshot_key", name="uq_source_snapshots_registry_key"),
|
||||||
|
CheckConstraint(
|
||||||
|
"freshness_status IN ('unknown', 'current', 'due', 'stale', 'not_applicable', 'review_required')",
|
||||||
|
name="ck_source_snapshots_freshness_status",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"ingest_status IN ('registered', 'configured', 'not_configured', 'available', 'ingested', "
|
||||||
|
"'failed', 'quarantined', 'legacy_unverified')",
|
||||||
|
name="ck_source_snapshots_ingest_status",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"checksum_sha256 = lower(checksum_sha256) AND checksum_sha256 ~ '^[0-9a-f]{64}$'",
|
||||||
|
name="ck_source_snapshots_checksum_sha256",
|
||||||
|
),
|
||||||
|
Index("ix_source_snapshots_registry_fetched", "source_registry_id", "fetched_at"),
|
||||||
|
Index("ix_source_snapshots_checksum", "checksum_sha256"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||||
|
source_registry_id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="CASCADE"), nullable=False
|
||||||
|
)
|
||||||
|
snapshot_key: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
source_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||||
|
snapshot_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, server_default=func.now())
|
||||||
|
source_url: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
checksum_sha256: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
crs: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
units: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||||
|
spatial_resolution_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
temporal_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
geographic_coverage_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
observed_schema_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
freshness_status: Mapped[str] = mapped_column(
|
||||||
|
String(32), nullable=False, default="unknown", server_default="unknown"
|
||||||
|
)
|
||||||
|
ingest_status: Mapped[str] = mapped_column(
|
||||||
|
String(32), nullable=False, default="registered", server_default="registered"
|
||||||
|
)
|
||||||
|
known_limitations_json: Mapped[list] = mapped_column(JSON, nullable=False, default=list)
|
||||||
|
snapshot_metadata_json: Mapped[dict] = mapped_column(JSON, nullable=False, default=dict)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
|
||||||
|
source_registry: Mapped[SourceRegistry] = relationship("SourceRegistry", back_populates="snapshots")
|
||||||
|
datasets: Mapped[list["Dataset"]] = relationship("Dataset", back_populates="source_snapshot")
|
||||||
|
dataset_versions: Mapped[list["DatasetVersion"]] = relationship("DatasetVersion", back_populates="source_snapshot")
|
||||||
|
quarantines: Mapped[list["DatasetQuarantine"]] = relationship("DatasetQuarantine", back_populates="source_snapshot")
|
||||||
|
|
||||||
|
|
||||||
class Dataset(Base):
|
class Dataset(Base):
|
||||||
__tablename__ = "datasets"
|
__tablename__ = "datasets"
|
||||||
__table_args__ = (
|
__table_args__ = (
|
||||||
@@ -49,6 +197,27 @@ class Dataset(Base):
|
|||||||
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
|
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
|
||||||
name="ck_datasets_temporal_valid_range",
|
name="ck_datasets_temporal_valid_range",
|
||||||
),
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"validation_status IN ('not_validated', 'passed', 'failed')",
|
||||||
|
name="ck_datasets_validation_status",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"provenance_status IN ('complete', 'incomplete', 'not_applicable')",
|
||||||
|
name="ck_datasets_provenance_status",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"lineage_status IN ('complete', 'incomplete', 'not_applicable')",
|
||||||
|
name="ck_datasets_lineage_status",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"quarantine_status IN ('not_quarantined', 'quarantined')",
|
||||||
|
name="ck_datasets_quarantine_status",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"ingest_key IS NULL OR btrim(ingest_key) <> ''",
|
||||||
|
name="ck_datasets_ingest_key_not_blank",
|
||||||
|
),
|
||||||
|
UniqueConstraint("project_id", "ingest_key", name="uq_datasets_project_ingest_key"),
|
||||||
Index(
|
Index(
|
||||||
"ix_datasets_project_temporal_series_observed",
|
"ix_datasets_project_temporal_series_observed",
|
||||||
"project_id",
|
"project_id",
|
||||||
@@ -69,6 +238,7 @@ class Dataset(Base):
|
|||||||
content_type: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
content_type: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||||
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
size_bytes: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||||
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
ingest_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||||
derived_from_dataset_id: Mapped[uuid.UUID | None] = mapped_column(
|
derived_from_dataset_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
UUID(as_uuid=True),
|
UUID(as_uuid=True),
|
||||||
ForeignKey("datasets.id", ondelete="SET NULL"),
|
ForeignKey("datasets.id", ondelete="SET NULL"),
|
||||||
@@ -84,6 +254,43 @@ class Dataset(Base):
|
|||||||
reference_layer_name: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
reference_layer_name: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||||
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
source_registry_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
data_contract_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||||
|
data_contract_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
validation_status: Mapped[str] = mapped_column(
|
||||||
|
String(32),
|
||||||
|
nullable=False,
|
||||||
|
default="not_validated",
|
||||||
|
server_default="not_validated",
|
||||||
|
comment="not_validated | passed | failed",
|
||||||
|
)
|
||||||
|
validation_report_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
provenance_status: Mapped[str] = mapped_column(
|
||||||
|
String(32),
|
||||||
|
nullable=False,
|
||||||
|
default="incomplete",
|
||||||
|
server_default="incomplete",
|
||||||
|
comment="complete | incomplete | not_applicable",
|
||||||
|
)
|
||||||
|
lineage_status: Mapped[str] = mapped_column(
|
||||||
|
String(32),
|
||||||
|
nullable=False,
|
||||||
|
default="incomplete",
|
||||||
|
server_default="incomplete",
|
||||||
|
comment="complete | incomplete | not_applicable",
|
||||||
|
)
|
||||||
|
quarantine_status: Mapped[str] = mapped_column(
|
||||||
|
String(32),
|
||||||
|
nullable=False,
|
||||||
|
default="not_quarantined",
|
||||||
|
server_default="not_quarantined",
|
||||||
|
comment="not_quarantined | quarantined",
|
||||||
|
)
|
||||||
imported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
imported_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
temporal_series_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
temporal_series_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||||
observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
observed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
@@ -106,6 +313,23 @@ class Dataset(Base):
|
|||||||
back_populates="dataset",
|
back_populates="dataset",
|
||||||
cascade="all, delete-orphan",
|
cascade="all, delete-orphan",
|
||||||
)
|
)
|
||||||
|
source_registry: Mapped[SourceRegistry | None] = relationship("SourceRegistry", back_populates="datasets")
|
||||||
|
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="datasets")
|
||||||
|
parent_lineage_edges: Mapped[list["DatasetLineageEdge"]] = relationship(
|
||||||
|
"DatasetLineageEdge",
|
||||||
|
foreign_keys="DatasetLineageEdge.parent_dataset_id",
|
||||||
|
back_populates="parent_dataset",
|
||||||
|
cascade="all, delete-orphan",
|
||||||
|
)
|
||||||
|
child_lineage_edges: Mapped[list["DatasetLineageEdge"]] = relationship(
|
||||||
|
"DatasetLineageEdge",
|
||||||
|
foreign_keys="DatasetLineageEdge.child_dataset_id",
|
||||||
|
back_populates="child_dataset",
|
||||||
|
cascade="all, delete-orphan",
|
||||||
|
)
|
||||||
|
quarantines: Mapped[list["DatasetQuarantine"]] = relationship(
|
||||||
|
"DatasetQuarantine", back_populates="dataset", cascade="all, delete-orphan"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class DatasetVersion(Base):
|
class DatasetVersion(Base):
|
||||||
@@ -115,7 +339,24 @@ class DatasetVersion(Base):
|
|||||||
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
|
"valid_to IS NULL OR valid_from IS NULL OR valid_to >= valid_from",
|
||||||
name="ck_dataset_versions_temporal_valid_range",
|
name="ck_dataset_versions_temporal_valid_range",
|
||||||
),
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"validation_status IN ('not_validated', 'passed', 'failed')",
|
||||||
|
name="ck_dataset_versions_validation_status",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"provenance_status IN ('complete', 'incomplete', 'not_applicable')",
|
||||||
|
name="ck_dataset_versions_provenance_status",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"lineage_status IN ('complete', 'incomplete', 'not_applicable')",
|
||||||
|
name="ck_dataset_versions_lineage_status",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"ingest_key IS NULL OR btrim(ingest_key) <> ''",
|
||||||
|
name="ck_dataset_versions_ingest_key_not_blank",
|
||||||
|
),
|
||||||
Index("ix_dataset_versions_dataset_version", "dataset_id", "version", unique=True),
|
Index("ix_dataset_versions_dataset_version", "dataset_id", "version", unique=True),
|
||||||
|
UniqueConstraint("dataset_id", "ingest_key", name="uq_dataset_versions_dataset_ingest_key"),
|
||||||
)
|
)
|
||||||
|
|
||||||
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||||
@@ -127,11 +368,135 @@ class DatasetVersion(Base):
|
|||||||
valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
valid_from: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
valid_to: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
ingest_key: Mapped[str | None] = mapped_column(String(255), nullable=True)
|
||||||
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
source_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
provenance_metadata: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
source_registry_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("source_registry.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
data_contract_key: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||||
|
data_contract_version: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
validation_status: Mapped[str] = mapped_column(
|
||||||
|
String(32),
|
||||||
|
nullable=False,
|
||||||
|
default="not_validated",
|
||||||
|
server_default="not_validated",
|
||||||
|
comment="not_validated | passed | failed",
|
||||||
|
)
|
||||||
|
validation_report_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
provenance_status: Mapped[str] = mapped_column(
|
||||||
|
String(32),
|
||||||
|
nullable=False,
|
||||||
|
default="incomplete",
|
||||||
|
server_default="incomplete",
|
||||||
|
comment="complete | incomplete | not_applicable",
|
||||||
|
)
|
||||||
|
lineage_status: Mapped[str] = mapped_column(
|
||||||
|
String(32),
|
||||||
|
nullable=False,
|
||||||
|
default="incomplete",
|
||||||
|
server_default="incomplete",
|
||||||
|
comment="complete | incomplete | not_applicable",
|
||||||
|
)
|
||||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
|
||||||
dataset: Mapped[Dataset] = relationship("Dataset", back_populates="versions")
|
dataset: Mapped[Dataset] = relationship("Dataset", back_populates="versions")
|
||||||
|
source_registry: Mapped[SourceRegistry | None] = relationship("SourceRegistry", back_populates="dataset_versions")
|
||||||
|
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="dataset_versions")
|
||||||
|
quarantines: Mapped[list["DatasetQuarantine"]] = relationship("DatasetQuarantine", back_populates="dataset_version")
|
||||||
|
|
||||||
|
|
||||||
|
class DatasetLineageEdge(Base):
|
||||||
|
"""Immutable relationship between input/output datasets and transforms."""
|
||||||
|
|
||||||
|
__tablename__ = "dataset_lineage_edges"
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint("parent_dataset_id <> child_dataset_id", name="ck_dataset_lineage_edges_distinct_datasets"),
|
||||||
|
UniqueConstraint(
|
||||||
|
"parent_dataset_id",
|
||||||
|
"child_dataset_id",
|
||||||
|
"relation_type",
|
||||||
|
"transformation_name",
|
||||||
|
name="uq_dataset_lineage_edges_relation",
|
||||||
|
),
|
||||||
|
Index("ix_dataset_lineage_edges_parent", "parent_dataset_id"),
|
||||||
|
Index("ix_dataset_lineage_edges_child", "child_dataset_id"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||||
|
parent_dataset_id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False
|
||||||
|
)
|
||||||
|
child_dataset_id: Mapped[uuid.UUID] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="CASCADE"), nullable=False
|
||||||
|
)
|
||||||
|
parent_dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
child_dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
relation_type: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
transformation_name: Mapped[str] = mapped_column(String(255), nullable=False)
|
||||||
|
transformation_version: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||||
|
parameters_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
input_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
output_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
|
||||||
|
parent_dataset: Mapped[Dataset] = relationship(
|
||||||
|
"Dataset", foreign_keys=[parent_dataset_id], back_populates="parent_lineage_edges"
|
||||||
|
)
|
||||||
|
child_dataset: Mapped[Dataset] = relationship(
|
||||||
|
"Dataset", foreign_keys=[child_dataset_id], back_populates="child_lineage_edges"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class DatasetQuarantine(Base):
|
||||||
|
"""Durable fail-closed record for rejected or doubtful source artifacts."""
|
||||||
|
|
||||||
|
__tablename__ = "dataset_quarantines"
|
||||||
|
__table_args__ = (
|
||||||
|
CheckConstraint(
|
||||||
|
"dataset_id IS NOT NULL OR dataset_version_id IS NOT NULL OR source_snapshot_id IS NOT NULL",
|
||||||
|
name="ck_dataset_quarantines_target_present",
|
||||||
|
),
|
||||||
|
CheckConstraint(
|
||||||
|
"status IN ('quarantined', 'released', 'rejected')",
|
||||||
|
name="ck_dataset_quarantines_status",
|
||||||
|
),
|
||||||
|
Index("ix_dataset_quarantines_dataset_status", "dataset_id", "status"),
|
||||||
|
Index("ix_dataset_quarantines_snapshot_status", "source_snapshot_id", "status"),
|
||||||
|
)
|
||||||
|
|
||||||
|
id: Mapped[uuid.UUID] = mapped_column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
||||||
|
dataset_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("datasets.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
dataset_version_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("dataset_versions.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
source_snapshot_id: Mapped[uuid.UUID | None] = mapped_column(
|
||||||
|
UUID(as_uuid=True), ForeignKey("source_snapshots.id", ondelete="SET NULL"), nullable=True
|
||||||
|
)
|
||||||
|
stage: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||||
|
reason_code: Mapped[str] = mapped_column(String(120), nullable=False)
|
||||||
|
details_json: Mapped[dict | None] = mapped_column(JSON, nullable=True)
|
||||||
|
artifact_path: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||||
|
artifact_checksum_sha256: Mapped[str | None] = mapped_column(String(64), nullable=True)
|
||||||
|
status: Mapped[str] = mapped_column(
|
||||||
|
String(32), nullable=False, default="quarantined", server_default="quarantined"
|
||||||
|
)
|
||||||
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), server_default=func.now())
|
||||||
|
resolved_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||||
|
resolved_by: Mapped[str | None] = mapped_column(String(120), nullable=True)
|
||||||
|
|
||||||
|
dataset: Mapped[Dataset | None] = relationship("Dataset", back_populates="quarantines")
|
||||||
|
dataset_version: Mapped[DatasetVersion | None] = relationship("DatasetVersion", back_populates="quarantines")
|
||||||
|
source_snapshot: Mapped[SourceSnapshot | None] = relationship("SourceSnapshot", back_populates="quarantines")
|
||||||
|
|
||||||
|
|
||||||
class VectorFeature(Base):
|
class VectorFeature(Base):
|
||||||
|
|||||||
@@ -32,6 +32,14 @@ from .source_catalog import (
|
|||||||
SourceCatalogProbeReport,
|
SourceCatalogProbeReport,
|
||||||
SourceCatalogProbeSummary,
|
SourceCatalogProbeSummary,
|
||||||
)
|
)
|
||||||
|
from .source_registry import (
|
||||||
|
DatasetProvenanceRead,
|
||||||
|
DatasetLineageEdgeRead,
|
||||||
|
DatasetQuarantineRead,
|
||||||
|
SourceRegistryDetailRead,
|
||||||
|
SourceRegistryRead,
|
||||||
|
SourceSnapshotRead,
|
||||||
|
)
|
||||||
from .grb_refresh import GrbRefreshLayerPlan, GrbRefreshPlan, GrbRefreshPlanSummary
|
from .grb_refresh import GrbRefreshLayerPlan, GrbRefreshPlan, GrbRefreshPlanSummary
|
||||||
from .grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead
|
from .grb import GrbAcquireRequest, GrbAcquisitionResult, GrbProductRead
|
||||||
from .official_vector import (
|
from .official_vector import (
|
||||||
@@ -207,6 +215,12 @@ __all__ = [
|
|||||||
"SourceCatalogProbeItem",
|
"SourceCatalogProbeItem",
|
||||||
"SourceCatalogProbeReport",
|
"SourceCatalogProbeReport",
|
||||||
"SourceCatalogProbeSummary",
|
"SourceCatalogProbeSummary",
|
||||||
|
"SourceRegistryRead",
|
||||||
|
"SourceRegistryDetailRead",
|
||||||
|
"SourceSnapshotRead",
|
||||||
|
"DatasetLineageEdgeRead",
|
||||||
|
"DatasetQuarantineRead",
|
||||||
|
"DatasetProvenanceRead",
|
||||||
"GrbRefreshLayerPlan",
|
"GrbRefreshLayerPlan",
|
||||||
"GrbRefreshPlan",
|
"GrbRefreshPlan",
|
||||||
"GrbRefreshPlanSummary",
|
"GrbRefreshPlanSummary",
|
||||||
@@ -271,6 +285,10 @@ __all__ = [
|
|||||||
"FloodHazardSelectionSummary",
|
"FloodHazardSelectionSummary",
|
||||||
"BathymetryProfileAcquireRequest",
|
"BathymetryProfileAcquireRequest",
|
||||||
"BathymetryProfileAcquisitionResult",
|
"BathymetryProfileAcquisitionResult",
|
||||||
|
"BathymetryRasterMetric",
|
||||||
|
"BathymetryRasterSelectionRequest",
|
||||||
|
"BathymetryRasterSelectionResponse",
|
||||||
|
"BathymetryRasterSelectionSummary",
|
||||||
"BathymetryPartitionFinalizeRequest",
|
"BathymetryPartitionFinalizeRequest",
|
||||||
"BathymetryPartitionFinalizationResult",
|
"BathymetryPartitionFinalizationResult",
|
||||||
"BathymetrySourceProbeRead",
|
"BathymetrySourceProbeRead",
|
||||||
|
|||||||
@@ -35,6 +35,16 @@ class DatasetCreateResponse(BaseModel):
|
|||||||
reference_layer_name: str | None = None
|
reference_layer_name: str | None = None
|
||||||
source_metadata: dict | None = None
|
source_metadata: dict | None = None
|
||||||
provenance_metadata: dict | None = None
|
provenance_metadata: dict | None = None
|
||||||
|
ingest_key: str | None = None
|
||||||
|
source_registry_id: UUID | None = None
|
||||||
|
source_snapshot_id: UUID | None = None
|
||||||
|
data_contract_key: str | None = None
|
||||||
|
data_contract_version: str | None = None
|
||||||
|
validation_status: str | None = None
|
||||||
|
validation_report_json: dict | None = None
|
||||||
|
provenance_status: str | None = None
|
||||||
|
lineage_status: str | None = None
|
||||||
|
quarantine_status: str | None = None
|
||||||
imported_at: datetime | None = None
|
imported_at: datetime | None = None
|
||||||
temporal_series_key: str | None = None
|
temporal_series_key: str | None = None
|
||||||
observed_at: datetime | None = None
|
observed_at: datetime | None = None
|
||||||
@@ -97,6 +107,15 @@ class DatasetVersionRead(BaseModel):
|
|||||||
checksum_sha256: str | None = None
|
checksum_sha256: str | None = None
|
||||||
source_metadata: dict | None = None
|
source_metadata: dict | None = None
|
||||||
provenance_metadata: dict | None = None
|
provenance_metadata: dict | None = None
|
||||||
|
ingest_key: str | None = None
|
||||||
|
source_registry_id: UUID | None = None
|
||||||
|
source_snapshot_id: UUID | None = None
|
||||||
|
data_contract_key: str | None = None
|
||||||
|
data_contract_version: str | None = None
|
||||||
|
validation_status: str | None = None
|
||||||
|
validation_report_json: dict | None = None
|
||||||
|
provenance_status: str | None = None
|
||||||
|
lineage_status: str | None = None
|
||||||
created_at: datetime | None = None
|
created_at: datetime | None = None
|
||||||
|
|
||||||
model_config = {"from_attributes": True}
|
model_config = {"from_attributes": True}
|
||||||
|
|||||||
@@ -144,6 +144,8 @@ class YoloPreflightChecks(BaseModel):
|
|||||||
accelerator_ready: bool | None = None
|
accelerator_ready: bool | None = None
|
||||||
model_path_set: bool | None = None
|
model_path_set: bool | None = None
|
||||||
model_file_exists: bool | None = None
|
model_file_exists: bool | None = None
|
||||||
|
model_provenance_manifest_path: str | None = None
|
||||||
|
model_provenance_valid: bool | None = None
|
||||||
model_load_requested: bool
|
model_load_requested: bool
|
||||||
model_load_ok: bool | None = None
|
model_load_ok: bool | None = None
|
||||||
manifest_path_set: bool | None = None
|
manifest_path_set: bool | None = None
|
||||||
|
|||||||
@@ -0,0 +1,120 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from pydantic import BaseModel, Field
|
||||||
|
|
||||||
|
|
||||||
|
class SourceRegistryRead(BaseModel):
|
||||||
|
"""Read-only, server-owned source-authority definition."""
|
||||||
|
|
||||||
|
id: UUID
|
||||||
|
source_key: str
|
||||||
|
display_name: str
|
||||||
|
classification: str
|
||||||
|
authority_name: str
|
||||||
|
authority_scope_json: dict
|
||||||
|
provider_adapter_key: str | None = None
|
||||||
|
source_url: str | None = None
|
||||||
|
license_name: str
|
||||||
|
license_url: str | None = None
|
||||||
|
usage_restrictions: str
|
||||||
|
default_crs: str
|
||||||
|
default_units: str
|
||||||
|
spatial_resolution_json: dict
|
||||||
|
temporal_coverage_json: dict
|
||||||
|
geographic_coverage_json: dict
|
||||||
|
expected_geometry_types_json: list
|
||||||
|
expected_attributes_json: dict
|
||||||
|
usage_policy_json: dict
|
||||||
|
freshness_status: str
|
||||||
|
ingest_status: str
|
||||||
|
known_limitations_json: list
|
||||||
|
registry_metadata_json: dict
|
||||||
|
created_at: datetime | None = None
|
||||||
|
updated_at: datetime | None = None
|
||||||
|
snapshot_count: int = 0
|
||||||
|
|
||||||
|
model_config = {"from_attributes": True}
|
||||||
|
|
||||||
|
|
||||||
|
class SourceSnapshotRead(BaseModel):
|
||||||
|
"""Immutable version/snapshot evidence attached to an imported dataset."""
|
||||||
|
|
||||||
|
id: UUID
|
||||||
|
source_registry_id: UUID
|
||||||
|
snapshot_key: str
|
||||||
|
source_version: str | None = None
|
||||||
|
snapshot_at: datetime | None = None
|
||||||
|
fetched_at: datetime | None = None
|
||||||
|
source_url: str | None = None
|
||||||
|
checksum_sha256: str | None = None
|
||||||
|
crs: str | None = None
|
||||||
|
units: str | None = None
|
||||||
|
spatial_resolution_json: dict
|
||||||
|
temporal_coverage_json: dict
|
||||||
|
geographic_coverage_json: dict
|
||||||
|
observed_schema_json: dict
|
||||||
|
freshness_status: str
|
||||||
|
ingest_status: str
|
||||||
|
known_limitations_json: list
|
||||||
|
snapshot_metadata_json: dict
|
||||||
|
created_at: datetime | None = None
|
||||||
|
|
||||||
|
model_config = {"from_attributes": True}
|
||||||
|
|
||||||
|
|
||||||
|
class SourceRegistryDetailRead(BaseModel):
|
||||||
|
source: SourceRegistryRead
|
||||||
|
snapshots: list[SourceSnapshotRead]
|
||||||
|
|
||||||
|
|
||||||
|
class DatasetLineageEdgeRead(BaseModel):
|
||||||
|
id: UUID
|
||||||
|
parent_dataset_id: UUID
|
||||||
|
child_dataset_id: UUID
|
||||||
|
parent_dataset_version_id: UUID | None = None
|
||||||
|
child_dataset_version_id: UUID | None = None
|
||||||
|
relation_type: str
|
||||||
|
transformation_name: str
|
||||||
|
transformation_version: str | None = None
|
||||||
|
parameters_json: dict | None = None
|
||||||
|
input_checksum_sha256: str | None = None
|
||||||
|
output_checksum_sha256: str | None = None
|
||||||
|
created_at: datetime | None = None
|
||||||
|
|
||||||
|
model_config = {"from_attributes": True}
|
||||||
|
|
||||||
|
|
||||||
|
class DatasetQuarantineRead(BaseModel):
|
||||||
|
id: UUID
|
||||||
|
dataset_id: UUID | None = None
|
||||||
|
dataset_version_id: UUID | None = None
|
||||||
|
source_snapshot_id: UUID | None = None
|
||||||
|
stage: str
|
||||||
|
reason_code: str
|
||||||
|
details_json: dict | None = None
|
||||||
|
artifact_path: str | None = None
|
||||||
|
artifact_checksum_sha256: str | None = None
|
||||||
|
status: str
|
||||||
|
created_at: datetime | None = None
|
||||||
|
resolved_at: datetime | None = None
|
||||||
|
resolved_by: str | None = None
|
||||||
|
|
||||||
|
model_config = {"from_attributes": True}
|
||||||
|
|
||||||
|
|
||||||
|
class DatasetProvenanceRead(BaseModel):
|
||||||
|
dataset_id: UUID
|
||||||
|
source: SourceRegistryRead | None = None
|
||||||
|
snapshot: SourceSnapshotRead | None = None
|
||||||
|
data_contract_key: str | None = None
|
||||||
|
data_contract_version: str | None = None
|
||||||
|
validation_status: str | None = None
|
||||||
|
validation_report_json: dict | None = None
|
||||||
|
provenance_status: str | None = None
|
||||||
|
lineage_status: str | None = None
|
||||||
|
quarantine_status: str | None = None
|
||||||
|
lineage: list[DatasetLineageEdgeRead] = Field(default_factory=list)
|
||||||
|
quarantines: list[DatasetQuarantineRead] = Field(default_factory=list)
|
||||||
@@ -12,6 +12,7 @@ from sqlalchemy.orm import Session
|
|||||||
|
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.models import Area, Dataset, Project
|
from app.models import Area, Dataset, Project
|
||||||
|
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||||
from app.schemas.coverage import (
|
from app.schemas.coverage import (
|
||||||
CoverageBBox,
|
CoverageBBox,
|
||||||
CoverageCatalogResponse,
|
CoverageCatalogResponse,
|
||||||
@@ -463,6 +464,10 @@ class CoverageRegistryService:
|
|||||||
for dataset in datasets:
|
for dataset in datasets:
|
||||||
if dataset.status != "ready" or dataset.source_name not in definition.materialized_source_names:
|
if dataset.status != "ready" or dataset.source_name not in definition.materialized_source_names:
|
||||||
continue
|
continue
|
||||||
|
# A source-name claim alone must not cause an unsafe artifact to
|
||||||
|
# appear as operational authoritative coverage.
|
||||||
|
if not DatasetConsumptionGate.eligible_for_authoritative_coverage(dataset):
|
||||||
|
continue
|
||||||
layer_names = definition.materialized_layer_names
|
layer_names = definition.materialized_layer_names
|
||||||
if definition.contract.source_name == "digitaal_vlaanderen":
|
if definition.contract.source_name == "digitaal_vlaanderen":
|
||||||
theme_sources = FLANDERS_THEME_DATASETS.get(theme, {})
|
theme_sources = FLANDERS_THEME_DATASETS.get(theme, {})
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,160 @@
|
|||||||
|
"""Fail-closed quarantine decisions for validated data assets.
|
||||||
|
|
||||||
|
Persistence is intentionally delegated to the caller's transaction. This
|
||||||
|
module derives stable decisions from immutable validation reports and blocks a
|
||||||
|
quarantined or failed asset from training, production inference and derived
|
||||||
|
processing until an explicit, separately persisted release action exists.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from enum import StrEnum
|
||||||
|
from hashlib import sha256
|
||||||
|
from typing import Any
|
||||||
|
import json
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.services.data_contract_validation import QuarantineStatus, ValidationReport, ValidationStatus
|
||||||
|
|
||||||
|
|
||||||
|
class AssetUse(StrEnum):
|
||||||
|
TRAINING = "training"
|
||||||
|
PRODUCTION_INFERENCE = "production_inference"
|
||||||
|
DERIVED_PROCESSING = "derived_processing"
|
||||||
|
EXPORT = "export"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class QuarantineDecision:
|
||||||
|
"""A deterministic, auditable quarantine decision for one report."""
|
||||||
|
|
||||||
|
asset_id: str
|
||||||
|
data_contract_key: str
|
||||||
|
data_contract_version: str
|
||||||
|
validation_report_sha256: str
|
||||||
|
validation_status: ValidationStatus
|
||||||
|
quarantine_status: QuarantineStatus
|
||||||
|
reason_codes: tuple[str, ...]
|
||||||
|
idempotency_key: str
|
||||||
|
requires_explicit_release: bool = False
|
||||||
|
|
||||||
|
@property
|
||||||
|
def eligible_for_use(self) -> bool:
|
||||||
|
return self.validation_status == ValidationStatus.PASSED and self.quarantine_status == QuarantineStatus.NOT_QUARANTINED
|
||||||
|
|
||||||
|
def to_dict(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"asset_id": self.asset_id,
|
||||||
|
"data_contract_key": self.data_contract_key,
|
||||||
|
"data_contract_version": self.data_contract_version,
|
||||||
|
"validation_report_sha256": self.validation_report_sha256,
|
||||||
|
"validation_status": self.validation_status.value,
|
||||||
|
"quarantine_status": self.quarantine_status.value,
|
||||||
|
"reason_codes": list(self.reason_codes),
|
||||||
|
"idempotency_key": self.idempotency_key,
|
||||||
|
"requires_explicit_release": self.requires_explicit_release,
|
||||||
|
"eligible_for_use": self.eligible_for_use,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class DataQuarantineService:
|
||||||
|
"""Derive and enforce safe use decisions from validation results."""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def decide(
|
||||||
|
report: ValidationReport,
|
||||||
|
*,
|
||||||
|
previous: QuarantineDecision | None = None,
|
||||||
|
) -> QuarantineDecision:
|
||||||
|
"""Create a stable decision without silently releasing old quarantines.
|
||||||
|
|
||||||
|
A fresh passing validation report can be persisted as a new validated
|
||||||
|
version by the import transaction. It cannot automatically release an
|
||||||
|
existing quarantined record: the caller must explicitly record that
|
||||||
|
reviewed state transition against the new report/version.
|
||||||
|
"""
|
||||||
|
|
||||||
|
is_quarantined = report.validation_status == ValidationStatus.FAILED or report.quarantine_status == QuarantineStatus.QUARANTINED
|
||||||
|
failure_codes = tuple(
|
||||||
|
sorted(
|
||||||
|
{
|
||||||
|
issue.code
|
||||||
|
for issue in report.issues
|
||||||
|
if issue.severity.value == "error" or is_quarantined
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
requires_explicit_release = False
|
||||||
|
reason_codes = failure_codes
|
||||||
|
status = QuarantineStatus.QUARANTINED if is_quarantined else QuarantineStatus.NOT_QUARANTINED
|
||||||
|
|
||||||
|
if previous is not None and previous.quarantine_status == QuarantineStatus.QUARANTINED and not is_quarantined:
|
||||||
|
status = QuarantineStatus.QUARANTINED
|
||||||
|
requires_explicit_release = True
|
||||||
|
reason_codes = ("QUARANTINE_RELEASE_REQUIRES_EXPLICIT_PERSISTENCE",)
|
||||||
|
|
||||||
|
idempotency_key = _decision_key(
|
||||||
|
asset_id=report.asset_id,
|
||||||
|
contract_key=report.data_contract_key,
|
||||||
|
contract_version=report.data_contract_version,
|
||||||
|
report_sha256=report.report_sha256,
|
||||||
|
quarantine_status=status,
|
||||||
|
reason_codes=reason_codes,
|
||||||
|
requires_explicit_release=requires_explicit_release,
|
||||||
|
)
|
||||||
|
return QuarantineDecision(
|
||||||
|
asset_id=report.asset_id,
|
||||||
|
data_contract_key=report.data_contract_key,
|
||||||
|
data_contract_version=report.data_contract_version,
|
||||||
|
validation_report_sha256=report.report_sha256,
|
||||||
|
validation_status=report.validation_status,
|
||||||
|
quarantine_status=status,
|
||||||
|
reason_codes=reason_codes,
|
||||||
|
idempotency_key=idempotency_key,
|
||||||
|
requires_explicit_release=requires_explicit_release,
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def require_eligible(decision: QuarantineDecision, *, use: AssetUse) -> None:
|
||||||
|
"""Raise a typed error before a non-eligible artifact reaches a pipeline."""
|
||||||
|
|
||||||
|
if decision.eligible_for_use:
|
||||||
|
return
|
||||||
|
raise AppError(
|
||||||
|
code="DATASET_QUARANTINED",
|
||||||
|
message="Dataset is quarantined or failed validation and cannot enter this pipeline.",
|
||||||
|
status_code=409,
|
||||||
|
details={
|
||||||
|
"asset_id": decision.asset_id,
|
||||||
|
"use": use.value,
|
||||||
|
"quarantine_status": decision.quarantine_status.value,
|
||||||
|
"validation_status": decision.validation_status.value,
|
||||||
|
"reason_codes": list(decision.reason_codes),
|
||||||
|
"validation_report_sha256": decision.validation_report_sha256,
|
||||||
|
"idempotency_key": decision.idempotency_key,
|
||||||
|
"requires_explicit_release": decision.requires_explicit_release,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _decision_key(
|
||||||
|
*,
|
||||||
|
asset_id: str,
|
||||||
|
contract_key: str,
|
||||||
|
contract_version: str,
|
||||||
|
report_sha256: str,
|
||||||
|
quarantine_status: QuarantineStatus,
|
||||||
|
reason_codes: tuple[str, ...],
|
||||||
|
requires_explicit_release: bool,
|
||||||
|
) -> str:
|
||||||
|
payload = {
|
||||||
|
"asset_id": asset_id,
|
||||||
|
"contract_key": contract_key,
|
||||||
|
"contract_version": contract_version,
|
||||||
|
"report_sha256": report_sha256,
|
||||||
|
"quarantine_status": quarantine_status.value,
|
||||||
|
"reason_codes": list(reason_codes),
|
||||||
|
"requires_explicit_release": requires_explicit_release,
|
||||||
|
}
|
||||||
|
return sha256(json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")).hexdigest()
|
||||||
@@ -0,0 +1,440 @@
|
|||||||
|
"""Fail-closed provenance gates at data-consumption boundaries.
|
||||||
|
|
||||||
|
Import validation protects newly staged assets, but a persisted record can
|
||||||
|
subsequently become quarantined or have its provenance marked incomplete. The
|
||||||
|
callers of this service therefore re-check the durable Dataset state directly
|
||||||
|
before production inference, QA, derived processing, export, or authoritative
|
||||||
|
coverage reporting.
|
||||||
|
|
||||||
|
The only legacy relaxation is deliberately narrow: an *explicitly tagged*
|
||||||
|
fixture with no Phase-2 state can be used for fixture QA. A caller-provided
|
||||||
|
``fixture_mode`` flag alone never creates that trust claim. Fixture data can
|
||||||
|
never become a production inference, derived-processing, export or
|
||||||
|
authoritative-coverage input, and it never relaxes a recorded failed,
|
||||||
|
incomplete, or quarantined state.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from dataclasses import dataclass
|
||||||
|
import re
|
||||||
|
from typing import Any, Literal
|
||||||
|
|
||||||
|
from sqlalchemy import inspect as sa_inspect
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.models import Dataset
|
||||||
|
|
||||||
|
|
||||||
|
DatasetConsumptionPurpose = Literal[
|
||||||
|
"production_inference",
|
||||||
|
"quality_assessment",
|
||||||
|
"reference_validation",
|
||||||
|
"derived_processing",
|
||||||
|
"authoritative_coverage",
|
||||||
|
"export",
|
||||||
|
]
|
||||||
|
|
||||||
|
_FIXTURE_SOURCE_KEYS = {"fixture", "test", "test_fixture", "test-fixture", "unit-test-fixture"}
|
||||||
|
_UNTRUSTED_SOURCE_KEYS = {"manual", "fixture", "experimental", "legacy_unknown"}
|
||||||
|
_CHECKSUM_SHA256 = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE)
|
||||||
|
_CONSUMABLE_SNAPSHOT_FRESHNESS = {"current", "not_applicable"}
|
||||||
|
_VALID_PURPOSES = {
|
||||||
|
"production_inference",
|
||||||
|
"quality_assessment",
|
||||||
|
"reference_validation",
|
||||||
|
"derived_processing",
|
||||||
|
"authoritative_coverage",
|
||||||
|
"export",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class DatasetConsumptionDecision:
|
||||||
|
"""Auditable decision returned by the consumption gate."""
|
||||||
|
|
||||||
|
eligible: bool
|
||||||
|
purpose: DatasetConsumptionPurpose
|
||||||
|
fixture_legacy_exception: bool
|
||||||
|
reasons: tuple[str, ...]
|
||||||
|
evidence: dict[str, Any]
|
||||||
|
|
||||||
|
|
||||||
|
class DatasetConsumptionGate:
|
||||||
|
"""Evaluate durable provenance before a Dataset is consumed downstream."""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _value(dataset: Any, field: str, default: Any = None) -> Any:
|
||||||
|
if isinstance(dataset, Mapping):
|
||||||
|
return dataset.get(field, default)
|
||||||
|
return getattr(dataset, field, default)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _mapping(value: Any) -> dict[str, Any]:
|
||||||
|
return dict(value) if isinstance(value, Mapping) else {}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _normalise(cls, value: Any) -> str:
|
||||||
|
return str(value or "").strip().lower()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def is_explicit_fixture(cls, dataset: Any) -> bool:
|
||||||
|
source = cls._normalise(cls._value(dataset, "source"))
|
||||||
|
source_name = cls._normalise(cls._value(dataset, "source_name"))
|
||||||
|
metadata = cls._mapping(cls._value(dataset, "metadata_json"))
|
||||||
|
source_metadata = cls._mapping(cls._value(dataset, "source_metadata"))
|
||||||
|
provenance = cls._mapping(cls._value(dataset, "provenance_metadata"))
|
||||||
|
return bool(
|
||||||
|
source in _FIXTURE_SOURCE_KEYS
|
||||||
|
or source_name in _FIXTURE_SOURCE_KEYS
|
||||||
|
or metadata.get("fixture") is True
|
||||||
|
or metadata.get("fixture_mode") is True
|
||||||
|
or source_metadata.get("fixture") is True
|
||||||
|
or source_metadata.get("fixture_mode") is True
|
||||||
|
or provenance.get("fixture") is True
|
||||||
|
or provenance.get("fixture_mode") is True
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _phase2_state_is_absent(cls, dataset: Any) -> bool:
|
||||||
|
fields = (
|
||||||
|
"data_contract_key",
|
||||||
|
"data_contract_version",
|
||||||
|
"validation_status",
|
||||||
|
"provenance_status",
|
||||||
|
"lineage_status",
|
||||||
|
"quarantine_status",
|
||||||
|
"source_registry_id",
|
||||||
|
"source_snapshot_id",
|
||||||
|
)
|
||||||
|
return all(cls._value(dataset, field) in {None, ""} for field in fields)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _is_transient_orm_dataset(dataset: Any) -> bool:
|
||||||
|
"""Recognize only unpersisted ORM fixtures, never database rows."""
|
||||||
|
|
||||||
|
if not isinstance(dataset, Dataset):
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
return bool(sa_inspect(dataset).transient)
|
||||||
|
except Exception: # pragma: no cover - defensive for unusual test doubles
|
||||||
|
return False
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _evidence(
|
||||||
|
cls,
|
||||||
|
dataset: Any,
|
||||||
|
purpose: DatasetConsumptionPurpose,
|
||||||
|
reference_task: str | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
source_registry = cls._value(dataset, "source_registry")
|
||||||
|
source_snapshot = cls._value(dataset, "source_snapshot")
|
||||||
|
source_policy = cls._mapping(cls._value(source_registry, "usage_policy_json"))
|
||||||
|
validation_authority = cls._mapping(source_policy.get("validation_authority"))
|
||||||
|
reference_approvals = cls._mapping(source_policy.get("reference_validation_approvals"))
|
||||||
|
return {
|
||||||
|
"dataset_id": str(cls._value(dataset, "id") or ""),
|
||||||
|
"purpose": purpose,
|
||||||
|
"dataset_status": cls._normalise(cls._value(dataset, "status")),
|
||||||
|
"source": cls._normalise(cls._value(dataset, "source")),
|
||||||
|
"source_name": cls._normalise(cls._value(dataset, "source_name")),
|
||||||
|
"data_contract_key": cls._value(dataset, "data_contract_key"),
|
||||||
|
"data_contract_version": cls._value(dataset, "data_contract_version"),
|
||||||
|
"validation_status": cls._normalise(cls._value(dataset, "validation_status")),
|
||||||
|
"provenance_status": cls._normalise(cls._value(dataset, "provenance_status")),
|
||||||
|
"lineage_status": cls._normalise(cls._value(dataset, "lineage_status")),
|
||||||
|
"quarantine_status": cls._normalise(cls._value(dataset, "quarantine_status")),
|
||||||
|
"checksum_sha256": cls._value(dataset, "checksum_sha256"),
|
||||||
|
"source_registry_id": str(cls._value(dataset, "source_registry_id") or ""),
|
||||||
|
"source_snapshot_id": str(cls._value(dataset, "source_snapshot_id") or ""),
|
||||||
|
"source_classification": cls._normalise(cls._value(source_registry, "classification")),
|
||||||
|
"source_key": cls._normalise(cls._value(source_registry, "source_key")),
|
||||||
|
"source_ground_truth_allowed": source_policy.get("ground_truth_allowed") is True,
|
||||||
|
"source_validation_authority": validation_authority,
|
||||||
|
"source_reference_validation_approvals": reference_approvals,
|
||||||
|
"source_authority_scope": cls._mapping(cls._value(source_registry, "authority_scope_json")),
|
||||||
|
"reference_task": cls._normalise(reference_task),
|
||||||
|
"snapshot_source_registry_id": str(cls._value(source_snapshot, "source_registry_id") or ""),
|
||||||
|
"snapshot_ingest_status": cls._normalise(cls._value(source_snapshot, "ingest_status")),
|
||||||
|
"snapshot_freshness_status": cls._normalise(cls._value(source_snapshot, "freshness_status")),
|
||||||
|
"snapshot_checksum_sha256": cls._value(source_snapshot, "checksum_sha256"),
|
||||||
|
}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def evaluate(
|
||||||
|
cls,
|
||||||
|
dataset: Any,
|
||||||
|
*,
|
||||||
|
purpose: DatasetConsumptionPurpose,
|
||||||
|
fixture_mode: bool = False,
|
||||||
|
reference_task: str | None = None,
|
||||||
|
) -> DatasetConsumptionDecision:
|
||||||
|
"""Return a stable decision without mutating the Dataset.
|
||||||
|
|
||||||
|
``fixture_mode`` can be supplied only by an explicitly fixture-only
|
||||||
|
caller. It is evidence for a QA fixture path, never a relaxation for
|
||||||
|
a production boundary or an explicit unsafe state.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if purpose not in _VALID_PURPOSES:
|
||||||
|
raise ValueError(f"Unsupported dataset-consumption purpose: {purpose}")
|
||||||
|
|
||||||
|
evidence = cls._evidence(dataset, purpose, reference_task)
|
||||||
|
reasons: list[str] = []
|
||||||
|
explicit_fixture = cls.is_explicit_fixture(dataset)
|
||||||
|
phase2_absent = cls._phase2_state_is_absent(dataset)
|
||||||
|
|
||||||
|
# These are irrevocable safety states. They are checked before a
|
||||||
|
# fixture exception, so fixture rows cannot hide a bad recorded state.
|
||||||
|
if evidence["dataset_status"] in {"failed", "quarantined"}:
|
||||||
|
reasons.append("dataset_status_unsafe")
|
||||||
|
if evidence["quarantine_status"] == "quarantined":
|
||||||
|
reasons.append("dataset_quarantined")
|
||||||
|
if evidence["validation_status"] == "failed":
|
||||||
|
reasons.append("validation_failed")
|
||||||
|
if evidence["provenance_status"] == "incomplete":
|
||||||
|
reasons.append("provenance_incomplete")
|
||||||
|
if evidence["lineage_status"] == "incomplete":
|
||||||
|
reasons.append("lineage_incomplete")
|
||||||
|
if evidence["snapshot_ingest_status"] in {"failed", "quarantined"}:
|
||||||
|
reasons.append("source_snapshot_unsafe")
|
||||||
|
# A source family can be authoritative while an individual snapshot
|
||||||
|
# remains too old or insufficiently described to trust. Historical
|
||||||
|
# data that is intentionally valid needs an explicit
|
||||||
|
# ``not_applicable`` contract policy; an omitted, due or stale status
|
||||||
|
# cannot silently enter a production boundary.
|
||||||
|
if evidence["source_snapshot_id"] and evidence["snapshot_freshness_status"] not in _CONSUMABLE_SNAPSHOT_FRESHNESS:
|
||||||
|
reasons.append("source_snapshot_freshness_not_eligible")
|
||||||
|
|
||||||
|
if reasons:
|
||||||
|
return DatasetConsumptionDecision(
|
||||||
|
eligible=False,
|
||||||
|
purpose=purpose,
|
||||||
|
fixture_legacy_exception=False,
|
||||||
|
reasons=tuple(sorted(set(reasons))),
|
||||||
|
evidence=evidence,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Fixtures are evidence for tests and QA only. They cannot become
|
||||||
|
# production inference, derived processing or export inputs merely by
|
||||||
|
# presenting a fixture flag at a public service boundary.
|
||||||
|
if phase2_absent and explicit_fixture:
|
||||||
|
if purpose == "quality_assessment":
|
||||||
|
return DatasetConsumptionDecision(
|
||||||
|
eligible=True,
|
||||||
|
purpose=purpose,
|
||||||
|
fixture_legacy_exception=True,
|
||||||
|
reasons=(),
|
||||||
|
evidence=evidence,
|
||||||
|
)
|
||||||
|
if purpose == "authoritative_coverage":
|
||||||
|
reasons.append("fixture_not_authoritative_coverage")
|
||||||
|
else:
|
||||||
|
reasons.append("fixture_qa_only")
|
||||||
|
elif phase2_absent and fixture_mode:
|
||||||
|
# `fixture_mode` is a caller flag, not a source trust claim. A
|
||||||
|
# manual/unknown production dataset must never self-designate as a
|
||||||
|
# fixture merely by supplying this parameter.
|
||||||
|
reasons.append("fixture_source_required")
|
||||||
|
|
||||||
|
# Existing service tests construct transient SQLAlchemy Dataset objects
|
||||||
|
# directly rather than retrieving a persisted row. A production
|
||||||
|
# `db.get()` result is persistent and never enters this branch. This
|
||||||
|
# compatibility path is intentionally unavailable to coverage, where
|
||||||
|
# a fixture must never appear authoritative.
|
||||||
|
if (
|
||||||
|
phase2_absent
|
||||||
|
and not reasons
|
||||||
|
and cls._is_transient_orm_dataset(dataset)
|
||||||
|
and purpose == "quality_assessment"
|
||||||
|
):
|
||||||
|
return DatasetConsumptionDecision(
|
||||||
|
eligible=True,
|
||||||
|
purpose=purpose,
|
||||||
|
fixture_legacy_exception=True,
|
||||||
|
reasons=(),
|
||||||
|
evidence=evidence,
|
||||||
|
)
|
||||||
|
|
||||||
|
# QA unit tests intentionally use projection objects
|
||||||
|
# instead of persisted ORM Datasets. Those projections cannot enter an
|
||||||
|
# application API boundary; keep the exception isolated to read-only
|
||||||
|
# candidate verification. Inference, reference validation, derived
|
||||||
|
# processing, export and coverage never accept a projection.
|
||||||
|
if phase2_absent and not reasons and not isinstance(dataset, Dataset) and purpose == "quality_assessment":
|
||||||
|
return DatasetConsumptionDecision(
|
||||||
|
eligible=True,
|
||||||
|
purpose=purpose,
|
||||||
|
fixture_legacy_exception=True,
|
||||||
|
reasons=(),
|
||||||
|
evidence=evidence,
|
||||||
|
)
|
||||||
|
|
||||||
|
if phase2_absent:
|
||||||
|
reasons.append("phase2_provenance_missing")
|
||||||
|
if evidence["dataset_status"] != "ready":
|
||||||
|
reasons.append("dataset_not_ready")
|
||||||
|
if evidence["validation_status"] != "passed":
|
||||||
|
reasons.append("validation_not_passed")
|
||||||
|
if evidence["provenance_status"] != "complete":
|
||||||
|
reasons.append("provenance_not_complete")
|
||||||
|
if evidence["lineage_status"] not in {"complete", "not_applicable"}:
|
||||||
|
reasons.append("lineage_not_complete")
|
||||||
|
if evidence["quarantine_status"] != "not_quarantined":
|
||||||
|
reasons.append("quarantine_status_not_clear")
|
||||||
|
if not evidence["data_contract_key"] or not evidence["data_contract_version"]:
|
||||||
|
reasons.append("data_contract_not_versioned")
|
||||||
|
if not _CHECKSUM_SHA256.fullmatch(str(evidence["checksum_sha256"] or "")):
|
||||||
|
reasons.append("dataset_checksum_invalid")
|
||||||
|
if not evidence["source_registry_id"]:
|
||||||
|
reasons.append("source_registry_missing")
|
||||||
|
if not evidence["source_snapshot_id"]:
|
||||||
|
reasons.append("source_snapshot_missing")
|
||||||
|
if evidence["source_registry_id"] and not evidence["source_classification"]:
|
||||||
|
reasons.append("source_registry_unresolved")
|
||||||
|
if evidence["source_snapshot_id"] and evidence["snapshot_ingest_status"] != "ingested":
|
||||||
|
reasons.append("source_snapshot_not_ingested")
|
||||||
|
if evidence["source_snapshot_id"] and not _CHECKSUM_SHA256.fullmatch(
|
||||||
|
str(evidence["snapshot_checksum_sha256"] or "")
|
||||||
|
):
|
||||||
|
reasons.append("source_snapshot_checksum_invalid")
|
||||||
|
if (
|
||||||
|
evidence["source_registry_id"]
|
||||||
|
and evidence["source_snapshot_id"]
|
||||||
|
and evidence["snapshot_source_registry_id"]
|
||||||
|
and evidence["source_registry_id"] != evidence["snapshot_source_registry_id"]
|
||||||
|
):
|
||||||
|
reasons.append("source_snapshot_registry_mismatch")
|
||||||
|
if (
|
||||||
|
_CHECKSUM_SHA256.fullmatch(str(evidence["checksum_sha256"] or ""))
|
||||||
|
and _CHECKSUM_SHA256.fullmatch(str(evidence["snapshot_checksum_sha256"] or ""))
|
||||||
|
and str(evidence["checksum_sha256"]).lower() != str(evidence["snapshot_checksum_sha256"]).lower()
|
||||||
|
):
|
||||||
|
reasons.append("source_snapshot_checksum_mismatch")
|
||||||
|
|
||||||
|
if evidence["source_key"] and evidence["source_name"] and evidence["source_key"] != evidence["source_name"]:
|
||||||
|
reasons.append("source_registry_identity_mismatch")
|
||||||
|
|
||||||
|
# Contextual, corroborative, authoritative and properly derived
|
||||||
|
# sources can serve their declared non-ground-truth roles once the
|
||||||
|
# full governed contract passes. Experimental/manual sources cannot
|
||||||
|
# cross a production boundary; only an explicitly marked fixture may
|
||||||
|
# participate in candidate QA.
|
||||||
|
experimental_source = (
|
||||||
|
evidence["source_classification"] == "experimental"
|
||||||
|
or evidence["source_key"] in _UNTRUSTED_SOURCE_KEYS
|
||||||
|
)
|
||||||
|
if experimental_source:
|
||||||
|
if purpose == "quality_assessment" and explicit_fixture:
|
||||||
|
pass
|
||||||
|
elif purpose == "quality_assessment":
|
||||||
|
reasons.append("experimental_source_requires_fixture_qa")
|
||||||
|
else:
|
||||||
|
reasons.append("experimental_source_not_allowed_for_purpose")
|
||||||
|
|
||||||
|
if purpose == "authoritative_coverage":
|
||||||
|
if evidence["source_classification"] != "authoritative":
|
||||||
|
reasons.append("coverage_source_not_authoritative")
|
||||||
|
if evidence["source_key"] and evidence["source_name"] and evidence["source_key"] != evidence["source_name"]:
|
||||||
|
reasons.append("coverage_source_identity_mismatch")
|
||||||
|
|
||||||
|
if purpose == "reference_validation":
|
||||||
|
if cls._normalise(cls._value(dataset, "dataset_role")) != "reference":
|
||||||
|
reasons.append("reference_dataset_role_required")
|
||||||
|
if evidence["source_classification"] != "authoritative":
|
||||||
|
reasons.append("reference_source_not_authoritative")
|
||||||
|
if evidence["source_ground_truth_allowed"] is not True:
|
||||||
|
reasons.append("reference_source_not_ground_truth_allowed")
|
||||||
|
if not evidence["reference_task"]:
|
||||||
|
reasons.append("reference_validation_task_required")
|
||||||
|
elif not cls._reference_task_is_approved(evidence):
|
||||||
|
reasons.append("reference_task_authority_not_approved")
|
||||||
|
|
||||||
|
return DatasetConsumptionDecision(
|
||||||
|
eligible=not reasons,
|
||||||
|
purpose=purpose,
|
||||||
|
fixture_legacy_exception=False,
|
||||||
|
reasons=tuple(sorted(set(reasons))),
|
||||||
|
evidence=evidence,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def assert_eligible(
|
||||||
|
cls,
|
||||||
|
dataset: Any,
|
||||||
|
*,
|
||||||
|
purpose: DatasetConsumptionPurpose,
|
||||||
|
fixture_mode: bool = False,
|
||||||
|
reference_task: str | None = None,
|
||||||
|
) -> DatasetConsumptionDecision:
|
||||||
|
decision = cls.evaluate(
|
||||||
|
dataset,
|
||||||
|
purpose=purpose,
|
||||||
|
fixture_mode=fixture_mode,
|
||||||
|
reference_task=reference_task,
|
||||||
|
)
|
||||||
|
if decision.eligible:
|
||||||
|
return decision
|
||||||
|
code = "DATASET_QUARANTINED" if any(
|
||||||
|
reason in {"dataset_status_unsafe", "dataset_quarantined", "validation_failed", "source_snapshot_unsafe"}
|
||||||
|
for reason in decision.reasons
|
||||||
|
) else "DATASET_PROVENANCE_INCOMPLETE"
|
||||||
|
raise AppError(
|
||||||
|
code=code,
|
||||||
|
message="Dataset cannot be consumed until its provenance and validation gates are satisfied.",
|
||||||
|
status_code=409,
|
||||||
|
details={
|
||||||
|
"dataset_id": decision.evidence["dataset_id"],
|
||||||
|
"purpose": purpose,
|
||||||
|
"reasons": list(decision.reasons),
|
||||||
|
"fixture_legacy_exception": decision.fixture_legacy_exception,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def eligible_for_authoritative_coverage(cls, dataset: Any) -> bool:
|
||||||
|
"""Return false instead of raising so coverage can report a gap safely."""
|
||||||
|
|
||||||
|
return cls.evaluate(dataset, purpose="authoritative_coverage").eligible
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _reference_task_is_approved(evidence: Mapping[str, Any]) -> bool:
|
||||||
|
"""Require task-specific primary authority or an explicit zone approval.
|
||||||
|
|
||||||
|
``classification=authoritative`` is deliberately not a blanket
|
||||||
|
permission to serve as building truth. A source may be authoritative
|
||||||
|
for an address lifecycle or elevation product while remaining only
|
||||||
|
corroborative for footprint QA. A regional product that is marked
|
||||||
|
``*_pending_contract`` similarly remains blocked until an operator
|
||||||
|
records a narrow product-and-zone approval in its server-owned policy.
|
||||||
|
"""
|
||||||
|
|
||||||
|
task = str(evidence.get("reference_task") or "").strip().lower()
|
||||||
|
authority = DatasetConsumptionGate._normalise(
|
||||||
|
DatasetConsumptionGate._mapping(evidence.get("source_validation_authority")).get(task)
|
||||||
|
)
|
||||||
|
if authority == "primary":
|
||||||
|
return True
|
||||||
|
if authority not in {"approved", "approved_product_zone"}:
|
||||||
|
return False
|
||||||
|
|
||||||
|
approvals = DatasetConsumptionGate._mapping(evidence.get("source_reference_validation_approvals"))
|
||||||
|
approval = DatasetConsumptionGate._mapping(approvals.get(task))
|
||||||
|
if approval.get("approved") is not True:
|
||||||
|
return False
|
||||||
|
|
||||||
|
source_key = str(evidence.get("source_key") or "").strip().lower()
|
||||||
|
source_scope = DatasetConsumptionGate._mapping(evidence.get("source_authority_scope"))
|
||||||
|
source_zone = str(source_scope.get("zone") or source_scope.get("scope") or "").strip()
|
||||||
|
approved_keys = approval.get("source_keys")
|
||||||
|
approved_zones = approval.get("zones")
|
||||||
|
if not isinstance(approved_keys, list) or source_key not in {
|
||||||
|
str(value).strip().lower() for value in approved_keys
|
||||||
|
}:
|
||||||
|
return False
|
||||||
|
if not isinstance(approved_zones, list) or source_zone not in {
|
||||||
|
str(value).strip() for value in approved_zones
|
||||||
|
}:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -12,6 +12,7 @@ from sqlalchemy.orm import Session
|
|||||||
|
|
||||||
from app.models import Area, Dataset, DatasetVersion, Metric, Project, QualityCheck
|
from app.models import Area, Dataset, DatasetVersion, Metric, Project, QualityCheck
|
||||||
from app.schemas.demo import DemoWorkflowResponse
|
from app.schemas.demo import DemoWorkflowResponse
|
||||||
|
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
|
||||||
from app.services.geojson_service import parse_geojson_payload
|
from app.services.geojson_service import parse_geojson_payload
|
||||||
from app.services.qa_service import QaService
|
from app.services.qa_service import QaService
|
||||||
from app.services.quality_service import QualityService
|
from app.services.quality_service import QualityService
|
||||||
@@ -30,17 +31,17 @@ class DemoWorkflowService:
|
|||||||
EXPECTED_METRICS_FILENAME = "expected_qa_metrics.json"
|
EXPECTED_METRICS_FILENAME = "expected_qa_metrics.json"
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _add_initial_version(db: Session, dataset: Dataset) -> None:
|
def _add_initial_version(db: Session, dataset: Dataset) -> DatasetVersion:
|
||||||
db.add(
|
version = DatasetVersion(
|
||||||
DatasetVersion(
|
dataset_id=dataset.id,
|
||||||
dataset_id=dataset.id,
|
version=1,
|
||||||
version=1,
|
storage_path=dataset.storage_path,
|
||||||
storage_path=dataset.storage_path,
|
checksum_sha256=dataset.checksum_sha256,
|
||||||
checksum_sha256=dataset.checksum_sha256,
|
source_metadata=dataset.source_metadata,
|
||||||
source_metadata=dataset.source_metadata,
|
provenance_metadata=dataset.provenance_metadata,
|
||||||
provenance_metadata=dataset.provenance_metadata,
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
|
db.add(version)
|
||||||
|
return version
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _repo_root() -> Path:
|
def _repo_root() -> Path:
|
||||||
@@ -233,24 +234,34 @@ class DemoWorkflowService:
|
|||||||
crs=metadata.get("crs"),
|
crs=metadata.get("crs"),
|
||||||
bounds_json=metadata.get("bounds_json"),
|
bounds_json=metadata.get("bounds_json"),
|
||||||
metadata_json=metadata,
|
metadata_json=metadata,
|
||||||
status="ready",
|
status="validating",
|
||||||
)
|
)
|
||||||
db.add(dataset)
|
db.add(dataset)
|
||||||
DemoWorkflowService._add_initial_version(db, dataset)
|
version = DemoWorkflowService._add_initial_version(db, dataset)
|
||||||
|
is_ready = DerivedDatasetGovernanceService.govern_vector(
|
||||||
|
db,
|
||||||
|
dataset=dataset,
|
||||||
|
dataset_version=version,
|
||||||
|
feature_collection=payload,
|
||||||
|
source_key="fixture",
|
||||||
|
operation="demo.fixture_vector",
|
||||||
|
operation_parameters={"fixture_name": filename, "role": role},
|
||||||
|
)
|
||||||
|
if is_ready:
|
||||||
|
VectorFeatureService.persist_geojson_features(
|
||||||
|
db=db,
|
||||||
|
dataset_id=dataset.id,
|
||||||
|
payload=payload,
|
||||||
|
feature_class=reference_layer_name or "building",
|
||||||
|
commit=False,
|
||||||
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
db.refresh(dataset)
|
db.refresh(dataset)
|
||||||
VectorFeatureService.persist_geojson_features(
|
|
||||||
db=db,
|
|
||||||
dataset_id=dataset.id,
|
|
||||||
payload=payload,
|
|
||||||
feature_class=reference_layer_name or "building",
|
|
||||||
)
|
|
||||||
return dataset
|
return dataset
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _create_demo_raster_bytes() -> bytes:
|
def _create_demo_raster_bytes() -> bytes:
|
||||||
numpy = importlib.import_module("numpy")
|
numpy = importlib.import_module("numpy")
|
||||||
rasterio = importlib.import_module("rasterio")
|
|
||||||
rasterio_io = importlib.import_module("rasterio.io")
|
rasterio_io = importlib.import_module("rasterio.io")
|
||||||
rasterio_transform = importlib.import_module("rasterio.transform")
|
rasterio_transform = importlib.import_module("rasterio.transform")
|
||||||
|
|
||||||
@@ -318,10 +329,19 @@ class DemoWorkflowService:
|
|||||||
crs=metadata.get("crs"),
|
crs=metadata.get("crs"),
|
||||||
bounds_json=bounds_json,
|
bounds_json=bounds_json,
|
||||||
metadata_json=metadata,
|
metadata_json=metadata,
|
||||||
status="ready",
|
status="validating",
|
||||||
)
|
)
|
||||||
db.add(dataset)
|
db.add(dataset)
|
||||||
DemoWorkflowService._add_initial_version(db, dataset)
|
version = DemoWorkflowService._add_initial_version(db, dataset)
|
||||||
|
DerivedDatasetGovernanceService.govern_raster(
|
||||||
|
db,
|
||||||
|
dataset=dataset,
|
||||||
|
dataset_version=version,
|
||||||
|
raster_metadata=metadata,
|
||||||
|
source_key="fixture",
|
||||||
|
operation="demo.fixture_raster",
|
||||||
|
operation_parameters={"fixture_name": DemoWorkflowService.RASTER_FILENAME},
|
||||||
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
db.refresh(dataset)
|
db.refresh(dataset)
|
||||||
return dataset
|
return dataset
|
||||||
|
|||||||
@@ -0,0 +1,516 @@
|
|||||||
|
"""Governance for persisted derived and fixture datasets.
|
||||||
|
|
||||||
|
Dataset importers own raw-source ingestion. This small service owns the
|
||||||
|
other persistence boundary: artifacts produced inside the workbench (vector
|
||||||
|
and raster operations) and the explicitly local demo fixtures. It is kept
|
||||||
|
separate from :mod:`dataset_service` so an operation can never create a ready
|
||||||
|
dataset without a source registry binding, immutable snapshot, validation
|
||||||
|
report and, for derived results, a durable lineage edge.
|
||||||
|
|
||||||
|
The ``Session.query`` capability check deliberately preserves lightweight
|
||||||
|
unit-test doubles used by pre-Phase-2 tests. Real SQLAlchemy sessions always
|
||||||
|
take the governed branch; the compatibility branch is not reachable in the
|
||||||
|
application runtime.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from hashlib import sha256
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.models import Dataset, DatasetVersion
|
||||||
|
from app.services.data_contract_validation import (
|
||||||
|
IssueSeverity,
|
||||||
|
LineageStatus,
|
||||||
|
LineageEvidence,
|
||||||
|
ProvenanceStatus,
|
||||||
|
QuarantineStatus,
|
||||||
|
TransformationEvidence,
|
||||||
|
ValidationIssue,
|
||||||
|
ValidationReport,
|
||||||
|
ValidationStatus,
|
||||||
|
build_raster_ingest_input,
|
||||||
|
build_vector_ingest_input,
|
||||||
|
validate_registered_asset,
|
||||||
|
)
|
||||||
|
from app.services.data_quarantine_service import DataQuarantineService
|
||||||
|
from app.services.dataset_consumption_gate_service import DatasetConsumptionDecision, DatasetConsumptionGate
|
||||||
|
from app.services.source_registry_service import SourceRegistryService
|
||||||
|
|
||||||
|
|
||||||
|
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||||
|
_CANONICAL_VECTOR_CRS = "EPSG:4326"
|
||||||
|
|
||||||
|
|
||||||
|
class DerivedDatasetGovernanceService:
|
||||||
|
"""Apply Phase-2 provenance rules to non-importer Dataset creation.
|
||||||
|
|
||||||
|
Callers add the dataset and first immutable version, then call one of the
|
||||||
|
``govern_*`` methods before committing. A failed contract deliberately
|
||||||
|
leaves the artifact and its durable quarantine record in the transaction;
|
||||||
|
it is never silently promoted to ``ready``.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def persistence_available(db: Session) -> bool:
|
||||||
|
"""Return whether this is a real ORM persistence session.
|
||||||
|
|
||||||
|
Historical unit tests use minimal fakes with ``add``/``commit`` only.
|
||||||
|
Keeping that explicitly isolated avoids pretending a fake test store
|
||||||
|
has source-registry guarantees while production remains fail-closed.
|
||||||
|
"""
|
||||||
|
|
||||||
|
return callable(getattr(db, "query", None)) and callable(getattr(db, "flush", None))
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def govern_vector(
|
||||||
|
cls,
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
dataset: Dataset,
|
||||||
|
dataset_version: DatasetVersion,
|
||||||
|
feature_collection: Mapping[str, Any],
|
||||||
|
source_key: str,
|
||||||
|
operation: str,
|
||||||
|
parent_dataset: Dataset | None = None,
|
||||||
|
operation_parameters: Mapping[str, Any] | None = None,
|
||||||
|
) -> bool:
|
||||||
|
"""Validate and bind a vector result, returning ``True`` when ready."""
|
||||||
|
|
||||||
|
if not cls.persistence_available(db):
|
||||||
|
# Explicit compatibility for historical minimal test fixtures.
|
||||||
|
# Production sessions always have query/flush and never take this
|
||||||
|
# branch.
|
||||||
|
dataset.status = "ready"
|
||||||
|
return True
|
||||||
|
|
||||||
|
parent_gate = cls._parent_derived_processing_gate(parent_dataset)
|
||||||
|
metadata = dict(dataset.metadata_json or {})
|
||||||
|
output_crs = str(metadata.get("crs") or dataset.crs or _CANONICAL_VECTOR_CRS)
|
||||||
|
source = SourceRegistryService.ensure_server_owned_source(db, source_key)
|
||||||
|
snapshot = cls._record_snapshot(
|
||||||
|
db,
|
||||||
|
source_key=source_key,
|
||||||
|
dataset=dataset,
|
||||||
|
operation=operation,
|
||||||
|
source_crs=output_crs,
|
||||||
|
spatial_resolution=metadata.get("resolution_json") or {"status": "not_applicable"},
|
||||||
|
geographic_coverage={"bounds": metadata.get("bounds_json") or dataset.bounds_json},
|
||||||
|
observed_schema={
|
||||||
|
"dataset_type": "vector",
|
||||||
|
"geometry_types": metadata.get("geometry_types") or [],
|
||||||
|
"feature_count": metadata.get("feature_count"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
lineage = cls._lineage_evidence(parent_dataset, operation, operation_parameters)
|
||||||
|
report = validate_registered_asset(
|
||||||
|
build_vector_ingest_input(
|
||||||
|
asset_id=str(dataset.id),
|
||||||
|
source_crs=output_crs,
|
||||||
|
storage_crs=output_crs,
|
||||||
|
feature_collection=feature_collection,
|
||||||
|
checksum_sha256=dataset.checksum_sha256,
|
||||||
|
computed_checksum_sha256=dataset.checksum_sha256,
|
||||||
|
source_registry_id=str(source.id),
|
||||||
|
source_snapshot_id=str(snapshot.id),
|
||||||
|
imported_at=dataset.imported_at or datetime.now(timezone.utc),
|
||||||
|
metadata=cls._contract_metadata(metadata, source.license_name),
|
||||||
|
observed_at=dataset.observed_at,
|
||||||
|
valid_from=dataset.valid_from,
|
||||||
|
valid_to=dataset.valid_to,
|
||||||
|
temporal_unknown_reason=cls._temporal_unknown_reason(dataset),
|
||||||
|
source_version=dataset.source_version,
|
||||||
|
source_version_unknown_reason=cls._source_version_unknown_reason(dataset),
|
||||||
|
lineage=lineage,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if parent_gate is not None and not parent_gate.eligible:
|
||||||
|
report = cls._with_parent_gate_failure(report, parent_gate)
|
||||||
|
return cls._apply(
|
||||||
|
db,
|
||||||
|
dataset=dataset,
|
||||||
|
dataset_version=dataset_version,
|
||||||
|
source=source,
|
||||||
|
snapshot=snapshot,
|
||||||
|
report=report,
|
||||||
|
stage="derived_vector_validation",
|
||||||
|
parent_dataset=parent_dataset,
|
||||||
|
operation=operation,
|
||||||
|
operation_parameters=operation_parameters,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def govern_raster(
|
||||||
|
cls,
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
dataset: Dataset,
|
||||||
|
dataset_version: DatasetVersion,
|
||||||
|
raster_metadata: Mapping[str, Any],
|
||||||
|
source_key: str,
|
||||||
|
operation: str,
|
||||||
|
parent_dataset: Dataset | None = None,
|
||||||
|
operation_parameters: Mapping[str, Any] | None = None,
|
||||||
|
) -> bool:
|
||||||
|
"""Validate and bind a raster result, returning ``True`` when ready."""
|
||||||
|
|
||||||
|
if not cls.persistence_available(db):
|
||||||
|
dataset.status = "ready"
|
||||||
|
return True
|
||||||
|
|
||||||
|
parent_gate = cls._parent_derived_processing_gate(parent_dataset)
|
||||||
|
metadata = dict(raster_metadata or {})
|
||||||
|
output_crs = str(metadata.get("crs") or dataset.crs or "") or None
|
||||||
|
source = SourceRegistryService.ensure_server_owned_source(db, source_key)
|
||||||
|
snapshot = cls._record_snapshot(
|
||||||
|
db,
|
||||||
|
source_key=source_key,
|
||||||
|
dataset=dataset,
|
||||||
|
operation=operation,
|
||||||
|
source_crs=output_crs,
|
||||||
|
spatial_resolution=cls._raster_resolution(metadata, output_crs),
|
||||||
|
geographic_coverage={"bounds": metadata.get("bounds") or dataset.bounds_json},
|
||||||
|
observed_schema={
|
||||||
|
"dataset_type": "raster",
|
||||||
|
"width": metadata.get("width"),
|
||||||
|
"height": metadata.get("height"),
|
||||||
|
"band_count": metadata.get("band_count"),
|
||||||
|
"dtype": metadata.get("dtype"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
lineage = cls._lineage_evidence(parent_dataset, operation, operation_parameters)
|
||||||
|
report = validate_registered_asset(
|
||||||
|
build_raster_ingest_input(
|
||||||
|
asset_id=str(dataset.id),
|
||||||
|
source_crs=output_crs,
|
||||||
|
storage_crs=output_crs,
|
||||||
|
raster_profile=metadata,
|
||||||
|
bounds=metadata.get("bounds") or dataset.bounds_json,
|
||||||
|
resolution=cls._raster_resolution(metadata, output_crs),
|
||||||
|
checksum_sha256=dataset.checksum_sha256,
|
||||||
|
computed_checksum_sha256=dataset.checksum_sha256,
|
||||||
|
source_registry_id=str(source.id),
|
||||||
|
source_snapshot_id=str(snapshot.id),
|
||||||
|
imported_at=dataset.imported_at or datetime.now(timezone.utc),
|
||||||
|
metadata=cls._contract_metadata(metadata, source.license_name),
|
||||||
|
observed_at=dataset.observed_at,
|
||||||
|
valid_from=dataset.valid_from,
|
||||||
|
valid_to=dataset.valid_to,
|
||||||
|
temporal_unknown_reason=cls._temporal_unknown_reason(dataset),
|
||||||
|
source_version=dataset.source_version,
|
||||||
|
source_version_unknown_reason=cls._source_version_unknown_reason(dataset),
|
||||||
|
lineage=lineage,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if parent_gate is not None and not parent_gate.eligible:
|
||||||
|
report = cls._with_parent_gate_failure(report, parent_gate)
|
||||||
|
return cls._apply(
|
||||||
|
db,
|
||||||
|
dataset=dataset,
|
||||||
|
dataset_version=dataset_version,
|
||||||
|
source=source,
|
||||||
|
snapshot=snapshot,
|
||||||
|
report=report,
|
||||||
|
stage="derived_raster_validation",
|
||||||
|
parent_dataset=parent_dataset,
|
||||||
|
operation=operation,
|
||||||
|
operation_parameters=operation_parameters,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _record_snapshot(
|
||||||
|
cls,
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
source_key: str,
|
||||||
|
dataset: Dataset,
|
||||||
|
operation: str,
|
||||||
|
source_crs: str | None,
|
||||||
|
spatial_resolution: Mapping[str, Any] | None,
|
||||||
|
geographic_coverage: Mapping[str, Any] | None,
|
||||||
|
observed_schema: Mapping[str, Any] | None,
|
||||||
|
):
|
||||||
|
checksum = cls._checksum_or_placeholder(dataset.checksum_sha256)
|
||||||
|
# A derived artifact has a distinct creation event even when its bytes
|
||||||
|
# equal a prior output. Include the immutable dataset id so source
|
||||||
|
# snapshots never collide on a different fetched_at timestamp.
|
||||||
|
snapshot_key = f"{source_key}:{operation}:{dataset.id}:{checksum}"
|
||||||
|
return SourceRegistryService.record_snapshot(
|
||||||
|
db,
|
||||||
|
source_key=source_key,
|
||||||
|
snapshot_key=snapshot_key,
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
source_version=dataset.source_version or f"{operation}:1.0.0",
|
||||||
|
snapshot_at=dataset.observed_at,
|
||||||
|
fetched_at=dataset.imported_at or datetime.now(timezone.utc),
|
||||||
|
crs=source_crs,
|
||||||
|
units=cls._units_for_crs(source_crs),
|
||||||
|
spatial_resolution=dict(spatial_resolution or {"status": "unknown"}),
|
||||||
|
temporal_coverage={
|
||||||
|
"observed_at": cls._datetime_value(dataset.observed_at),
|
||||||
|
"valid_from": cls._datetime_value(dataset.valid_from),
|
||||||
|
"valid_to": cls._datetime_value(dataset.valid_to),
|
||||||
|
},
|
||||||
|
geographic_coverage=dict(geographic_coverage or {"status": "unknown"}),
|
||||||
|
observed_schema=dict(observed_schema or {"status": "unknown"}),
|
||||||
|
# A transform cannot establish source freshness. With no source
|
||||||
|
# observation it is not applicable; with one it still needs an
|
||||||
|
# explicit policy review rather than a fabricated "current" flag.
|
||||||
|
freshness_status="not_applicable" if dataset.observed_at is None else "review_required",
|
||||||
|
ingest_status="ingested",
|
||||||
|
known_limitations=[
|
||||||
|
"Derived and fixture artifacts inherit no automatic source authority beyond their explicit registry entry and lineage.",
|
||||||
|
],
|
||||||
|
snapshot_metadata={
|
||||||
|
"operation": operation,
|
||||||
|
"dataset_id": str(dataset.id),
|
||||||
|
"storage_path": dataset.storage_path,
|
||||||
|
"artifact_checksum_sha256": dataset.checksum_sha256,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _apply(
|
||||||
|
cls,
|
||||||
|
db: Session,
|
||||||
|
*,
|
||||||
|
dataset: Dataset,
|
||||||
|
dataset_version: DatasetVersion,
|
||||||
|
source: Any,
|
||||||
|
snapshot: Any,
|
||||||
|
report: ValidationReport,
|
||||||
|
stage: str,
|
||||||
|
parent_dataset: Dataset | None,
|
||||||
|
operation: str,
|
||||||
|
operation_parameters: Mapping[str, Any] | None,
|
||||||
|
) -> bool:
|
||||||
|
fields = report.persistence_fields()
|
||||||
|
dataset.validation_report_json = fields["validation_report_json"]
|
||||||
|
dataset_version.validation_report_json = fields["validation_report_json"]
|
||||||
|
SourceRegistryService.bind_dataset_provenance(
|
||||||
|
dataset,
|
||||||
|
source=source,
|
||||||
|
snapshot=snapshot,
|
||||||
|
data_contract_key=fields["data_contract_key"],
|
||||||
|
data_contract_version=fields["data_contract_version"],
|
||||||
|
validation_status=fields["validation_status"],
|
||||||
|
provenance_status=fields["provenance_status"],
|
||||||
|
lineage_status=fields["lineage_status"],
|
||||||
|
)
|
||||||
|
SourceRegistryService.bind_dataset_version_provenance(
|
||||||
|
dataset_version,
|
||||||
|
source=source,
|
||||||
|
snapshot=snapshot,
|
||||||
|
data_contract_key=fields["data_contract_key"],
|
||||||
|
data_contract_version=fields["data_contract_version"],
|
||||||
|
validation_status=fields["validation_status"],
|
||||||
|
provenance_status=fields["provenance_status"],
|
||||||
|
lineage_status=fields["lineage_status"],
|
||||||
|
)
|
||||||
|
|
||||||
|
# IDs for DatasetVersion defaults exist only after the caller adds and
|
||||||
|
# flushes both rows. The operation services call this before commit.
|
||||||
|
db.flush()
|
||||||
|
if parent_dataset is not None:
|
||||||
|
SourceRegistryService.record_lineage_edge(
|
||||||
|
db,
|
||||||
|
parent_dataset_id=parent_dataset.id,
|
||||||
|
child_dataset_id=dataset.id,
|
||||||
|
parent_dataset_version_id=cls._latest_parent_version_id(db, parent_dataset),
|
||||||
|
child_dataset_version_id=dataset_version.id,
|
||||||
|
relation_type="derived_from",
|
||||||
|
transformation_name=operation,
|
||||||
|
transformation_version="1.0.0",
|
||||||
|
parameters=dict(operation_parameters or {}),
|
||||||
|
input_checksum_sha256=cls._valid_checksum(parent_dataset.checksum_sha256),
|
||||||
|
output_checksum_sha256=cls._valid_checksum(dataset.checksum_sha256),
|
||||||
|
)
|
||||||
|
|
||||||
|
decision = DataQuarantineService.decide(report)
|
||||||
|
if decision.eligible_for_use:
|
||||||
|
dataset.status = "ready"
|
||||||
|
dataset.quarantine_status = "not_quarantined"
|
||||||
|
return True
|
||||||
|
|
||||||
|
SourceRegistryService.quarantine_dataset(
|
||||||
|
db,
|
||||||
|
dataset=dataset,
|
||||||
|
dataset_version=dataset_version,
|
||||||
|
source_snapshot=snapshot,
|
||||||
|
stage=stage,
|
||||||
|
reason_code=(decision.reason_codes[0] if decision.reason_codes else "DATA_CONTRACT_FAILED"),
|
||||||
|
details={"validation_report": report.to_dict(), "quarantine_decision": decision.to_dict()},
|
||||||
|
artifact_path=dataset.storage_path,
|
||||||
|
artifact_checksum_sha256=cls._valid_checksum(dataset.checksum_sha256),
|
||||||
|
)
|
||||||
|
return False
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _contract_metadata(metadata: Mapping[str, Any], license_name: str) -> dict[str, Any]:
|
||||||
|
values = dict(metadata)
|
||||||
|
values.setdefault("license", license_name)
|
||||||
|
return values
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _lineage_evidence(
|
||||||
|
cls,
|
||||||
|
parent_dataset: Dataset | None,
|
||||||
|
operation: str,
|
||||||
|
operation_parameters: Mapping[str, Any] | None,
|
||||||
|
) -> LineageEvidence:
|
||||||
|
upstream_ids: tuple[str, ...] = ()
|
||||||
|
upstream_checksums: tuple[str, ...] = ()
|
||||||
|
if parent_dataset is not None:
|
||||||
|
upstream_ids = (str(parent_dataset.id),)
|
||||||
|
# An invalid/missing parent checksum intentionally fails the
|
||||||
|
# derived contract rather than inventing traceability. The same
|
||||||
|
# applies to a legacy/unvalidated parent: it may remain visible
|
||||||
|
# as evidence, but cannot create a new ready derived asset.
|
||||||
|
parent_is_governed = (
|
||||||
|
parent_dataset.status == "ready"
|
||||||
|
and parent_dataset.quarantine_status == "not_quarantined"
|
||||||
|
and parent_dataset.validation_status == "passed"
|
||||||
|
and parent_dataset.provenance_status == "complete"
|
||||||
|
and parent_dataset.source_registry_id is not None
|
||||||
|
and parent_dataset.source_snapshot_id is not None
|
||||||
|
)
|
||||||
|
upstream_checksums = (
|
||||||
|
parent_dataset.checksum_sha256 if parent_is_governed else "parent_dataset_not_governed",
|
||||||
|
)
|
||||||
|
transform_checksum = cls._stable_hash(
|
||||||
|
{"operation": operation, "version": "1.0.0", "parameters": dict(operation_parameters or {})}
|
||||||
|
)
|
||||||
|
return LineageEvidence(
|
||||||
|
upstream_asset_ids=upstream_ids,
|
||||||
|
upstream_checksums_sha256=upstream_checksums,
|
||||||
|
transformations=(
|
||||||
|
TransformationEvidence(
|
||||||
|
name=operation,
|
||||||
|
version="1.0.0",
|
||||||
|
checksum_sha256=transform_checksum,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _parent_derived_processing_gate(parent_dataset: Dataset | None) -> DatasetConsumptionDecision | None:
|
||||||
|
"""Evaluate the durable parent boundary before creating a ready child.
|
||||||
|
|
||||||
|
We deliberately turn a rejected parent into a child validation failure
|
||||||
|
(instead of simply raising): the output is then persisted with its
|
||||||
|
source snapshot, validation evidence and durable quarantine record.
|
||||||
|
This makes an attempted derivation from a manual or experimental
|
||||||
|
dataset observable and prevents a caller from bypassing the boundary
|
||||||
|
by invoking the governance service directly.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if parent_dataset is None:
|
||||||
|
return None
|
||||||
|
return DatasetConsumptionGate.evaluate(parent_dataset, purpose="derived_processing")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _with_parent_gate_failure(
|
||||||
|
report: ValidationReport,
|
||||||
|
decision: DatasetConsumptionDecision,
|
||||||
|
) -> ValidationReport:
|
||||||
|
"""Attach an auditable, fail-closed lineage failure to a report."""
|
||||||
|
|
||||||
|
issue = ValidationIssue(
|
||||||
|
code="PARENT_DATASET_NOT_ELIGIBLE_FOR_DERIVED_PROCESSING",
|
||||||
|
category="lineage",
|
||||||
|
field="lineage.parent_dataset",
|
||||||
|
message="Parent dataset failed the governed derived-processing consumption gate.",
|
||||||
|
severity=IssueSeverity.ERROR,
|
||||||
|
expected="eligible governed parent dataset",
|
||||||
|
observed={
|
||||||
|
"dataset_id": decision.evidence.get("dataset_id"),
|
||||||
|
"reasons": list(decision.reasons),
|
||||||
|
"source_key": decision.evidence.get("source_key"),
|
||||||
|
"source_classification": decision.evidence.get("source_classification"),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return ValidationReport(
|
||||||
|
asset_id=report.asset_id,
|
||||||
|
data_contract_key=report.data_contract_key,
|
||||||
|
data_contract_version=report.data_contract_version,
|
||||||
|
contract_fingerprint_sha256=report.contract_fingerprint_sha256,
|
||||||
|
validation_status=ValidationStatus.FAILED,
|
||||||
|
provenance_status=(
|
||||||
|
ProvenanceStatus.INCOMPLETE
|
||||||
|
if report.provenance_status == ProvenanceStatus.COMPLETE
|
||||||
|
else report.provenance_status
|
||||||
|
),
|
||||||
|
lineage_status=LineageStatus.INCOMPLETE,
|
||||||
|
quarantine_status=QuarantineStatus.QUARANTINED,
|
||||||
|
validation_scope=report.validation_scope,
|
||||||
|
checked_at=report.checked_at,
|
||||||
|
issues=(*report.issues, issue),
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _latest_parent_version_id(db: Session, dataset: Dataset):
|
||||||
|
version = (
|
||||||
|
db.query(DatasetVersion)
|
||||||
|
.filter(DatasetVersion.dataset_id == dataset.id)
|
||||||
|
.order_by(DatasetVersion.version.desc())
|
||||||
|
.first()
|
||||||
|
)
|
||||||
|
return version.id if version is not None else None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _raster_resolution(metadata: Mapping[str, Any], crs: str | None) -> dict[str, Any] | None:
|
||||||
|
values = metadata.get("resolution")
|
||||||
|
if not isinstance(values, (list, tuple)) or len(values) < 2:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return {"x": abs(float(values[0])), "y": abs(float(values[1])), "unit": DerivedDatasetGovernanceService._resolution_unit(crs)}
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _resolution_unit(crs: str | None) -> str:
|
||||||
|
return "degree" if str(crs or "").upper() == _CANONICAL_VECTOR_CRS else "m"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _units_for_crs(crs: str | None) -> str:
|
||||||
|
return "degrees" if str(crs or "").upper() == _CANONICAL_VECTOR_CRS else "metres"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _temporal_unknown_reason(dataset: Dataset) -> str | None:
|
||||||
|
if dataset.observed_at is not None:
|
||||||
|
return None
|
||||||
|
return "Derived or fixture artifact inherits no precise observation timestamp from its input."
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _source_version_unknown_reason(dataset: Dataset) -> str | None:
|
||||||
|
if dataset.source_version:
|
||||||
|
return None
|
||||||
|
return "Derived or fixture artifact has no source edition; the transform version is recorded separately."
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _datetime_value(value: datetime | None) -> str | None:
|
||||||
|
return value.astimezone(timezone.utc).isoformat() if value is not None else None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _stable_hash(value: Mapping[str, Any]) -> str:
|
||||||
|
return sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True).encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _valid_checksum(value: str | None) -> str | None:
|
||||||
|
normalized = str(value or "").strip().lower()
|
||||||
|
return normalized if _SHA256.fullmatch(normalized) else None
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _checksum_or_placeholder(cls, value: str | None) -> str:
|
||||||
|
checksum = cls._valid_checksum(value)
|
||||||
|
if checksum is not None:
|
||||||
|
return checksum
|
||||||
|
# The contract receives the original invalid/missing checksum and
|
||||||
|
# quarantines it. A deterministic placeholder only permits storing
|
||||||
|
# the rejected snapshot without fabricating a valid artifact hash.
|
||||||
|
return cls._stable_hash({"invalid_dataset_checksum": value or "missing"})
|
||||||
@@ -19,10 +19,12 @@ from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project, Vect
|
|||||||
from app.schemas.detection import DetectionListResponse, DetectionRead, DetectionRunListResponse, DetectionRunRead, DetectionRunResponse
|
from app.schemas.detection import DetectionListResponse, DetectionRead, DetectionRunListResponse, DetectionRunRead, DetectionRunResponse
|
||||||
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
|
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
|
||||||
from app.services.detection_qa_service import DetectionQaService
|
from app.services.detection_qa_service import DetectionQaService
|
||||||
|
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||||
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
||||||
from app.services.model_registry_service import ModelRegistryService
|
from app.services.model_registry_service import ModelRegistryService
|
||||||
from app.services.qa_service import QaService
|
from app.services.qa_service import QaService
|
||||||
from app.services.quality_service import QualityService
|
from app.services.quality_service import QualityService
|
||||||
|
from app.services.runtime_model_provenance_service import RuntimeModelProvenance, RuntimeModelProvenanceService
|
||||||
from app.services.temporal_compatibility_service import TemporalCompatibilityService
|
from app.services.temporal_compatibility_service import TemporalCompatibilityService
|
||||||
from app.services.yolo_adapter import YoloDetectionAdapter
|
from app.services.yolo_adapter import YoloDetectionAdapter
|
||||||
|
|
||||||
@@ -97,6 +99,18 @@ class DetectionService:
|
|||||||
if model.model_id == resolved_settings.yolo_model_id and resolved_settings.yolo_enforce_validation_scope:
|
if model.model_id == resolved_settings.yolo_model_id and resolved_settings.yolo_enforce_validation_scope:
|
||||||
DetectionService._validate_model_area_scope(db, dataset, resolved_settings)
|
DetectionService._validate_model_area_scope(db, dataset, resolved_settings)
|
||||||
|
|
||||||
|
# Never enter a production inference path with a persisted dataset
|
||||||
|
# that has failed validation, incomplete provenance, or an active
|
||||||
|
# quarantine. Fixture detection is a separate QA/test-only path.
|
||||||
|
if model.model_id == "manual-fixture-detector":
|
||||||
|
DatasetConsumptionGate.assert_eligible(
|
||||||
|
dataset,
|
||||||
|
purpose="quality_assessment",
|
||||||
|
fixture_mode=True,
|
||||||
|
)
|
||||||
|
elif model.model_id == resolved_settings.yolo_model_id and model.configured:
|
||||||
|
DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
|
||||||
|
|
||||||
run_parameters = {
|
run_parameters = {
|
||||||
"model_id": model.model_id,
|
"model_id": model.model_id,
|
||||||
"model_asset_id": selected_model_asset.model_asset_id if selected_model_asset else None,
|
"model_asset_id": selected_model_asset.model_asset_id if selected_model_asset else None,
|
||||||
@@ -352,15 +366,6 @@ class DetectionService:
|
|||||||
reference_dataset,
|
reference_dataset,
|
||||||
)
|
)
|
||||||
|
|
||||||
detections = DetectionService._query_detection_rows(
|
|
||||||
db,
|
|
||||||
analysis_run_id=analysis_run_id,
|
|
||||||
dataset_id=run.dataset_id,
|
|
||||||
class_name=class_name,
|
|
||||||
min_confidence=min_confidence,
|
|
||||||
)
|
|
||||||
raw_candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in detections]
|
|
||||||
candidate_geometries = raw_candidate_geometries
|
|
||||||
run_parameters = run.parameters_json if isinstance(run.parameters_json, dict) else {}
|
run_parameters = run.parameters_json if isinstance(run.parameters_json, dict) else {}
|
||||||
manifest_path = DetectionQaService.tile_manifest_path(run_parameters)
|
manifest_path = DetectionQaService.tile_manifest_path(run_parameters)
|
||||||
resolved_settings = get_settings()
|
resolved_settings = get_settings()
|
||||||
@@ -375,6 +380,33 @@ class DetectionService:
|
|||||||
status_code=422,
|
status_code=422,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
fixture_parameters = run_parameters.get("parameters_json")
|
||||||
|
fixture_mode = bool(
|
||||||
|
run.model_name == "manual-fixture-detector"
|
||||||
|
and isinstance(fixture_parameters, dict)
|
||||||
|
and fixture_parameters.get("fixture_mode") is True
|
||||||
|
)
|
||||||
|
DatasetConsumptionGate.assert_eligible(
|
||||||
|
candidate_dataset,
|
||||||
|
purpose="quality_assessment",
|
||||||
|
fixture_mode=fixture_mode,
|
||||||
|
)
|
||||||
|
DatasetConsumptionGate.assert_eligible(
|
||||||
|
reference_dataset,
|
||||||
|
purpose="reference_validation",
|
||||||
|
reference_task="building_validation",
|
||||||
|
)
|
||||||
|
|
||||||
|
detections = DetectionService._query_detection_rows(
|
||||||
|
db,
|
||||||
|
analysis_run_id=analysis_run_id,
|
||||||
|
dataset_id=run.dataset_id,
|
||||||
|
class_name=class_name,
|
||||||
|
min_confidence=min_confidence,
|
||||||
|
)
|
||||||
|
raw_candidate_geometries = [({"id": str(row.id), "class_name": row.class_name}, to_shape(row.geometry)) for row in detections]
|
||||||
|
candidate_geometries = raw_candidate_geometries
|
||||||
|
|
||||||
coverage = None
|
coverage = None
|
||||||
if manifest_path:
|
if manifest_path:
|
||||||
manifest = DetectionService._load_tile_manifest(manifest_path, resolved_settings.yolo_max_tiles)
|
manifest = DetectionService._load_tile_manifest(manifest_path, resolved_settings.yolo_max_tiles)
|
||||||
@@ -728,6 +760,19 @@ class DetectionService:
|
|||||||
) -> tuple[list[Detection], dict[str, Any]]:
|
) -> tuple[list[Detection], dict[str, Any]]:
|
||||||
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles)
|
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles)
|
||||||
model_path = Path(settings.yolo_model_path or "").expanduser()
|
model_path = Path(settings.yolo_model_path or "").expanduser()
|
||||||
|
runtime_model_provenance = RuntimeModelProvenanceService.validate_for_production_runtime(
|
||||||
|
db=db,
|
||||||
|
model_path=model_path,
|
||||||
|
model_id=model_name,
|
||||||
|
task_type="object_detection",
|
||||||
|
expected_model_version=model_version,
|
||||||
|
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
|
||||||
|
)
|
||||||
|
DetectionService._attach_runtime_model_provenance(
|
||||||
|
analysis_run,
|
||||||
|
job,
|
||||||
|
runtime_model_provenance,
|
||||||
|
)
|
||||||
adapter = yolo_adapter_class(settings)
|
adapter = yolo_adapter_class(settings)
|
||||||
model = adapter.load_model(model_path)
|
model = adapter.load_model(model_path)
|
||||||
allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)}
|
allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)}
|
||||||
@@ -785,7 +830,10 @@ class DetectionService:
|
|||||||
"y_max": float(bbox[3]),
|
"y_max": float(bbox[3]),
|
||||||
},
|
},
|
||||||
source_tile_path=candidate["source_tile_path"],
|
source_tile_path=candidate["source_tile_path"],
|
||||||
properties_json=candidate["properties"],
|
properties_json={
|
||||||
|
**candidate["properties"],
|
||||||
|
"runtime_model_provenance": runtime_model_provenance.as_dict(),
|
||||||
|
},
|
||||||
)
|
)
|
||||||
db.add(detection)
|
db.add(detection)
|
||||||
persisted.append(detection)
|
persisted.append(detection)
|
||||||
@@ -796,8 +844,30 @@ class DetectionService:
|
|||||||
"raw_detection_count": len(candidates),
|
"raw_detection_count": len(candidates),
|
||||||
"suppressed_detection_count": len(candidates) - len(filtered_candidates),
|
"suppressed_detection_count": len(candidates) - len(filtered_candidates),
|
||||||
"duplicate_iou_threshold": float(settings.yolo_duplicate_iou_threshold),
|
"duplicate_iou_threshold": float(settings.yolo_duplicate_iou_threshold),
|
||||||
|
"runtime_model_provenance": runtime_model_provenance.as_dict(),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _attach_runtime_model_provenance(
|
||||||
|
analysis_run: AnalysisRun,
|
||||||
|
job: Job,
|
||||||
|
provenance: RuntimeModelProvenance,
|
||||||
|
) -> None:
|
||||||
|
"""Persist byte-bound model evidence with the run before adapter loading.
|
||||||
|
|
||||||
|
Individual detections retain the same evidence in ``properties_json``;
|
||||||
|
this run-level copy is the compact audit root for a complete inference.
|
||||||
|
Assigning fresh dictionaries matters for SQLAlchemy JSON change tracking.
|
||||||
|
"""
|
||||||
|
|
||||||
|
evidence = provenance.as_dict()
|
||||||
|
analysis_parameters = dict(analysis_run.parameters_json or {})
|
||||||
|
analysis_parameters["runtime_model_provenance"] = evidence
|
||||||
|
analysis_run.parameters_json = analysis_parameters
|
||||||
|
job_parameters = dict(job.parameters_json or {})
|
||||||
|
job_parameters["runtime_model_provenance"] = evidence
|
||||||
|
job.parameters_json = job_parameters
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _canonical_class_name(value: Any) -> str:
|
def _canonical_class_name(value: Any) -> str:
|
||||||
return str(value or "").strip().casefold()
|
return str(value or "").strip().casefold()
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ from app.schemas.flood_hazard import FloodHazardPartitionSelectionRequest, Flood
|
|||||||
from app.schemas.temporal import TemporalComparisonRequest
|
from app.schemas.temporal import TemporalComparisonRequest
|
||||||
from app.schemas.thematic_raster import ThematicRasterSelectionRequest
|
from app.schemas.thematic_raster import ThematicRasterSelectionRequest
|
||||||
from app.services.dataset_service import DatasetService
|
from app.services.dataset_service import DatasetService
|
||||||
|
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||||
from app.services.detection_service import DetectionService
|
from app.services.detection_service import DetectionService
|
||||||
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
|
from app.services.flood_hazard_analysis_service import FloodHazardAnalysisService
|
||||||
from app.services.segmentation_service import SegmentationService
|
from app.services.segmentation_service import SegmentationService
|
||||||
@@ -34,12 +35,39 @@ from app.services.vector_feature_service import VectorFeatureService
|
|||||||
|
|
||||||
|
|
||||||
class ExportService:
|
class ExportService:
|
||||||
|
@staticmethod
|
||||||
|
def _assert_run_source_dataset_exportable(db: Session, run: AnalysisRun) -> Dataset:
|
||||||
|
"""Block an output export when its persisted source dataset is unsafe."""
|
||||||
|
|
||||||
|
if not run.dataset_id:
|
||||||
|
raise AppError(
|
||||||
|
code="DATASET_PROVENANCE_INCOMPLETE",
|
||||||
|
message="Analysis output cannot be exported without a persisted source dataset.",
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
|
dataset = db.get(Dataset, run.dataset_id)
|
||||||
|
if not dataset or dataset.project_id != run.project_id:
|
||||||
|
raise AppError(code="DATASET_NOT_FOUND", message="Analysis source dataset not found", status_code=404)
|
||||||
|
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
|
||||||
|
return dataset
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def export_map_result(
|
def export_map_result(
|
||||||
db: Session,
|
db: Session,
|
||||||
payload: MapResultExportRequest,
|
payload: MapResultExportRequest,
|
||||||
) -> ExportCreateResponse:
|
) -> ExportCreateResponse:
|
||||||
if payload.mode == "evolution":
|
if payload.mode == "evolution":
|
||||||
|
earlier_dataset = db.get(Dataset, payload.earlier_dataset_id)
|
||||||
|
later_dataset = db.get(Dataset, payload.later_dataset_id)
|
||||||
|
if (
|
||||||
|
not earlier_dataset
|
||||||
|
or not later_dataset
|
||||||
|
or earlier_dataset.project_id != payload.project_id
|
||||||
|
or later_dataset.project_id != payload.project_id
|
||||||
|
):
|
||||||
|
raise AppError(code="DATASET_NOT_FOUND", message="Temporal export dataset not found", status_code=404)
|
||||||
|
DatasetConsumptionGate.assert_eligible(earlier_dataset, purpose="export")
|
||||||
|
DatasetConsumptionGate.assert_eligible(later_dataset, purpose="export")
|
||||||
comparison = TemporalAnalysisService.compare(
|
comparison = TemporalAnalysisService.compare(
|
||||||
db,
|
db,
|
||||||
project_id=payload.project_id,
|
project_id=payload.project_id,
|
||||||
@@ -86,6 +114,7 @@ class ExportService:
|
|||||||
dataset = db.get(Dataset, payload.dataset_id)
|
dataset = db.get(Dataset, payload.dataset_id)
|
||||||
if not dataset or dataset.project_id != payload.project_id:
|
if not dataset or dataset.project_id != payload.project_id:
|
||||||
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
raise AppError(code="DATASET_NOT_FOUND", message="Dataset not found", status_code=404)
|
||||||
|
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
|
||||||
if dataset.dataset_type in DatasetService.VECTOR_TYPES:
|
if dataset.dataset_type in DatasetService.VECTOR_TYPES:
|
||||||
if payload.partitioned:
|
if payload.partitioned:
|
||||||
return ExportService.export_partitioned_vector_selection_geojson(
|
return ExportService.export_partitioned_vector_selection_geojson(
|
||||||
@@ -219,6 +248,7 @@ class ExportService:
|
|||||||
limit: int = 1000,
|
limit: int = 1000,
|
||||||
name: str | None = None,
|
name: str | None = None,
|
||||||
) -> ExportCreateResponse:
|
) -> ExportCreateResponse:
|
||||||
|
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
|
||||||
if dataset.source_name != "vmm_vha_bathymetry_profiles" or partition_scope_key != "flanders":
|
if dataset.source_name != "vmm_vha_bathymetry_profiles" or partition_scope_key != "flanders":
|
||||||
raise AppError(
|
raise AppError(
|
||||||
code="PARTITIONED_VECTOR_EXPORT_UNSUPPORTED",
|
code="PARTITIONED_VECTOR_EXPORT_UNSUPPORTED",
|
||||||
@@ -308,6 +338,7 @@ class ExportService:
|
|||||||
details={"dataset_type": dataset.dataset_type},
|
details={"dataset_type": dataset.dataset_type},
|
||||||
status_code=400,
|
status_code=400,
|
||||||
)
|
)
|
||||||
|
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
|
||||||
|
|
||||||
selection_kwargs: dict[str, Any] = {
|
selection_kwargs: dict[str, Any] = {
|
||||||
"dataset_id": dataset_id,
|
"dataset_id": dataset_id,
|
||||||
@@ -374,6 +405,7 @@ class ExportService:
|
|||||||
details={"dataset_type": dataset.dataset_type},
|
details={"dataset_type": dataset.dataset_type},
|
||||||
status_code=400,
|
status_code=400,
|
||||||
)
|
)
|
||||||
|
DatasetConsumptionGate.assert_eligible(dataset, purpose="export")
|
||||||
|
|
||||||
feature_collection = DatasetService.get_dataset_geojson(db, dataset_id)
|
feature_collection = DatasetService.get_dataset_geojson(db, dataset_id)
|
||||||
filename = ExportService._filename(name, f"{dataset.id}.geojson", ".geojson")
|
filename = ExportService._filename(name, f"{dataset.id}.geojson", ".geojson")
|
||||||
@@ -401,6 +433,7 @@ class ExportService:
|
|||||||
run = db.get(AnalysisRun, analysis_run_id)
|
run = db.get(AnalysisRun, analysis_run_id)
|
||||||
if not run or run.analysis_type != "detection":
|
if not run or run.analysis_type != "detection":
|
||||||
raise AppError(code="DETECTION_RUN_NOT_FOUND", message="Detection run not found", status_code=404)
|
raise AppError(code="DETECTION_RUN_NOT_FOUND", message="Detection run not found", status_code=404)
|
||||||
|
ExportService._assert_run_source_dataset_exportable(db, run)
|
||||||
|
|
||||||
feature_collection = DetectionService.detections_to_geojson(db, analysis_run_id=analysis_run_id)
|
feature_collection = DetectionService.detections_to_geojson(db, analysis_run_id=analysis_run_id)
|
||||||
filename = ExportService._filename(name, f"{run.id}-detections.geojson", ".geojson")
|
filename = ExportService._filename(name, f"{run.id}-detections.geojson", ".geojson")
|
||||||
@@ -428,6 +461,7 @@ class ExportService:
|
|||||||
run = db.get(AnalysisRun, analysis_run_id)
|
run = db.get(AnalysisRun, analysis_run_id)
|
||||||
if not run or run.analysis_type != "segmentation":
|
if not run or run.analysis_type != "segmentation":
|
||||||
raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404)
|
raise AppError(code="SEGMENTATION_RUN_NOT_FOUND", message="Segmentation run not found", status_code=404)
|
||||||
|
ExportService._assert_run_source_dataset_exportable(db, run)
|
||||||
|
|
||||||
feature_collection = SegmentationService.segmentations_to_geojson(db, analysis_run_id=analysis_run_id)
|
feature_collection = SegmentationService.segmentations_to_geojson(db, analysis_run_id=analysis_run_id)
|
||||||
filename = ExportService._filename(name, f"{run.id}-segmentations.geojson", ".geojson")
|
filename = ExportService._filename(name, f"{run.id}-segmentations.geojson", ".geojson")
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ from app.services.segmentation_adapter import (
|
|||||||
SamSegmentationAdapter,
|
SamSegmentationAdapter,
|
||||||
YoloSegmentationAdapter,
|
YoloSegmentationAdapter,
|
||||||
)
|
)
|
||||||
|
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||||
from app.services.yolo_adapter import YoloDetectionAdapter
|
from app.services.yolo_adapter import YoloDetectionAdapter
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
|
|
||||||
@@ -144,9 +145,24 @@ class ModelRegistryService:
|
|||||||
elif not model_path.exists() or not model_path.is_file():
|
elif not model_path.exists() or not model_path.is_file():
|
||||||
limitation = "YOLO_SEG_MODEL_PATH does not point to an existing local model file. GeoIntel will not download segmentation model weights automatically."
|
limitation = "YOLO_SEG_MODEL_PATH does not point to an existing local model file. GeoIntel will not download segmentation model weights automatically."
|
||||||
else:
|
else:
|
||||||
configured = True
|
try:
|
||||||
status = "configured"
|
RuntimeModelProvenanceService.validate_for_runtime(
|
||||||
limitation = "Configured for local YOLO segmentation inference over an existing raster tile manifest."
|
model_path=model_path,
|
||||||
|
model_id=settings.yolo_seg_model_id,
|
||||||
|
task_type="segmentation",
|
||||||
|
expected_model_version=settings.yolo_seg_model_version,
|
||||||
|
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
|
||||||
|
)
|
||||||
|
except AppError as exc:
|
||||||
|
status = "contract_incomplete"
|
||||||
|
limitation = (
|
||||||
|
"Configured YOLO segmentation weights are not runnable until their immutable "
|
||||||
|
f"runtime provenance sidecar validates: {exc.message}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
configured = True
|
||||||
|
status = "configured"
|
||||||
|
limitation = "Configured for local YOLO segmentation inference over an existing raster tile manifest."
|
||||||
|
|
||||||
return DetectionModelCapability(
|
return DetectionModelCapability(
|
||||||
model_id=settings.yolo_seg_model_id,
|
model_id=settings.yolo_seg_model_id,
|
||||||
@@ -186,9 +202,24 @@ class ModelRegistryService:
|
|||||||
elif not model_path.exists() or not model_path.is_file():
|
elif not model_path.exists() or not model_path.is_file():
|
||||||
limitation = "SAM_MODEL_PATH does not point to an existing local model file. GeoIntel will not download segmentation model weights automatically."
|
limitation = "SAM_MODEL_PATH does not point to an existing local model file. GeoIntel will not download segmentation model weights automatically."
|
||||||
else:
|
else:
|
||||||
configured = True
|
try:
|
||||||
status = "configured"
|
RuntimeModelProvenanceService.validate_for_runtime(
|
||||||
limitation = "Configured for local class-agnostic SAM segmentation over an existing raster tile manifest."
|
model_path=model_path,
|
||||||
|
model_id=settings.sam_model_id,
|
||||||
|
task_type="segmentation",
|
||||||
|
expected_model_version=settings.sam_model_version,
|
||||||
|
allowed_frameworks=("ultralytics/sam", "sam", "ultralytics", "pytorch"),
|
||||||
|
)
|
||||||
|
except AppError as exc:
|
||||||
|
status = "contract_incomplete"
|
||||||
|
limitation = (
|
||||||
|
"Configured SAM weights are not runnable until their immutable runtime provenance "
|
||||||
|
f"sidecar validates: {exc.message}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
configured = True
|
||||||
|
status = "configured"
|
||||||
|
limitation = "Configured for local class-agnostic SAM segmentation over an existing raster tile manifest."
|
||||||
|
|
||||||
return DetectionModelCapability(
|
return DetectionModelCapability(
|
||||||
model_id=settings.sam_model_id,
|
model_id=settings.sam_model_id,
|
||||||
@@ -233,9 +264,24 @@ class ModelRegistryService:
|
|||||||
status = "accelerator_unavailable"
|
status = "accelerator_unavailable"
|
||||||
limitation = exc.message
|
limitation = exc.message
|
||||||
else:
|
else:
|
||||||
configured = True
|
try:
|
||||||
status = "configured"
|
RuntimeModelProvenanceService.validate_for_runtime(
|
||||||
limitation = "Configured for local YOLO inference over an existing raster tile manifest within its validated area scope."
|
model_path=model_path,
|
||||||
|
model_id=settings.yolo_model_id,
|
||||||
|
task_type="object_detection",
|
||||||
|
expected_model_version=settings.yolo_model_version,
|
||||||
|
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
|
||||||
|
)
|
||||||
|
except AppError as exc:
|
||||||
|
status = "contract_incomplete"
|
||||||
|
limitation = (
|
||||||
|
"Configured YOLO weights are not runnable until their immutable runtime provenance "
|
||||||
|
f"sidecar validates: {exc.message}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
configured = True
|
||||||
|
status = "configured"
|
||||||
|
limitation = "Configured for local YOLO inference over an existing raster tile manifest within its validated area scope."
|
||||||
|
|
||||||
return DetectionModelCapability(
|
return DetectionModelCapability(
|
||||||
model_id=settings.yolo_model_id,
|
model_id=settings.yolo_model_id,
|
||||||
|
|||||||
@@ -11,10 +11,10 @@ from shapely.geometry.base import BaseGeometry
|
|||||||
from shapely.strtree import STRtree
|
from shapely.strtree import STRtree
|
||||||
from shapely.ops import unary_union
|
from shapely.ops import unary_union
|
||||||
from shapely.validation import make_valid
|
from shapely.validation import make_valid
|
||||||
from shapely.geometry import shape
|
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.models import Area, Dataset
|
from app.models import Area, Dataset
|
||||||
from app.schemas.qa import QaProviderComparisonResult
|
from app.schemas.qa import QaProviderComparisonResult
|
||||||
|
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||||
from app.services.vector_operations_service import VectorOperationsService
|
from app.services.vector_operations_service import VectorOperationsService
|
||||||
|
|
||||||
|
|
||||||
@@ -264,6 +264,12 @@ class QaService:
|
|||||||
reference_dataset_id,
|
reference_dataset_id,
|
||||||
expected_project_id=project_id,
|
expected_project_id=project_id,
|
||||||
)
|
)
|
||||||
|
DatasetConsumptionGate.assert_eligible(candidate_dataset, purpose="quality_assessment")
|
||||||
|
DatasetConsumptionGate.assert_eligible(
|
||||||
|
reference_dataset,
|
||||||
|
purpose="reference_validation",
|
||||||
|
reference_task="building_validation",
|
||||||
|
)
|
||||||
|
|
||||||
area_geometry = QaService._validate_area(
|
area_geometry = QaService._validate_area(
|
||||||
db,
|
db,
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ from shapely.validation import make_valid
|
|||||||
|
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.models import Area, Dataset, DatasetVersion
|
from app.models import Area, Dataset, DatasetVersion
|
||||||
|
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
|
||||||
from app.services.raster_service import extract_raster_metadata
|
from app.services.raster_service import extract_raster_metadata
|
||||||
from app.services.storage_service import StorageService
|
from app.services.storage_service import StorageService
|
||||||
|
|
||||||
@@ -334,7 +335,10 @@ class RasterOperationsService:
|
|||||||
dataset_type="raster",
|
dataset_type="raster",
|
||||||
source=f"operation:{operation_name}",
|
source=f"operation:{operation_name}",
|
||||||
dataset_role="derived",
|
dataset_role="derived",
|
||||||
source_name=source_dataset.source_name,
|
# The source identity of this artifact is the server-owned
|
||||||
|
# derived-operation registry entry. The parent remains explicit
|
||||||
|
# in provenance and the lineage edge below.
|
||||||
|
source_name="derived",
|
||||||
source_metadata=source_dataset.source_metadata,
|
source_metadata=source_dataset.source_metadata,
|
||||||
provenance_metadata=provenance,
|
provenance_metadata=provenance,
|
||||||
imported_at=datetime.now(timezone.utc),
|
imported_at=datetime.now(timezone.utc),
|
||||||
@@ -360,22 +364,31 @@ class RasterOperationsService:
|
|||||||
resolution_json=metadata_payload.get("resolution"),
|
resolution_json=metadata_payload.get("resolution"),
|
||||||
bands_json={"dtype": metadata_payload.get("dtype")} if metadata_payload.get("dtype") is not None else None,
|
bands_json={"dtype": metadata_payload.get("dtype")} if metadata_payload.get("dtype") is not None else None,
|
||||||
metadata_json=metadata_payload,
|
metadata_json=metadata_payload,
|
||||||
status="ready",
|
status="validating",
|
||||||
)
|
)
|
||||||
db.add(derived_dataset)
|
db.add(derived_dataset)
|
||||||
db.add(
|
dataset_version = DatasetVersion(
|
||||||
DatasetVersion(
|
dataset_id=derived_dataset.id,
|
||||||
dataset_id=derived_dataset.id,
|
version=1,
|
||||||
version=1,
|
storage_path=derived_dataset.storage_path,
|
||||||
storage_path=derived_dataset.storage_path,
|
source_version=derived_dataset.source_version,
|
||||||
source_version=derived_dataset.source_version,
|
observed_at=derived_dataset.observed_at,
|
||||||
observed_at=derived_dataset.observed_at,
|
valid_from=derived_dataset.valid_from,
|
||||||
valid_from=derived_dataset.valid_from,
|
valid_to=derived_dataset.valid_to,
|
||||||
valid_to=derived_dataset.valid_to,
|
checksum_sha256=derived_dataset.checksum_sha256,
|
||||||
checksum_sha256=derived_dataset.checksum_sha256,
|
source_metadata=derived_dataset.source_metadata,
|
||||||
source_metadata=derived_dataset.source_metadata,
|
provenance_metadata=derived_dataset.provenance_metadata,
|
||||||
provenance_metadata=derived_dataset.provenance_metadata,
|
)
|
||||||
)
|
db.add(dataset_version)
|
||||||
|
DerivedDatasetGovernanceService.govern_raster(
|
||||||
|
db,
|
||||||
|
dataset=derived_dataset,
|
||||||
|
dataset_version=dataset_version,
|
||||||
|
raster_metadata=metadata_payload,
|
||||||
|
source_key="derived",
|
||||||
|
operation=operation_name,
|
||||||
|
parent_dataset=source_dataset,
|
||||||
|
operation_parameters=metadata_payload.get("operation_parameters"),
|
||||||
)
|
)
|
||||||
db.commit()
|
db.commit()
|
||||||
db.refresh(derived_dataset)
|
db.refresh(derived_dataset)
|
||||||
|
|||||||
@@ -0,0 +1,652 @@
|
|||||||
|
"""Fail-closed provenance validation for configured local model files.
|
||||||
|
|
||||||
|
Configured YOLO and SAM weights are intentionally not trusted merely because a
|
||||||
|
file exists on the server. Before an adapter is allowed to load local weights,
|
||||||
|
this service verifies a neighbouring immutable sidecar manifest, validates the
|
||||||
|
model artifact against ``geointel.model.pytorch@1.0.0`` and binds that sidecar
|
||||||
|
to the server-owned source registry and source snapshot recorded in Postgres.
|
||||||
|
|
||||||
|
``validate_for_runtime`` remains a structural sidecar check for catalog and
|
||||||
|
preflight inspection. Production inference must call
|
||||||
|
``validate_for_production_runtime`` with a database session; it rejects an
|
||||||
|
unregistered, mismatched, stale, unsafe or quarantined source snapshot before
|
||||||
|
an adapter can load model bytes. This keeps focused unit tests able to inspect
|
||||||
|
sidecars without inventing database rows while keeping the production boundary
|
||||||
|
strict.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from hashlib import sha256
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
from typing import Any, Mapping
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.models import SourceRegistry, SourceSnapshot
|
||||||
|
from app.services.data_contract_validation import (
|
||||||
|
PYTORCH_MODEL_CONTRACT_KEY,
|
||||||
|
PYTORCH_MODEL_CONTRACT_VERSION,
|
||||||
|
LineageEvidence,
|
||||||
|
TransformationEvidence,
|
||||||
|
build_model_validation_input,
|
||||||
|
validate_registered_asset,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
_SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||||
|
_CONSUMABLE_FRESHNESS = {"current", "not_applicable"}
|
||||||
|
_SAFE_SOURCE_REGISTRY_INGEST_STATUSES = {"configured", "ingested"}
|
||||||
|
_SAFE_SOURCE_SNAPSHOT_INGEST_STATUS = "ingested"
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RuntimeModelProvenance:
|
||||||
|
"""Validated, immutable evidence attached to one local inference run."""
|
||||||
|
|
||||||
|
model_id: str
|
||||||
|
task_type: str
|
||||||
|
model_path: str
|
||||||
|
manifest_path: str
|
||||||
|
model_sha256: str
|
||||||
|
manifest_sha256: str
|
||||||
|
runtime_manifest_sha256: str
|
||||||
|
data_contract_key: str
|
||||||
|
data_contract_version: str
|
||||||
|
validation_report_sha256: str
|
||||||
|
source_registry_id: str
|
||||||
|
source_snapshot_id: str
|
||||||
|
source_snapshot_checksum_sha256: str
|
||||||
|
source_version: str
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, str]:
|
||||||
|
return {
|
||||||
|
"model_id": self.model_id,
|
||||||
|
"task_type": self.task_type,
|
||||||
|
"model_path": self.model_path,
|
||||||
|
"manifest_path": self.manifest_path,
|
||||||
|
"model_sha256": self.model_sha256,
|
||||||
|
"manifest_sha256": self.manifest_sha256,
|
||||||
|
"runtime_manifest_sha256": self.runtime_manifest_sha256,
|
||||||
|
"data_contract_key": self.data_contract_key,
|
||||||
|
"data_contract_version": self.data_contract_version,
|
||||||
|
"validation_report_sha256": self.validation_report_sha256,
|
||||||
|
"source_registry_id": self.source_registry_id,
|
||||||
|
"source_snapshot_id": self.source_snapshot_id,
|
||||||
|
"source_snapshot_checksum_sha256": self.source_snapshot_checksum_sha256,
|
||||||
|
"source_version": self.source_version,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class RuntimeModelProvenanceService:
|
||||||
|
"""Validate model sidecars and their production database binding.
|
||||||
|
|
||||||
|
A sidecar lives next to its model as ``<model-file>.geointel-model.json``.
|
||||||
|
Its ``metadata.runtime_manifest_sha256`` is the SHA-256 of canonical JSON
|
||||||
|
after omitting that one self-referential field. Any other mutation of the
|
||||||
|
manifest therefore invalidates it. Structural validation is deliberately
|
||||||
|
separate from :meth:`validate_for_production_runtime`: discovery and
|
||||||
|
preflight have no database session, whereas every production inference
|
||||||
|
path must prove an active source registry/snapshot binding.
|
||||||
|
"""
|
||||||
|
|
||||||
|
MANIFEST_SUFFIX = ".geointel-model.json"
|
||||||
|
MANIFEST_SCHEMA_VERSION = "geointel.runtime-model-manifest/v1"
|
||||||
|
SOURCE_REGISTRY_KEY = "model"
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def manifest_path_for_model(cls, model_path: str | Path) -> Path:
|
||||||
|
path = Path(model_path).expanduser()
|
||||||
|
return Path(f"{path}{cls.MANIFEST_SUFFIX}")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def manifest_self_checksum(payload: Mapping[str, Any]) -> str:
|
||||||
|
"""Hash sidecar semantics without its self-referential checksum field."""
|
||||||
|
|
||||||
|
canonical_payload = json.loads(json.dumps(payload, sort_keys=True, ensure_ascii=True))
|
||||||
|
metadata = canonical_payload.get("metadata")
|
||||||
|
if isinstance(metadata, dict):
|
||||||
|
metadata.pop("runtime_manifest_sha256", None)
|
||||||
|
encoded = json.dumps(
|
||||||
|
canonical_payload,
|
||||||
|
sort_keys=True,
|
||||||
|
separators=(",", ":"),
|
||||||
|
ensure_ascii=True,
|
||||||
|
).encode("utf-8")
|
||||||
|
return sha256(encoded).hexdigest()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def validate_for_runtime(
|
||||||
|
cls,
|
||||||
|
*,
|
||||||
|
model_path: str | Path,
|
||||||
|
model_id: str,
|
||||||
|
task_type: str,
|
||||||
|
expected_model_version: str | None = None,
|
||||||
|
allowed_frameworks: tuple[str, ...] = (),
|
||||||
|
) -> RuntimeModelProvenance:
|
||||||
|
"""Return structural sidecar evidence without asserting database state.
|
||||||
|
|
||||||
|
This method is appropriate for read-only catalog/preflight checks and
|
||||||
|
focused sidecar unit tests. It is insufficient for production
|
||||||
|
inference; adapters must use :meth:`validate_for_production_runtime`.
|
||||||
|
"""
|
||||||
|
|
||||||
|
path = Path(model_path).expanduser()
|
||||||
|
if not path.exists() or not path.is_file():
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_MODEL_FILE_MISSING",
|
||||||
|
"Configured model file is missing; runtime provenance cannot be verified.",
|
||||||
|
model_path=str(path),
|
||||||
|
)
|
||||||
|
manifest_path = cls.manifest_path_for_model(path)
|
||||||
|
if not manifest_path.exists() or not manifest_path.is_file():
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_MANIFEST_MISSING",
|
||||||
|
"Configured model requires an immutable .geointel-model.json sidecar before inference.",
|
||||||
|
model_path=str(path),
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
raw_manifest = manifest_path.read_bytes()
|
||||||
|
payload = json.loads(raw_manifest.decode("utf-8"))
|
||||||
|
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as exc:
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||||
|
"Configured model sidecar must be a readable UTF-8 JSON object.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
error_type=type(exc).__name__,
|
||||||
|
)
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||||
|
"Configured model sidecar must contain a JSON object.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
)
|
||||||
|
|
||||||
|
cls._require_exact_text(
|
||||||
|
payload.get("schema_version"),
|
||||||
|
cls.MANIFEST_SCHEMA_VERSION,
|
||||||
|
field="schema_version",
|
||||||
|
manifest_path=manifest_path,
|
||||||
|
)
|
||||||
|
contract = cls._require_mapping(payload, "data_contract", manifest_path)
|
||||||
|
contract_key = cls._require_text(contract, "key", manifest_path)
|
||||||
|
contract_version = cls._require_text(contract, "version", manifest_path)
|
||||||
|
# A valid manifest for a different artifact family must never make a
|
||||||
|
# local PyTorch/SAM weight executable. The structural validator below
|
||||||
|
# has a registry lookup too, but pinning the identity here keeps this
|
||||||
|
# runtime gate fail-closed if more model contracts are introduced.
|
||||||
|
cls._require_exact_text(
|
||||||
|
contract_key,
|
||||||
|
PYTORCH_MODEL_CONTRACT_KEY,
|
||||||
|
field="data_contract.key",
|
||||||
|
manifest_path=manifest_path,
|
||||||
|
)
|
||||||
|
cls._require_exact_text(
|
||||||
|
contract_version,
|
||||||
|
PYTORCH_MODEL_CONTRACT_VERSION,
|
||||||
|
field="data_contract.version",
|
||||||
|
manifest_path=manifest_path,
|
||||||
|
)
|
||||||
|
model = cls._require_mapping(payload, "model", manifest_path)
|
||||||
|
declared_model_id = cls._require_text(model, "model_id", manifest_path)
|
||||||
|
declared_task_type = cls._require_text(model, "task_type", manifest_path)
|
||||||
|
cls._require_exact_text(declared_model_id, model_id, field="model.model_id", manifest_path=manifest_path)
|
||||||
|
cls._require_exact_text(declared_task_type, task_type, field="model.task_type", manifest_path=manifest_path)
|
||||||
|
|
||||||
|
declared_checksum = cls._require_checksum(model.get("sha256"), "model.sha256", manifest_path)
|
||||||
|
model_checksum = cls._file_sha256(path)
|
||||||
|
if declared_checksum != model_checksum:
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_MODEL_CHECKSUM_MISMATCH",
|
||||||
|
"Model bytes do not match the checksum bound by the runtime sidecar.",
|
||||||
|
model_path=str(path),
|
||||||
|
expected=declared_checksum,
|
||||||
|
observed=model_checksum,
|
||||||
|
)
|
||||||
|
|
||||||
|
model_format = cls._require_text(model, "model_format", manifest_path)
|
||||||
|
framework = cls._require_text(model, "framework", manifest_path)
|
||||||
|
class_mapping = model.get("class_mapping")
|
||||||
|
if not isinstance(class_mapping, (dict, list, tuple)) or not class_mapping:
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||||
|
"model.class_mapping must be a non-empty mapping or sequence.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
)
|
||||||
|
normalized_framework = framework.strip().lower()
|
||||||
|
if allowed_frameworks and normalized_framework not in {value.strip().lower() for value in allowed_frameworks}:
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_FRAMEWORK_MISMATCH",
|
||||||
|
"Model framework does not match the configured runtime adapter.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
expected=sorted({value.strip().lower() for value in allowed_frameworks}),
|
||||||
|
observed=framework,
|
||||||
|
)
|
||||||
|
source_version = cls._require_text(model, "source_version", manifest_path)
|
||||||
|
if expected_model_version and source_version != expected_model_version:
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_VERSION_MISMATCH",
|
||||||
|
"Model sidecar version does not match the configured model version.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
expected=expected_model_version,
|
||||||
|
observed=source_version,
|
||||||
|
)
|
||||||
|
|
||||||
|
source = cls._require_mapping(payload, "source", manifest_path)
|
||||||
|
source_registry_id = cls._require_uuid(source.get("source_registry_id"), "source.source_registry_id", manifest_path)
|
||||||
|
source_snapshot_id = cls._require_uuid(source.get("source_snapshot_id"), "source.source_snapshot_id", manifest_path)
|
||||||
|
cls._require_exact_text(
|
||||||
|
cls._require_text(source, "source_registry_key", manifest_path),
|
||||||
|
cls.SOURCE_REGISTRY_KEY,
|
||||||
|
field="source.source_registry_key",
|
||||||
|
manifest_path=manifest_path,
|
||||||
|
)
|
||||||
|
snapshot_checksum = cls._require_checksum(
|
||||||
|
source.get("source_snapshot_checksum_sha256"),
|
||||||
|
"source.source_snapshot_checksum_sha256",
|
||||||
|
manifest_path,
|
||||||
|
)
|
||||||
|
if snapshot_checksum != model_checksum:
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_SNAPSHOT_CHECKSUM_MISMATCH",
|
||||||
|
"Model source snapshot checksum must bind the exact model bytes.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
expected=model_checksum,
|
||||||
|
observed=snapshot_checksum,
|
||||||
|
)
|
||||||
|
|
||||||
|
metadata = cls._require_mapping(payload, "metadata", manifest_path)
|
||||||
|
training_manifest_sha256 = cls._require_checksum(
|
||||||
|
metadata.get("training_manifest_sha256"),
|
||||||
|
"metadata.training_manifest_sha256",
|
||||||
|
manifest_path,
|
||||||
|
)
|
||||||
|
declared_runtime_manifest_sha256 = cls._require_checksum(
|
||||||
|
metadata.get("runtime_manifest_sha256"),
|
||||||
|
"metadata.runtime_manifest_sha256",
|
||||||
|
manifest_path,
|
||||||
|
)
|
||||||
|
computed_runtime_manifest_sha256 = cls.manifest_self_checksum(payload)
|
||||||
|
if declared_runtime_manifest_sha256 != computed_runtime_manifest_sha256:
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_MANIFEST_CHECKSUM_MISMATCH",
|
||||||
|
"Runtime model sidecar integrity checksum does not match its canonical contents.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
expected=declared_runtime_manifest_sha256,
|
||||||
|
observed=computed_runtime_manifest_sha256,
|
||||||
|
)
|
||||||
|
|
||||||
|
lineage = cls._lineage_evidence(payload, manifest_path)
|
||||||
|
imported_at = cls._parse_imported_at(payload.get("imported_at"), manifest_path)
|
||||||
|
report = validate_registered_asset(
|
||||||
|
build_model_validation_input(
|
||||||
|
asset_id=f"{model_id}:{model_checksum}",
|
||||||
|
model_metadata={
|
||||||
|
"model_format": model_format,
|
||||||
|
"framework": framework,
|
||||||
|
"class_mapping": class_mapping,
|
||||||
|
},
|
||||||
|
checksum_sha256=declared_checksum,
|
||||||
|
computed_checksum_sha256=model_checksum,
|
||||||
|
source_registry_id=source_registry_id,
|
||||||
|
source_snapshot_id=source_snapshot_id,
|
||||||
|
imported_at=imported_at,
|
||||||
|
metadata={
|
||||||
|
"training_manifest_sha256": training_manifest_sha256,
|
||||||
|
"runtime_manifest_sha256": declared_runtime_manifest_sha256,
|
||||||
|
},
|
||||||
|
source_version=source_version,
|
||||||
|
lineage=lineage,
|
||||||
|
data_contract_key=contract_key,
|
||||||
|
data_contract_version=contract_version,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if report.failed:
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_CONTRACT_FAILED",
|
||||||
|
"Configured model sidecar failed the exact versioned model data contract.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
data_contract=f"{contract_key}@{contract_version}",
|
||||||
|
issue_codes=[issue.code for issue in report.issues],
|
||||||
|
validation_report_sha256=report.report_sha256,
|
||||||
|
)
|
||||||
|
return RuntimeModelProvenance(
|
||||||
|
model_id=model_id,
|
||||||
|
task_type=task_type,
|
||||||
|
model_path=str(path.resolve()),
|
||||||
|
manifest_path=str(manifest_path.resolve()),
|
||||||
|
model_sha256=model_checksum,
|
||||||
|
manifest_sha256=sha256(raw_manifest).hexdigest(),
|
||||||
|
runtime_manifest_sha256=declared_runtime_manifest_sha256,
|
||||||
|
data_contract_key=contract_key,
|
||||||
|
data_contract_version=contract_version,
|
||||||
|
validation_report_sha256=report.report_sha256,
|
||||||
|
source_registry_id=source_registry_id,
|
||||||
|
source_snapshot_id=source_snapshot_id,
|
||||||
|
source_snapshot_checksum_sha256=snapshot_checksum,
|
||||||
|
source_version=source_version,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def validate_for_production_runtime(
|
||||||
|
cls,
|
||||||
|
*,
|
||||||
|
db: Any,
|
||||||
|
model_path: str | Path,
|
||||||
|
model_id: str,
|
||||||
|
task_type: str,
|
||||||
|
expected_model_version: str | None = None,
|
||||||
|
allowed_frameworks: tuple[str, ...] = (),
|
||||||
|
) -> RuntimeModelProvenance:
|
||||||
|
"""Validate byte-bound model evidence against governed database state.
|
||||||
|
|
||||||
|
The sidecar is not itself a source of authority. The model bytes may
|
||||||
|
enter production inference only when the declared source registry and
|
||||||
|
immutable source snapshot both exist, belong together, are safe to
|
||||||
|
consume and bind the same SHA-256 and source version as the sidecar.
|
||||||
|
This check is intentionally invoked immediately before adapter loading.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if db is None or not callable(getattr(db, "get", None)):
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_DATABASE_REQUIRED",
|
||||||
|
"Production model inference requires a database session for source snapshot provenance.",
|
||||||
|
)
|
||||||
|
|
||||||
|
evidence = cls.validate_for_runtime(
|
||||||
|
model_path=model_path,
|
||||||
|
model_id=model_id,
|
||||||
|
task_type=task_type,
|
||||||
|
expected_model_version=expected_model_version,
|
||||||
|
allowed_frameworks=allowed_frameworks,
|
||||||
|
)
|
||||||
|
cls._assert_database_binding(db, evidence)
|
||||||
|
return evidence
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _assert_database_binding(cls, db: Any, evidence: RuntimeModelProvenance) -> None:
|
||||||
|
registry_id = UUID(evidence.source_registry_id)
|
||||||
|
snapshot_id = UUID(evidence.source_snapshot_id)
|
||||||
|
source_registry = db.get(SourceRegistry, registry_id)
|
||||||
|
if not isinstance(source_registry, SourceRegistry):
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND",
|
||||||
|
"Configured model sidecar refers to a source registry record that does not exist.",
|
||||||
|
source_registry_id=evidence.source_registry_id,
|
||||||
|
model_id=evidence.model_id,
|
||||||
|
)
|
||||||
|
source_snapshot = db.get(SourceSnapshot, snapshot_id)
|
||||||
|
if not isinstance(source_snapshot, SourceSnapshot):
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_NOT_FOUND",
|
||||||
|
"Configured model sidecar refers to a source snapshot record that does not exist.",
|
||||||
|
source_snapshot_id=evidence.source_snapshot_id,
|
||||||
|
model_id=evidence.model_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
if str(source_registry.id) != evidence.source_registry_id or source_registry.source_key != cls.SOURCE_REGISTRY_KEY:
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_SOURCE_REGISTRY_IDENTITY_MISMATCH",
|
||||||
|
"Database source registry does not match the immutable model sidecar identity.",
|
||||||
|
expected_source_registry_id=evidence.source_registry_id,
|
||||||
|
observed_source_registry_id=str(source_registry.id),
|
||||||
|
expected_source_key=cls.SOURCE_REGISTRY_KEY,
|
||||||
|
observed_source_key=source_registry.source_key,
|
||||||
|
)
|
||||||
|
if str(source_snapshot.id) != evidence.source_snapshot_id or str(source_snapshot.source_registry_id) != evidence.source_registry_id:
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_REGISTRY_MISMATCH",
|
||||||
|
"Model source snapshot does not belong to the declared source registry.",
|
||||||
|
source_registry_id=evidence.source_registry_id,
|
||||||
|
source_snapshot_id=evidence.source_snapshot_id,
|
||||||
|
observed_snapshot_registry_id=str(source_snapshot.source_registry_id),
|
||||||
|
)
|
||||||
|
|
||||||
|
registry_ingest_status = cls._normalise_status(source_registry.ingest_status)
|
||||||
|
registry_freshness_status = cls._normalise_status(source_registry.freshness_status)
|
||||||
|
if (
|
||||||
|
registry_ingest_status not in _SAFE_SOURCE_REGISTRY_INGEST_STATUSES
|
||||||
|
or registry_freshness_status not in _CONSUMABLE_FRESHNESS
|
||||||
|
):
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_SOURCE_REGISTRY_UNSAFE",
|
||||||
|
"Configured model source registry is not in a safe configured/current state.",
|
||||||
|
source_registry_id=evidence.source_registry_id,
|
||||||
|
ingest_status=registry_ingest_status,
|
||||||
|
freshness_status=registry_freshness_status,
|
||||||
|
)
|
||||||
|
|
||||||
|
snapshot_ingest_status = cls._normalise_status(source_snapshot.ingest_status)
|
||||||
|
snapshot_freshness_status = cls._normalise_status(source_snapshot.freshness_status)
|
||||||
|
if (
|
||||||
|
snapshot_ingest_status != _SAFE_SOURCE_SNAPSHOT_INGEST_STATUS
|
||||||
|
or snapshot_freshness_status not in _CONSUMABLE_FRESHNESS
|
||||||
|
):
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_UNSAFE",
|
||||||
|
"Configured model source snapshot is not an ingested current immutable artifact.",
|
||||||
|
source_snapshot_id=evidence.source_snapshot_id,
|
||||||
|
ingest_status=snapshot_ingest_status,
|
||||||
|
freshness_status=snapshot_freshness_status,
|
||||||
|
)
|
||||||
|
|
||||||
|
snapshot_checksum = str(source_snapshot.checksum_sha256 or "").strip().lower()
|
||||||
|
if snapshot_checksum != evidence.source_snapshot_checksum_sha256 or snapshot_checksum != evidence.model_sha256:
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_DATABASE_SNAPSHOT_CHECKSUM_MISMATCH",
|
||||||
|
"Database model source snapshot checksum does not bind the exact sidecar and model bytes.",
|
||||||
|
source_snapshot_id=evidence.source_snapshot_id,
|
||||||
|
expected_model_sha256=evidence.model_sha256,
|
||||||
|
expected_sidecar_snapshot_sha256=evidence.source_snapshot_checksum_sha256,
|
||||||
|
observed_snapshot_sha256=snapshot_checksum,
|
||||||
|
)
|
||||||
|
|
||||||
|
snapshot_version = str(source_snapshot.source_version or "").strip()
|
||||||
|
if snapshot_version != evidence.source_version:
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_VERSION_MISMATCH",
|
||||||
|
"Database model source snapshot version does not match the immutable sidecar model version.",
|
||||||
|
source_snapshot_id=evidence.source_snapshot_id,
|
||||||
|
expected_source_version=evidence.source_version,
|
||||||
|
observed_source_version=snapshot_version or None,
|
||||||
|
)
|
||||||
|
|
||||||
|
# The persisted snapshot state is the primary quarantine signal. The
|
||||||
|
# relationship check is a defensive second line for an active
|
||||||
|
# quarantine record that predates or bypassed a status transition.
|
||||||
|
active_quarantines = getattr(source_snapshot, "quarantines", ())
|
||||||
|
if any(cls._normalise_status(getattr(item, "status", None)) == "quarantined" for item in active_quarantines):
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_SOURCE_SNAPSHOT_QUARANTINED",
|
||||||
|
"Configured model source snapshot has an active quarantine record.",
|
||||||
|
source_snapshot_id=evidence.source_snapshot_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _normalise_status(value: Any) -> str:
|
||||||
|
return str(value or "").strip().lower()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _lineage_evidence(cls, payload: Mapping[str, Any], manifest_path: Path) -> LineageEvidence:
|
||||||
|
lineage = cls._require_mapping(payload, "lineage", manifest_path)
|
||||||
|
raw_asset_ids = lineage.get("upstream_asset_ids")
|
||||||
|
raw_checksums = lineage.get("upstream_checksums_sha256")
|
||||||
|
raw_transformations = lineage.get("transformations")
|
||||||
|
if not isinstance(raw_asset_ids, list) or not raw_asset_ids or not all(
|
||||||
|
isinstance(value, str) and value.strip() for value in raw_asset_ids
|
||||||
|
):
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||||
|
"lineage.upstream_asset_ids must be a non-empty string list.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
)
|
||||||
|
if not isinstance(raw_checksums, list) or len(raw_checksums) != len(raw_asset_ids):
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||||
|
"lineage.upstream_checksums_sha256 must match upstream_asset_ids one-for-one.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
)
|
||||||
|
upstream_checksums = tuple(
|
||||||
|
cls._require_checksum(value, f"lineage.upstream_checksums_sha256[{index}]", manifest_path)
|
||||||
|
for index, value in enumerate(raw_checksums)
|
||||||
|
)
|
||||||
|
if not isinstance(raw_transformations, list) or not raw_transformations:
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||||
|
"lineage.transformations must contain at least one immutable transformation record.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
)
|
||||||
|
transformations: list[TransformationEvidence] = []
|
||||||
|
for index, raw in enumerate(raw_transformations):
|
||||||
|
if not isinstance(raw, Mapping):
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||||
|
"Each lineage transformation must be an object.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
index=index,
|
||||||
|
)
|
||||||
|
transformations.append(
|
||||||
|
TransformationEvidence(
|
||||||
|
name=cls._require_text(raw, "name", manifest_path, prefix=f"lineage.transformations[{index}]."),
|
||||||
|
version=cls._require_text(raw, "version", manifest_path, prefix=f"lineage.transformations[{index}]."),
|
||||||
|
checksum_sha256=cls._require_checksum(
|
||||||
|
raw.get("checksum_sha256"),
|
||||||
|
f"lineage.transformations[{index}].checksum_sha256",
|
||||||
|
manifest_path,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return LineageEvidence(
|
||||||
|
upstream_asset_ids=tuple(raw_asset_ids),
|
||||||
|
upstream_checksums_sha256=upstream_checksums,
|
||||||
|
transformations=tuple(transformations),
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _file_sha256(path: Path) -> str:
|
||||||
|
digest = sha256()
|
||||||
|
try:
|
||||||
|
with path.open("rb") as handle:
|
||||||
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
except OSError as exc:
|
||||||
|
RuntimeModelProvenanceService._raise(
|
||||||
|
"MODEL_PROVENANCE_MODEL_FILE_UNREADABLE",
|
||||||
|
"Configured model file could not be read for checksum validation.",
|
||||||
|
model_path=str(path),
|
||||||
|
error_type=type(exc).__name__,
|
||||||
|
)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _parse_imported_at(cls, value: Any, manifest_path: Path) -> datetime:
|
||||||
|
if not isinstance(value, str) or not value.strip():
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||||
|
"imported_at must be a timezone-aware ISO-8601 timestamp.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||||
|
except ValueError:
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||||
|
"imported_at must be a timezone-aware ISO-8601 timestamp.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
observed=value,
|
||||||
|
)
|
||||||
|
if parsed.tzinfo is None:
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||||
|
"imported_at must include a timezone offset.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
observed=value,
|
||||||
|
)
|
||||||
|
return parsed.astimezone(timezone.utc)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _require_mapping(cls, payload: Mapping[str, Any], key: str, manifest_path: Path) -> Mapping[str, Any]:
|
||||||
|
value = payload.get(key)
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||||
|
f"{key} must be a JSON object.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
)
|
||||||
|
return value
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _require_text(
|
||||||
|
cls,
|
||||||
|
payload: Mapping[str, Any],
|
||||||
|
key: str,
|
||||||
|
manifest_path: Path,
|
||||||
|
*,
|
||||||
|
prefix: str = "",
|
||||||
|
) -> str:
|
||||||
|
value = payload.get(key)
|
||||||
|
if not isinstance(value, str) or not value.strip():
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||||
|
f"{prefix}{key} must be a non-empty string.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
)
|
||||||
|
return value.strip()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _require_checksum(cls, value: Any, field: str, manifest_path: Path) -> str:
|
||||||
|
normalized = value.strip().lower() if isinstance(value, str) else ""
|
||||||
|
if not _SHA256.fullmatch(normalized) or value != normalized:
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||||
|
f"{field} must be a lowercase SHA-256 digest.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
)
|
||||||
|
return normalized
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _require_uuid(cls, value: Any, field: str, manifest_path: Path) -> str:
|
||||||
|
if not isinstance(value, str) or not value.strip():
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||||
|
f"{field} must be a UUID string.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
return str(UUID(value))
|
||||||
|
except (AttributeError, ValueError):
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||||
|
f"{field} must be a UUID string.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
observed=value,
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _require_exact_text(
|
||||||
|
cls,
|
||||||
|
observed: Any,
|
||||||
|
expected: str,
|
||||||
|
*,
|
||||||
|
field: str,
|
||||||
|
manifest_path: Path,
|
||||||
|
) -> None:
|
||||||
|
if not isinstance(observed, str) or observed.strip() != expected:
|
||||||
|
cls._raise(
|
||||||
|
"MODEL_PROVENANCE_MANIFEST_INVALID",
|
||||||
|
f"{field} does not match the configured runtime identity.",
|
||||||
|
manifest_path=str(manifest_path),
|
||||||
|
expected=expected,
|
||||||
|
observed=observed,
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _raise(code: str, message: str, **details: Any) -> None:
|
||||||
|
raise AppError(code=code, message=message, details=details, status_code=422)
|
||||||
@@ -21,9 +21,11 @@ from app.schemas.segmentation import (
|
|||||||
)
|
)
|
||||||
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
|
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
|
||||||
from app.services.detection_service import DetectionService
|
from app.services.detection_service import DetectionService
|
||||||
|
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||||
from app.services.model_registry_service import ModelRegistryService
|
from app.services.model_registry_service import ModelRegistryService
|
||||||
from app.services.qa_service import QaService
|
from app.services.qa_service import QaService
|
||||||
from app.services.quality_service import QualityService
|
from app.services.quality_service import QualityService
|
||||||
|
from app.services.runtime_model_provenance_service import RuntimeModelProvenance, RuntimeModelProvenanceService
|
||||||
from app.services.segmentation_adapter import (
|
from app.services.segmentation_adapter import (
|
||||||
FixtureSegmentationAdapter,
|
FixtureSegmentationAdapter,
|
||||||
SamSegmentationAdapter,
|
SamSegmentationAdapter,
|
||||||
@@ -89,6 +91,18 @@ class SegmentationService:
|
|||||||
status_code=400,
|
status_code=400,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Production segmentation must consume only a passed, complete and
|
||||||
|
# non-quarantined dataset. The fixture segmenter is QA/test-only and
|
||||||
|
# cannot be classified as production inference.
|
||||||
|
if model.model_id == "fixture-segmenter":
|
||||||
|
DatasetConsumptionGate.assert_eligible(
|
||||||
|
dataset,
|
||||||
|
purpose="quality_assessment",
|
||||||
|
fixture_mode=True,
|
||||||
|
)
|
||||||
|
elif model.model_id in configured_model_ids and model.configured:
|
||||||
|
DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
|
||||||
|
|
||||||
run_parameters = {
|
run_parameters = {
|
||||||
"model_id": model.model_id,
|
"model_id": model.model_id,
|
||||||
"confidence_threshold": confidence_threshold,
|
"confidence_threshold": confidence_threshold,
|
||||||
@@ -326,6 +340,27 @@ class SegmentationService:
|
|||||||
if reference_dataset.dataset_type not in {"vector", "geojson"}:
|
if reference_dataset.dataset_type not in {"vector", "geojson"}:
|
||||||
raise AppError(code="INVALID_DATASET_TYPE", message="Reference dataset must be vector data", status_code=400)
|
raise AppError(code="INVALID_DATASET_TYPE", message="Reference dataset must be vector data", status_code=400)
|
||||||
|
|
||||||
|
candidate_dataset = db.get(Dataset, run.dataset_id)
|
||||||
|
if not candidate_dataset:
|
||||||
|
raise AppError(code="DATASET_NOT_FOUND", message="Segmentation source dataset not found", status_code=404)
|
||||||
|
run_parameters = run.parameters_json if isinstance(run.parameters_json, dict) else {}
|
||||||
|
fixture_parameters = run_parameters.get("parameters_json")
|
||||||
|
fixture_mode = bool(
|
||||||
|
run.model_name == "fixture-segmenter"
|
||||||
|
and isinstance(fixture_parameters, dict)
|
||||||
|
and fixture_parameters.get("fixture_mode") is True
|
||||||
|
)
|
||||||
|
DatasetConsumptionGate.assert_eligible(
|
||||||
|
candidate_dataset,
|
||||||
|
purpose="quality_assessment",
|
||||||
|
fixture_mode=fixture_mode,
|
||||||
|
)
|
||||||
|
DatasetConsumptionGate.assert_eligible(
|
||||||
|
reference_dataset,
|
||||||
|
purpose="reference_validation",
|
||||||
|
reference_task="building_validation",
|
||||||
|
)
|
||||||
|
|
||||||
segmentations = SegmentationService._query_segmentation_rows(
|
segmentations = SegmentationService._query_segmentation_rows(
|
||||||
db,
|
db,
|
||||||
analysis_run_id=analysis_run_id,
|
analysis_run_id=analysis_run_id,
|
||||||
@@ -510,11 +545,26 @@ class SegmentationService:
|
|||||||
) -> tuple[list[Segmentation], dict[str, Any]]:
|
) -> tuple[list[Segmentation], dict[str, Any]]:
|
||||||
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles)
|
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles)
|
||||||
if model_name == settings.sam_model_id:
|
if model_name == settings.sam_model_id:
|
||||||
adapter = sam_adapter_class(settings)
|
|
||||||
model_path = Path(settings.sam_model_path or "").expanduser()
|
model_path = Path(settings.sam_model_path or "").expanduser()
|
||||||
|
allowed_frameworks = ("ultralytics/sam", "sam", "ultralytics", "pytorch")
|
||||||
|
adapter = sam_adapter_class(settings)
|
||||||
else:
|
else:
|
||||||
adapter = yolo_seg_adapter_class(settings)
|
|
||||||
model_path = Path(settings.yolo_seg_model_path or "").expanduser()
|
model_path = Path(settings.yolo_seg_model_path or "").expanduser()
|
||||||
|
allowed_frameworks = ("ultralytics/pytorch", "ultralytics", "pytorch")
|
||||||
|
adapter = yolo_seg_adapter_class(settings)
|
||||||
|
runtime_model_provenance = RuntimeModelProvenanceService.validate_for_production_runtime(
|
||||||
|
db=db,
|
||||||
|
model_path=model_path,
|
||||||
|
model_id=model_name,
|
||||||
|
task_type="segmentation",
|
||||||
|
expected_model_version=model_version,
|
||||||
|
allowed_frameworks=allowed_frameworks,
|
||||||
|
)
|
||||||
|
SegmentationService._attach_runtime_model_provenance(
|
||||||
|
analysis_run,
|
||||||
|
job,
|
||||||
|
runtime_model_provenance,
|
||||||
|
)
|
||||||
model = adapter.load_model(model_path)
|
model = adapter.load_model(model_path)
|
||||||
|
|
||||||
allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)}
|
allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)}
|
||||||
@@ -591,6 +641,7 @@ class SegmentationService:
|
|||||||
"tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()),
|
"tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()),
|
||||||
"tile_index": candidate["tile_index"],
|
"tile_index": candidate["tile_index"],
|
||||||
"device": settings.yolo_device,
|
"device": settings.yolo_device,
|
||||||
|
"runtime_model_provenance": runtime_model_provenance.as_dict(),
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
db.add(segmentation)
|
db.add(segmentation)
|
||||||
@@ -603,8 +654,25 @@ class SegmentationService:
|
|||||||
"suppressed_segmentation_count": len(candidates) - len(filtered_candidates),
|
"suppressed_segmentation_count": len(candidates) - len(filtered_candidates),
|
||||||
"duplicate_iou_threshold": float(settings.segmentation_duplicate_iou_threshold),
|
"duplicate_iou_threshold": float(settings.segmentation_duplicate_iou_threshold),
|
||||||
"tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()),
|
"tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()),
|
||||||
|
"runtime_model_provenance": runtime_model_provenance.as_dict(),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _attach_runtime_model_provenance(
|
||||||
|
analysis_run: AnalysisRun,
|
||||||
|
job: Job,
|
||||||
|
provenance: RuntimeModelProvenance,
|
||||||
|
) -> None:
|
||||||
|
"""Record immutable model evidence with a configured segmentation run."""
|
||||||
|
|
||||||
|
evidence = provenance.as_dict()
|
||||||
|
analysis_parameters = dict(analysis_run.parameters_json or {})
|
||||||
|
analysis_parameters["runtime_model_provenance"] = evidence
|
||||||
|
analysis_run.parameters_json = analysis_parameters
|
||||||
|
job_parameters = dict(job.parameters_json or {})
|
||||||
|
job_parameters["runtime_model_provenance"] = evidence
|
||||||
|
job.parameters_json = job_parameters
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _geodesic_area_m2(geometry: MultiPolygon | Polygon) -> float | None:
|
def _geodesic_area_m2(geometry: MultiPolygon | Polygon) -> float | None:
|
||||||
try:
|
try:
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -1,6 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
import math
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Iterable
|
from typing import Any, Iterable
|
||||||
from uuid import UUID
|
from uuid import UUID
|
||||||
@@ -8,6 +9,7 @@ from uuid import UUID
|
|||||||
from geoalchemy2.functions import ST_Intersects, ST_MakeEnvelope
|
from geoalchemy2.functions import ST_Intersects, ST_MakeEnvelope
|
||||||
from geoalchemy2.shape import from_shape
|
from geoalchemy2.shape import from_shape
|
||||||
from geoalchemy2.shape import to_shape
|
from geoalchemy2.shape import to_shape
|
||||||
|
from pyproj import CRS, Transformer
|
||||||
from shapely.geometry import box, mapping, shape
|
from shapely.geometry import box, mapping, shape
|
||||||
from shapely.ops import transform as transform_geometry
|
from shapely.ops import transform as transform_geometry
|
||||||
from shapely.validation import make_valid
|
from shapely.validation import make_valid
|
||||||
@@ -268,7 +270,14 @@ class VectorFeatureService:
|
|||||||
return ("municipality", municipality) if municipality else None
|
return ("municipality", municipality) if municipality else None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _feature_row(dataset_id: UUID, feature: dict[str, Any], index: int, feature_class: str | None) -> VectorFeature | None:
|
def _feature_row(
|
||||||
|
dataset_id: UUID,
|
||||||
|
feature: dict[str, Any],
|
||||||
|
index: int,
|
||||||
|
feature_class: str | None,
|
||||||
|
*,
|
||||||
|
source_crs: str = "EPSG:4326",
|
||||||
|
) -> VectorFeature | None:
|
||||||
geometry_payload = feature.get("geometry")
|
geometry_payload = feature.get("geometry")
|
||||||
if geometry_payload is None:
|
if geometry_payload is None:
|
||||||
return None
|
return None
|
||||||
@@ -282,8 +291,7 @@ class VectorFeatureService:
|
|||||||
geometry = make_valid(geometry)
|
geometry = make_valid(geometry)
|
||||||
if geometry.is_empty or not geometry.is_valid:
|
if geometry.is_empty or not geometry.is_valid:
|
||||||
raise AppError(code="INVALID_GEOMETRY", message=f"Invalid feature geometry at index {index}", status_code=400)
|
raise AppError(code="INVALID_GEOMETRY", message=f"Invalid feature geometry at index {index}", status_code=400)
|
||||||
if geometry.has_z:
|
geometry = VectorFeatureService._canonical_geometry(geometry, source_crs=source_crs, index=index)
|
||||||
geometry = transform_geometry(lambda x, y, z=None: (x, y), geometry)
|
|
||||||
|
|
||||||
properties = feature.get("properties") if isinstance(feature.get("properties"), dict) else {}
|
properties = feature.get("properties") if isinstance(feature.get("properties"), dict) else {}
|
||||||
source_feature_id = feature.get("id")
|
source_feature_id = feature.get("id")
|
||||||
@@ -298,6 +306,109 @@ class VectorFeatureService:
|
|||||||
geometry=from_shape(geometry, srid=4326),
|
geometry=from_shape(geometry, srid=4326),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _canonical_geometry(geometry: Any, *, source_crs: str, index: int):
|
||||||
|
"""Transform one source geometry to canonical EPSG:4326 safely."""
|
||||||
|
if geometry.has_z:
|
||||||
|
geometry = transform_geometry(lambda x, y, z=None: (x, y), geometry)
|
||||||
|
|
||||||
|
# VectorFeature is deliberately canonical WGS84 storage. Treating
|
||||||
|
# Lambert or another source CRS as EPSG:4326 produces geometries that
|
||||||
|
# look syntactically valid but are spatially wrong. All governed
|
||||||
|
# import callers therefore pass the declared source CRS; the default
|
||||||
|
# only preserves compatibility for legacy, already-WGS84 call sites.
|
||||||
|
try:
|
||||||
|
parsed_source_crs = CRS.from_user_input(source_crs)
|
||||||
|
target_crs = CRS.from_epsg(4326)
|
||||||
|
except Exception as exc:
|
||||||
|
raise AppError(
|
||||||
|
code="INVALID_DATASET_CRS",
|
||||||
|
message=f"Invalid source CRS for vector feature at index {index}",
|
||||||
|
details={"source_crs": source_crs},
|
||||||
|
status_code=400,
|
||||||
|
) from exc
|
||||||
|
if not parsed_source_crs.equals(target_crs):
|
||||||
|
try:
|
||||||
|
transformer = Transformer.from_crs(parsed_source_crs, target_crs, always_xy=True)
|
||||||
|
geometry = transform_geometry(transformer.transform, geometry)
|
||||||
|
except Exception as exc:
|
||||||
|
raise AppError(
|
||||||
|
code="VECTOR_CRS_TRANSFORMATION_FAILED",
|
||||||
|
message=f"Could not transform vector feature at index {index} to EPSG:4326",
|
||||||
|
details={"source_crs": source_crs},
|
||||||
|
status_code=400,
|
||||||
|
) from exc
|
||||||
|
if geometry.is_empty or not geometry.is_valid:
|
||||||
|
geometry = make_valid(geometry)
|
||||||
|
if geometry.is_empty or not geometry.is_valid:
|
||||||
|
raise AppError(
|
||||||
|
code="INVALID_GEOMETRY",
|
||||||
|
message=f"Invalid transformed feature geometry at index {index}",
|
||||||
|
status_code=400,
|
||||||
|
)
|
||||||
|
min_x, min_y, max_x, max_y = geometry.bounds
|
||||||
|
if (
|
||||||
|
not all(math.isfinite(value) for value in (min_x, min_y, max_x, max_y))
|
||||||
|
or min_x < -180
|
||||||
|
or max_x > 180
|
||||||
|
or min_y < -90
|
||||||
|
or max_y > 90
|
||||||
|
):
|
||||||
|
raise AppError(
|
||||||
|
code="VECTOR_GEOMETRY_OUTSIDE_EPSG4326",
|
||||||
|
message=f"Transformed feature geometry at index {index} is outside EPSG:4326 bounds",
|
||||||
|
details={"source_crs": source_crs, "bounds": [min_x, min_y, max_x, max_y]},
|
||||||
|
status_code=400,
|
||||||
|
)
|
||||||
|
return geometry
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def canonicalize_geojson_payload(payload: dict[str, Any], *, source_crs: str) -> dict[str, Any]:
|
||||||
|
"""Return a canonical-WGS84 feature collection without losing source attributes.
|
||||||
|
|
||||||
|
Callers use this payload for validation, spatial indexing and
|
||||||
|
map-safe consumption storage. A non-canonical source file, when
|
||||||
|
retained, belongs to explicit provenance evidence rather than the
|
||||||
|
Dataset consumption path; no implicit CRS assumption is recorded.
|
||||||
|
"""
|
||||||
|
features = payload.get("features")
|
||||||
|
if payload.get("type") != "FeatureCollection" or not isinstance(features, list):
|
||||||
|
raise AppError(code="INVALID_GEOJSON", message="GeoJSON payload must be a FeatureCollection", status_code=400)
|
||||||
|
canonical_features: list[dict[str, Any]] = []
|
||||||
|
for index, feature in enumerate(features):
|
||||||
|
if not isinstance(feature, dict):
|
||||||
|
raise AppError(code="INVALID_GEOJSON", message=f"Feature {index} must be an object", status_code=400)
|
||||||
|
canonical_feature = dict(feature)
|
||||||
|
geometry_payload = feature.get("geometry")
|
||||||
|
if geometry_payload is not None:
|
||||||
|
try:
|
||||||
|
geometry = shape(geometry_payload)
|
||||||
|
except Exception as exc:
|
||||||
|
raise AppError(
|
||||||
|
code="INVALID_GEOJSON",
|
||||||
|
message=f"Invalid feature geometry at index {index}",
|
||||||
|
status_code=400,
|
||||||
|
) from exc
|
||||||
|
if not geometry.is_empty:
|
||||||
|
if not geometry.is_valid:
|
||||||
|
geometry = make_valid(geometry)
|
||||||
|
if geometry.is_empty or not geometry.is_valid:
|
||||||
|
raise AppError(
|
||||||
|
code="INVALID_GEOMETRY",
|
||||||
|
message=f"Invalid feature geometry at index {index}",
|
||||||
|
status_code=400,
|
||||||
|
)
|
||||||
|
canonical_feature["geometry"] = mapping(
|
||||||
|
VectorFeatureService._canonical_geometry(geometry, source_crs=source_crs, index=index)
|
||||||
|
)
|
||||||
|
canonical_features.append(canonical_feature)
|
||||||
|
return {
|
||||||
|
**{key: value for key, value in payload.items() if key not in {"crs", "features"}},
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
|
||||||
|
"features": canonical_features,
|
||||||
|
}
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _normalize_selection_bbox(bbox: dict[str, Any]) -> dict[str, float | str]:
|
def _normalize_selection_bbox(bbox: dict[str, Any]) -> dict[str, float | str]:
|
||||||
try:
|
try:
|
||||||
@@ -882,6 +993,7 @@ class VectorFeatureService:
|
|||||||
feature_class: str | None = None,
|
feature_class: str | None = None,
|
||||||
*,
|
*,
|
||||||
commit: bool = True,
|
commit: bool = True,
|
||||||
|
source_crs: str = "EPSG:4326",
|
||||||
) -> list[VectorFeature]:
|
) -> list[VectorFeature]:
|
||||||
features = payload.get("features")
|
features = payload.get("features")
|
||||||
if payload.get("type") != "FeatureCollection" or not isinstance(features, list):
|
if payload.get("type") != "FeatureCollection" or not isinstance(features, list):
|
||||||
@@ -891,7 +1003,13 @@ class VectorFeatureService:
|
|||||||
for index, feature in enumerate(features):
|
for index, feature in enumerate(features):
|
||||||
if not isinstance(feature, dict):
|
if not isinstance(feature, dict):
|
||||||
raise AppError(code="INVALID_GEOJSON", message=f"Feature {index} must be an object", status_code=400)
|
raise AppError(code="INVALID_GEOJSON", message=f"Feature {index} must be an object", status_code=400)
|
||||||
row = VectorFeatureService._feature_row(dataset_id, feature, index, feature_class)
|
row = VectorFeatureService._feature_row(
|
||||||
|
dataset_id,
|
||||||
|
feature,
|
||||||
|
index,
|
||||||
|
feature_class,
|
||||||
|
source_crs=source_crs,
|
||||||
|
)
|
||||||
if row is None:
|
if row is None:
|
||||||
continue
|
continue
|
||||||
db.add(row)
|
db.add(row)
|
||||||
@@ -910,6 +1028,7 @@ class VectorFeatureService:
|
|||||||
feature_class: str | None = None,
|
feature_class: str | None = None,
|
||||||
*,
|
*,
|
||||||
batch_size: int = 1000,
|
batch_size: int = 1000,
|
||||||
|
source_crs: str = "EPSG:4326",
|
||||||
) -> int:
|
) -> int:
|
||||||
if batch_size <= 0:
|
if batch_size <= 0:
|
||||||
raise ValueError("batch_size must be positive")
|
raise ValueError("batch_size must be positive")
|
||||||
@@ -942,7 +1061,13 @@ class VectorFeatureService:
|
|||||||
message=f"Feature {index} in {path.name} must be an object",
|
message=f"Feature {index} in {path.name} must be an object",
|
||||||
status_code=400,
|
status_code=400,
|
||||||
)
|
)
|
||||||
row = VectorFeatureService._feature_row(dataset_id, feature, index, feature_class)
|
row = VectorFeatureService._feature_row(
|
||||||
|
dataset_id,
|
||||||
|
feature,
|
||||||
|
index,
|
||||||
|
feature_class,
|
||||||
|
source_crs=source_crs,
|
||||||
|
)
|
||||||
if row is None:
|
if row is None:
|
||||||
continue
|
continue
|
||||||
if row.source_feature_id:
|
if row.source_feature_id:
|
||||||
|
|||||||
@@ -3,13 +3,17 @@ from __future__ import annotations
|
|||||||
import json
|
import json
|
||||||
import uuid
|
import uuid
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timezone
|
||||||
|
from hashlib import sha256
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
import re
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
from geoalchemy2.shape import to_shape
|
from geoalchemy2.shape import to_shape
|
||||||
|
from pyproj import CRS, Transformer
|
||||||
from shapely.geometry import GeometryCollection, MultiPolygon, shape
|
from shapely.geometry import GeometryCollection, MultiPolygon, shape
|
||||||
from shapely.geometry.base import BaseGeometry
|
from shapely.geometry.base import BaseGeometry
|
||||||
from shapely.geometry import mapping
|
from shapely.geometry import mapping
|
||||||
|
from shapely.ops import transform as shapely_transform
|
||||||
from shapely.ops import unary_union
|
from shapely.ops import unary_union
|
||||||
from shapely.validation import make_valid
|
from shapely.validation import make_valid
|
||||||
from sqlalchemy.orm import Session
|
from sqlalchemy.orm import Session
|
||||||
@@ -18,12 +22,16 @@ from app.core.errors import AppError
|
|||||||
from app.models import Area, Dataset, DatasetVersion
|
from app.models import Area, Dataset, DatasetVersion
|
||||||
from app.schemas.dataset import DatasetCreateResponse
|
from app.schemas.dataset import DatasetCreateResponse
|
||||||
from app.schemas.operations import VectorOperationResult
|
from app.schemas.operations import VectorOperationResult
|
||||||
|
from app.services.derived_dataset_governance_service import DerivedDatasetGovernanceService
|
||||||
from app.services.geojson_service import parse_geojson_payload
|
from app.services.geojson_service import parse_geojson_payload
|
||||||
from app.services.storage_service import StorageService
|
from app.services.storage_service import StorageService
|
||||||
from app.services.vector_feature_service import VectorFeatureService
|
from app.services.vector_feature_service import VectorFeatureService
|
||||||
|
|
||||||
|
|
||||||
class VectorOperationsService:
|
class VectorOperationsService:
|
||||||
|
CANONICAL_VECTOR_CRS = "EPSG:4326"
|
||||||
|
_CHECKSUM_SHA256 = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _require_vector_dataset(dataset: Dataset) -> None:
|
def _require_vector_dataset(dataset: Dataset) -> None:
|
||||||
if dataset.dataset_type not in {"vector", "geojson"}:
|
if dataset.dataset_type not in {"vector", "geojson"}:
|
||||||
@@ -37,7 +45,8 @@ class VectorOperationsService:
|
|||||||
if not path.exists():
|
if not path.exists():
|
||||||
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
|
raise AppError(code="DATASET_FILE_MISSING", message="Stored file missing", status_code=404)
|
||||||
try:
|
try:
|
||||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
stored_bytes = path.read_bytes()
|
||||||
|
payload = json.loads(stored_bytes.decode("utf-8"))
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
raise AppError(code="INVALID_GEOJSON", message="Stored dataset is not valid JSON", status_code=400) from exc
|
raise AppError(code="INVALID_GEOJSON", message="Stored dataset is not valid JSON", status_code=400) from exc
|
||||||
|
|
||||||
@@ -47,6 +56,55 @@ class VectorOperationsService:
|
|||||||
features = payload.get("features")
|
features = payload.get("features")
|
||||||
if not isinstance(features, list):
|
if not isinstance(features, list):
|
||||||
raise AppError(code="INVALID_GEOJSON", message="Dataset payload is missing features", status_code=400)
|
raise AppError(code="INVALID_GEOJSON", message="Dataset payload is missing features", status_code=400)
|
||||||
|
|
||||||
|
# The normal Dataset storage path is a consumption artifact, not a
|
||||||
|
# provenance source archive. Refuse projected/original source bytes
|
||||||
|
# here rather than letting a spatial operation interpret them as
|
||||||
|
# canonical map coordinates.
|
||||||
|
raw_crs = payload.get("crs")
|
||||||
|
if isinstance(raw_crs, dict):
|
||||||
|
crs_properties = raw_crs.get("properties")
|
||||||
|
raw_crs = crs_properties.get("name") if isinstance(crs_properties, dict) else None
|
||||||
|
stored_crs = str(raw_crs or VectorOperationsService.CANONICAL_VECTOR_CRS).strip().upper()
|
||||||
|
dataset_crs = str(dataset.crs or "").strip().upper()
|
||||||
|
if stored_crs != VectorOperationsService.CANONICAL_VECTOR_CRS or (
|
||||||
|
dataset_crs and dataset_crs != VectorOperationsService.CANONICAL_VECTOR_CRS
|
||||||
|
):
|
||||||
|
raise AppError(
|
||||||
|
code="DATASET_STORAGE_CRS_MISMATCH",
|
||||||
|
message="Vector operations require canonical EPSG:4326 dataset storage.",
|
||||||
|
details={
|
||||||
|
"stored_crs": raw_crs or VectorOperationsService.CANONICAL_VECTOR_CRS,
|
||||||
|
"dataset_crs": dataset.crs,
|
||||||
|
"expected_crs": VectorOperationsService.CANONICAL_VECTOR_CRS,
|
||||||
|
},
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
|
|
||||||
|
expected_checksum = str(dataset.checksum_sha256 or "").strip().lower()
|
||||||
|
actual_checksum = sha256(stored_bytes).hexdigest()
|
||||||
|
governed_artifact = bool(
|
||||||
|
getattr(dataset, "data_contract_key", None)
|
||||||
|
or (
|
||||||
|
isinstance(getattr(dataset, "metadata_json", None), dict)
|
||||||
|
and dataset.metadata_json.get("canonical_storage_crs")
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if expected_checksum and VectorOperationsService._CHECKSUM_SHA256.fullmatch(expected_checksum):
|
||||||
|
if expected_checksum != actual_checksum:
|
||||||
|
raise AppError(
|
||||||
|
code="DATASET_STORAGE_CHECKSUM_MISMATCH",
|
||||||
|
message="Vector dataset storage no longer matches its validated checksum.",
|
||||||
|
details={"expected_checksum_sha256": expected_checksum, "actual_checksum_sha256": actual_checksum},
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
|
elif governed_artifact:
|
||||||
|
raise AppError(
|
||||||
|
code="DATASET_STORAGE_CHECKSUM_UNVERIFIABLE",
|
||||||
|
message="Governed vector storage requires a valid SHA-256 checksum before use.",
|
||||||
|
details={"checksum_sha256": dataset.checksum_sha256},
|
||||||
|
status_code=409,
|
||||||
|
)
|
||||||
return payload, [feature for feature in features if isinstance(feature, dict)]
|
return payload, [feature for feature in features if isinstance(feature, dict)]
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -73,6 +131,25 @@ class VectorOperationsService:
|
|||||||
raise AppError(code="EMPTY_VECTOR_DATASET", message="Dataset has no parseable geometries", status_code=422)
|
raise AppError(code="EMPTY_VECTOR_DATASET", message="Dataset has no parseable geometries", status_code=422)
|
||||||
return geometries
|
return geometries
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _buffer_in_metres(geometry: BaseGeometry, distance_m: float, source_crs: str) -> BaseGeometry:
|
||||||
|
"""Buffer in a Belgian projected CRS, never in angular degrees."""
|
||||||
|
|
||||||
|
try:
|
||||||
|
input_crs = CRS.from_user_input(source_crs)
|
||||||
|
metric_crs = CRS.from_epsg(31370)
|
||||||
|
if input_crs == metric_crs:
|
||||||
|
return geometry.buffer(distance_m)
|
||||||
|
forward = Transformer.from_crs(input_crs, metric_crs, always_xy=True)
|
||||||
|
backward = Transformer.from_crs(metric_crs, input_crs, always_xy=True)
|
||||||
|
return shapely_transform(backward.transform, shapely_transform(forward.transform, geometry).buffer(distance_m))
|
||||||
|
except Exception as exc:
|
||||||
|
raise AppError(
|
||||||
|
code="INVALID_CRS",
|
||||||
|
message="A valid explicit CRS is required for metre-based vector buffering.",
|
||||||
|
status_code=400,
|
||||||
|
) from exc
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def inspect(db: Session, dataset_id: uuid.UUID) -> VectorOperationResult:
|
def inspect(db: Session, dataset_id: uuid.UUID) -> VectorOperationResult:
|
||||||
dataset = db.get(Dataset, dataset_id)
|
dataset = db.get(Dataset, dataset_id)
|
||||||
@@ -94,7 +171,7 @@ class VectorOperationsService:
|
|||||||
feature_count=len(geometries),
|
feature_count=len(geometries),
|
||||||
geometry_type_summary=geometry_type_summary,
|
geometry_type_summary=geometry_type_summary,
|
||||||
bounds_json={"min_x": float(bounds[0]), "min_y": float(bounds[1]), "max_x": float(bounds[2]), "max_y": float(bounds[3])},
|
bounds_json={"min_x": float(bounds[0]), "min_y": float(bounds[1]), "max_x": float(bounds[2]), "max_y": float(bounds[3])},
|
||||||
crs=payload.get("crs") if isinstance(payload.get("crs"), str) else dataset.crs,
|
crs=VectorOperationsService.CANONICAL_VECTOR_CRS,
|
||||||
)
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -184,9 +261,13 @@ class VectorOperationsService:
|
|||||||
if distance_m <= 0:
|
if distance_m <= 0:
|
||||||
raise AppError(code="INVALID_PARAMETERS", message="distance_m must be greater than 0", status_code=400)
|
raise AppError(code="INVALID_PARAMETERS", message="distance_m must be greater than 0", status_code=400)
|
||||||
|
|
||||||
_, features = VectorOperationsService._load_dataset_payload(source_dataset)
|
payload, features = VectorOperationsService._load_dataset_payload(source_dataset)
|
||||||
|
source_crs = VectorOperationsService.CANONICAL_VECTOR_CRS
|
||||||
geometries = VectorOperationsService._extract_geometries(features)
|
geometries = VectorOperationsService._extract_geometries(features)
|
||||||
buffered_features = [(feature, geometry.buffer(distance_m)) for feature, geometry in geometries]
|
buffered_features = [
|
||||||
|
(feature, VectorOperationsService._buffer_in_metres(geometry, distance_m, source_crs))
|
||||||
|
for feature, geometry in geometries
|
||||||
|
]
|
||||||
|
|
||||||
output_features: list[dict[str, Any]] = []
|
output_features: list[dict[str, Any]] = []
|
||||||
for feature, geometry in buffered_features:
|
for feature, geometry in buffered_features:
|
||||||
@@ -431,7 +512,14 @@ class VectorOperationsService:
|
|||||||
if not output_name_value.strip():
|
if not output_name_value.strip():
|
||||||
output_name_value = f"{default_name}.geojson"
|
output_name_value = f"{default_name}.geojson"
|
||||||
|
|
||||||
stored = json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
# All current governed vector storage is EPSG:4326. A legacy source
|
||||||
|
# with another CRS is not relabelled here: the derived contract will
|
||||||
|
# quarantine the result instead of placing non-WGS84 coordinates on
|
||||||
|
# the map as if they were WGS84.
|
||||||
|
output_crs = VectorOperationsService.CANONICAL_VECTOR_CRS
|
||||||
|
output_feature_collection = dict(feature_collection)
|
||||||
|
output_feature_collection["crs"] = output_crs
|
||||||
|
stored = json.dumps(output_feature_collection, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||||
storage_info = StorageService.persist_dataset_file(
|
storage_info = StorageService.persist_dataset_file(
|
||||||
project_id=str(source_dataset.project_id),
|
project_id=str(source_dataset.project_id),
|
||||||
dataset_id=str(derived_id),
|
dataset_id=str(derived_id),
|
||||||
@@ -441,7 +529,7 @@ class VectorOperationsService:
|
|||||||
content_type="application/geo+json",
|
content_type="application/geo+json",
|
||||||
)
|
)
|
||||||
|
|
||||||
metadata = parse_geojson_payload(json.dumps(feature_collection, ensure_ascii=False, separators=(",", ":")))
|
metadata = parse_geojson_payload(json.dumps(output_feature_collection, ensure_ascii=False, separators=(",", ":")))
|
||||||
if metadata_extra:
|
if metadata_extra:
|
||||||
metadata.update(metadata_extra)
|
metadata.update(metadata_extra)
|
||||||
derived_dataset = Dataset(
|
derived_dataset = Dataset(
|
||||||
@@ -452,7 +540,7 @@ class VectorOperationsService:
|
|||||||
dataset_type="vector",
|
dataset_type="vector",
|
||||||
source=f"operation:{operation}",
|
source=f"operation:{operation}",
|
||||||
dataset_role=dataset_role,
|
dataset_role=dataset_role,
|
||||||
source_name=source_name,
|
source_name=source_name or "derived",
|
||||||
source_metadata=source_metadata,
|
source_metadata=source_metadata,
|
||||||
provenance_metadata=provenance_metadata,
|
provenance_metadata=provenance_metadata,
|
||||||
imported_at=datetime.now(timezone.utc),
|
imported_at=datetime.now(timezone.utc),
|
||||||
@@ -478,29 +566,44 @@ class VectorOperationsService:
|
|||||||
resolution_json=metadata.get("resolution_json"),
|
resolution_json=metadata.get("resolution_json"),
|
||||||
bands_json=metadata.get("bands_json"),
|
bands_json=metadata.get("bands_json"),
|
||||||
metadata_json=metadata,
|
metadata_json=metadata,
|
||||||
status="ready",
|
status="validating",
|
||||||
)
|
)
|
||||||
db.add(derived_dataset)
|
db.add(derived_dataset)
|
||||||
db.add(
|
dataset_version = DatasetVersion(
|
||||||
DatasetVersion(
|
dataset_id=derived_dataset.id,
|
||||||
dataset_id=derived_dataset.id,
|
version=1,
|
||||||
version=1,
|
storage_path=derived_dataset.storage_path,
|
||||||
storage_path=derived_dataset.storage_path,
|
source_version=derived_dataset.source_version,
|
||||||
source_version=derived_dataset.source_version,
|
observed_at=derived_dataset.observed_at,
|
||||||
observed_at=derived_dataset.observed_at,
|
valid_from=derived_dataset.valid_from,
|
||||||
valid_from=derived_dataset.valid_from,
|
valid_to=derived_dataset.valid_to,
|
||||||
valid_to=derived_dataset.valid_to,
|
checksum_sha256=derived_dataset.checksum_sha256,
|
||||||
checksum_sha256=derived_dataset.checksum_sha256,
|
source_metadata=derived_dataset.source_metadata,
|
||||||
source_metadata=derived_dataset.source_metadata,
|
provenance_metadata=derived_dataset.provenance_metadata,
|
||||||
provenance_metadata=derived_dataset.provenance_metadata,
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
db.commit()
|
db.add(dataset_version)
|
||||||
db.refresh(derived_dataset)
|
derived_source_key = "map_selection" if source_name == "map_selection" else "derived"
|
||||||
if persist_vector_features:
|
is_ready = DerivedDatasetGovernanceService.govern_vector(
|
||||||
|
db,
|
||||||
|
dataset=derived_dataset,
|
||||||
|
dataset_version=dataset_version,
|
||||||
|
feature_collection=output_feature_collection,
|
||||||
|
source_key=derived_source_key,
|
||||||
|
operation=f"vector.{operation}",
|
||||||
|
parent_dataset=source_dataset,
|
||||||
|
operation_parameters={
|
||||||
|
"operation": operation,
|
||||||
|
"output_name": output_name_value,
|
||||||
|
**(metadata_extra or {}),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if persist_vector_features and is_ready:
|
||||||
VectorFeatureService.persist_geojson_features(
|
VectorFeatureService.persist_geojson_features(
|
||||||
db=db,
|
db=db,
|
||||||
dataset_id=derived_dataset.id,
|
dataset_id=derived_dataset.id,
|
||||||
payload=feature_collection,
|
payload=output_feature_collection,
|
||||||
|
commit=False,
|
||||||
)
|
)
|
||||||
|
db.commit()
|
||||||
|
db.refresh(derived_dataset)
|
||||||
return derived_id
|
return derived_id
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ from app.core.config import Settings, get_settings
|
|||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.services.detection_service import DetectionService
|
from app.services.detection_service import DetectionService
|
||||||
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
||||||
|
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||||
from app.services.yolo_adapter import YoloDetectionAdapter
|
from app.services.yolo_adapter import YoloDetectionAdapter
|
||||||
|
|
||||||
|
|
||||||
@@ -41,6 +42,8 @@ class YoloPreflightService:
|
|||||||
"accelerator_ready": None,
|
"accelerator_ready": None,
|
||||||
"model_path_set": None,
|
"model_path_set": None,
|
||||||
"model_file_exists": None,
|
"model_file_exists": None,
|
||||||
|
"model_provenance_manifest_path": None,
|
||||||
|
"model_provenance_valid": None,
|
||||||
"model_load_requested": check_model_load,
|
"model_load_requested": check_model_load,
|
||||||
"model_load_ok": None,
|
"model_load_ok": None,
|
||||||
"manifest_path_set": None,
|
"manifest_path_set": None,
|
||||||
@@ -98,6 +101,29 @@ class YoloPreflightService:
|
|||||||
result["message"] = "YOLO_MODEL_PATH does not point to an existing local model file."
|
result["message"] = "YOLO_MODEL_PATH does not point to an existing local model file."
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
result["checks"]["model_provenance_manifest_path"] = str(
|
||||||
|
RuntimeModelProvenanceService.manifest_path_for_model(model_path)
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
RuntimeModelProvenanceService.validate_for_runtime(
|
||||||
|
model_path=model_path,
|
||||||
|
model_id=resolved_settings.yolo_model_id,
|
||||||
|
task_type="object_detection",
|
||||||
|
expected_model_version=resolved_settings.yolo_model_version,
|
||||||
|
allowed_frameworks=("ultralytics/pytorch", "ultralytics", "pytorch"),
|
||||||
|
)
|
||||||
|
except AppError as exc:
|
||||||
|
result["checks"]["model_provenance_valid"] = False
|
||||||
|
result["status"] = "contract_incomplete"
|
||||||
|
result["message"] = (
|
||||||
|
"Configured YOLO weights are not runnable until their immutable runtime provenance sidecar validates: "
|
||||||
|
f"{exc.message}"
|
||||||
|
)
|
||||||
|
result["error_code"] = exc.code
|
||||||
|
result["details"] = exc.details
|
||||||
|
return result
|
||||||
|
result["checks"]["model_provenance_valid"] = True
|
||||||
|
|
||||||
if check_model_load:
|
if check_model_load:
|
||||||
try:
|
try:
|
||||||
yolo_adapter_class(resolved_settings).load_model(model_path)
|
yolo_adapter_class(resolved_settings).load_model(model_path)
|
||||||
|
|||||||
@@ -0,0 +1,19 @@
|
|||||||
|
"""Canonical backend-test import boundary.
|
||||||
|
|
||||||
|
Pytest is intentionally runnable from ``backend/`` because that is the CI
|
||||||
|
entrypoint. Some contract tests exercise repository-level deterministic
|
||||||
|
scripts; put the canonical repository root ahead of the legacy
|
||||||
|
``backend/scripts`` helper directory so those imports resolve to the code that
|
||||||
|
is actually shipped by the root Docker build.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
REPOSITORY_ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
repository_root_text = str(REPOSITORY_ROOT)
|
||||||
|
if repository_root_text not in sys.path:
|
||||||
|
sys.path.insert(0, repository_root_text)
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
SCRIPT = ROOT / "scripts" / "run_accuracy_phase2_foundation_audit.py"
|
||||||
|
SPEC = importlib.util.spec_from_file_location("accuracy_phase2_foundation_audit", SCRIPT)
|
||||||
|
assert SPEC is not None and SPEC.loader is not None
|
||||||
|
MODULE = importlib.util.module_from_spec(SPEC)
|
||||||
|
SPEC.loader.exec_module(MODULE)
|
||||||
|
|
||||||
|
|
||||||
|
def test_phase2_foundation_audit_enumerates_exact_source_and_contract_policies() -> None:
|
||||||
|
payload = MODULE.collect()
|
||||||
|
|
||||||
|
assert payload["phase"] == "P2"
|
||||||
|
assert payload["migration_revision"] == "202608010001"
|
||||||
|
assert payload["source_registry"]["definition_count"] >= 40
|
||||||
|
assert payload["source_registry"]["required_building_policy"] == {
|
||||||
|
"grb_primary_building_validation": "primary",
|
||||||
|
"buildings_register_classification": "authoritative",
|
||||||
|
"sentinel_2_classification": "contextual",
|
||||||
|
"dhmv_classification": "authoritative",
|
||||||
|
"osm_ground_truth_allowed": False,
|
||||||
|
}
|
||||||
|
assert {(item["key"], item["version"]) for item in payload["data_contracts"]} == {
|
||||||
|
("geointel.vector.geojson", "1.0.0"),
|
||||||
|
("geointel.raster.geotiff", "1.0.0"),
|
||||||
|
("geointel.label.yolo", "1.0.0"),
|
||||||
|
("geointel.label.yolo", "1.1.0"),
|
||||||
|
("geointel.model.pytorch", "1.0.0"),
|
||||||
|
}
|
||||||
@@ -0,0 +1,772 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import timedelta
|
||||||
|
from importlib.util import module_from_spec, spec_from_file_location
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from sqlalchemy import CheckConstraint
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.models import (
|
||||||
|
Dataset,
|
||||||
|
DatasetLineageEdge,
|
||||||
|
DatasetQuarantine,
|
||||||
|
DatasetVersion,
|
||||||
|
SourceRegistry,
|
||||||
|
SourceSnapshot,
|
||||||
|
)
|
||||||
|
from app.services.coverage_registry_service import SOURCE_DEFINITIONS
|
||||||
|
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||||
|
from app.services.dataset_service import DatasetService
|
||||||
|
from app.services.source_registry_service import (
|
||||||
|
SERVER_OWNED_SOURCE_DEFINITIONS,
|
||||||
|
SourceRegistryService,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _Query:
|
||||||
|
def __init__(self, session: "InMemorySession", model: type) -> None:
|
||||||
|
self.session = session
|
||||||
|
self.model = model
|
||||||
|
self.predicates = []
|
||||||
|
|
||||||
|
def filter(self, *predicates):
|
||||||
|
self.predicates.extend(predicates)
|
||||||
|
return self
|
||||||
|
|
||||||
|
def one_or_none(self):
|
||||||
|
matches = self._matches()
|
||||||
|
if len(matches) > 1:
|
||||||
|
raise AssertionError(
|
||||||
|
f"Expected one {self.model.__name__}, found {len(matches)}"
|
||||||
|
)
|
||||||
|
return matches[0] if matches else None
|
||||||
|
|
||||||
|
def all(self):
|
||||||
|
return self._matches()
|
||||||
|
|
||||||
|
def _matches(self):
|
||||||
|
matches = list(self.session.objects.get(self.model, []))
|
||||||
|
for predicate in self.predicates:
|
||||||
|
field_name = predicate.left.key
|
||||||
|
expected = predicate.right.value
|
||||||
|
operator_name = getattr(predicate.operator, "__name__", "")
|
||||||
|
if operator_name == "in_op":
|
||||||
|
matches = [
|
||||||
|
item for item in matches if getattr(item, field_name) in expected
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
matches = [
|
||||||
|
item for item in matches if getattr(item, field_name) == expected
|
||||||
|
]
|
||||||
|
return matches
|
||||||
|
|
||||||
|
|
||||||
|
class InMemorySession:
|
||||||
|
def __init__(self, *objects: object) -> None:
|
||||||
|
self.objects: dict[type, list[object]] = {}
|
||||||
|
self.added: list[object] = []
|
||||||
|
self.flushes = 0
|
||||||
|
for item in objects:
|
||||||
|
self._store(item)
|
||||||
|
|
||||||
|
def query(self, model: type) -> _Query:
|
||||||
|
return _Query(self, model)
|
||||||
|
|
||||||
|
def add(self, item: object) -> None:
|
||||||
|
if getattr(item, "id", None) is None:
|
||||||
|
setattr(item, "id", uuid4())
|
||||||
|
self._store(item)
|
||||||
|
self.added.append(item)
|
||||||
|
|
||||||
|
def flush(self) -> None:
|
||||||
|
self.flushes += 1
|
||||||
|
|
||||||
|
def _store(self, item: object) -> None:
|
||||||
|
self.objects.setdefault(type(item), []).append(item)
|
||||||
|
|
||||||
|
|
||||||
|
def _registry(source_key: str) -> SourceRegistry:
|
||||||
|
definition = SERVER_OWNED_SOURCE_DEFINITIONS[source_key]
|
||||||
|
return SourceRegistry(id=uuid4(), **definition.as_model_values())
|
||||||
|
|
||||||
|
|
||||||
|
def _dataset() -> Dataset:
|
||||||
|
return Dataset(
|
||||||
|
id=uuid4(),
|
||||||
|
project_id=uuid4(),
|
||||||
|
name="candidate.tif",
|
||||||
|
dataset_type="raster",
|
||||||
|
source="governed",
|
||||||
|
status="ready",
|
||||||
|
validation_status="not_validated",
|
||||||
|
provenance_status="incomplete",
|
||||||
|
lineage_status="incomplete",
|
||||||
|
quarantine_status="not_quarantined",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_server_owned_definitions_encode_building_authority_and_non_ground_truth_sources() -> (
|
||||||
|
None
|
||||||
|
):
|
||||||
|
grb = SERVER_OWNED_SOURCE_DEFINITIONS["grb"]
|
||||||
|
buildings_register = SERVER_OWNED_SOURCE_DEFINITIONS[
|
||||||
|
"digitaal_vlaanderen_buildings_addresses_register"
|
||||||
|
]
|
||||||
|
sentinel = SERVER_OWNED_SOURCE_DEFINITIONS["sentinel_2"]
|
||||||
|
dhmv = SERVER_OWNED_SOURCE_DEFINITIONS["digitaal_vlaanderen_dhmv"]
|
||||||
|
osm = SERVER_OWNED_SOURCE_DEFINITIONS["osm"]
|
||||||
|
|
||||||
|
assert grb.classification == "authoritative"
|
||||||
|
assert grb.usage_policy["ground_truth_allowed"] is True
|
||||||
|
assert grb.usage_policy["validation_authority"]["building_validation"] == "primary"
|
||||||
|
assert buildings_register.classification == "authoritative"
|
||||||
|
assert buildings_register.usage_policy["ground_truth_allowed"] is False
|
||||||
|
assert (
|
||||||
|
buildings_register.usage_policy["validation_authority"]["building_validation"]
|
||||||
|
== "corroborative"
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
buildings_register.usage_policy["validation_authority"][
|
||||||
|
"building_register_validation"
|
||||||
|
]
|
||||||
|
== "primary"
|
||||||
|
)
|
||||||
|
assert sentinel.classification == "contextual"
|
||||||
|
assert dhmv.classification == "authoritative"
|
||||||
|
assert dhmv.usage_policy["ground_truth_allowed"] is False
|
||||||
|
assert (
|
||||||
|
dhmv.usage_policy["validation_authority"]["building_validation"]
|
||||||
|
== "corroborative"
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
dhmv.usage_policy["validation_authority"]["elevation_validation"] == "primary"
|
||||||
|
)
|
||||||
|
assert osm.classification == "contextual"
|
||||||
|
assert osm.usage_policy["ground_truth_allowed"] is False
|
||||||
|
assert osm.usage_policy["automatic_ground_truth"] is False
|
||||||
|
assert osm.usage_policy["training_allowed"] is False
|
||||||
|
assert {
|
||||||
|
"ngi_adminvector",
|
||||||
|
"rbins_marine_reporting_units",
|
||||||
|
"rbins_msp_2026",
|
||||||
|
"grb",
|
||||||
|
"digitaal_vlaanderen",
|
||||||
|
"vrbg",
|
||||||
|
"digitaal_vlaanderen_buildings_addresses_register",
|
||||||
|
"digitaal_vlaanderen_orthophoto",
|
||||||
|
"spw_orthophoto",
|
||||||
|
"urbis_orthophoto",
|
||||||
|
"digitaal_vlaanderen_dhmv",
|
||||||
|
"spw_terrain",
|
||||||
|
"spw_walous_land_cover",
|
||||||
|
"spw_geoportail",
|
||||||
|
"spw_picc",
|
||||||
|
"urbis",
|
||||||
|
"vmm_flood_hazard",
|
||||||
|
"vmm_vha_bathymetry_profiles",
|
||||||
|
"dov_soil_map",
|
||||||
|
"statbel",
|
||||||
|
"waterinfo",
|
||||||
|
"agentschap_landbouw_zeevisserij_agricultural_parcels",
|
||||||
|
"sentinel_2",
|
||||||
|
"osm",
|
||||||
|
"manual",
|
||||||
|
"fixture",
|
||||||
|
"map_selection",
|
||||||
|
"derived",
|
||||||
|
"training_label",
|
||||||
|
"model",
|
||||||
|
"experimental",
|
||||||
|
"mdk_bcp_bathymetry",
|
||||||
|
}.issubset(SERVER_OWNED_SOURCE_DEFINITIONS)
|
||||||
|
|
||||||
|
for umbrella_key in ("digitaal_vlaanderen", "spw_geoportail"):
|
||||||
|
definition = SERVER_OWNED_SOURCE_DEFINITIONS[umbrella_key]
|
||||||
|
assert definition.classification == "authoritative"
|
||||||
|
assert definition.usage_policy["ground_truth_allowed"] is False
|
||||||
|
assert definition.usage_policy["automatic_ground_truth"] is False
|
||||||
|
|
||||||
|
assert (
|
||||||
|
SERVER_OWNED_SOURCE_DEFINITIONS["mdk_bcp_bathymetry"].ingest_status
|
||||||
|
== "not_configured"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_coverage_and_direct_adapter_source_keys_are_registry_backed() -> None:
|
||||||
|
coverage_source_keys = {
|
||||||
|
definition.contract.source_name for definition in SOURCE_DEFINITIONS
|
||||||
|
}
|
||||||
|
coverage_materialization_keys = {
|
||||||
|
source_key
|
||||||
|
for definition in SOURCE_DEFINITIONS
|
||||||
|
for source_key in definition.materialized_source_names
|
||||||
|
}
|
||||||
|
direct_adapter_source_keys = {
|
||||||
|
"digitaal_vlaanderen",
|
||||||
|
"spw_geoportail",
|
||||||
|
"mdk_bcp_bathymetry",
|
||||||
|
}
|
||||||
|
|
||||||
|
assert (
|
||||||
|
coverage_source_keys
|
||||||
|
| coverage_materialization_keys
|
||||||
|
| direct_adapter_source_keys
|
||||||
|
<= set(SERVER_OWNED_SOURCE_DEFINITIONS)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_new_adapter_source_seed_rows_match_server_owned_registry_semantics() -> None:
|
||||||
|
migration_path = (
|
||||||
|
Path(__file__).resolve().parents[1]
|
||||||
|
/ "alembic"
|
||||||
|
/ "versions"
|
||||||
|
/ "202608010001_source_registry_provenance.py"
|
||||||
|
)
|
||||||
|
spec = spec_from_file_location("phase2_source_registry_migration", migration_path)
|
||||||
|
assert spec is not None and spec.loader is not None
|
||||||
|
migration = module_from_spec(spec)
|
||||||
|
spec.loader.exec_module(migration)
|
||||||
|
seed_rows = {row["source_key"]: row for row in migration._seed_rows()}
|
||||||
|
|
||||||
|
for source_key in ("digitaal_vlaanderen", "spw_geoportail", "mdk_bcp_bathymetry"):
|
||||||
|
expected = SERVER_OWNED_SOURCE_DEFINITIONS[source_key].as_model_values()
|
||||||
|
observed = seed_rows[source_key]
|
||||||
|
for field in (
|
||||||
|
"source_key",
|
||||||
|
"display_name",
|
||||||
|
"classification",
|
||||||
|
"authority_name",
|
||||||
|
"authority_scope_json",
|
||||||
|
"provider_adapter_key",
|
||||||
|
"source_url",
|
||||||
|
"default_crs",
|
||||||
|
"default_units",
|
||||||
|
"geographic_coverage_json",
|
||||||
|
"usage_policy_json",
|
||||||
|
"freshness_status",
|
||||||
|
"ingest_status",
|
||||||
|
"known_limitations_json",
|
||||||
|
):
|
||||||
|
assert observed[field] == expected[field]
|
||||||
|
|
||||||
|
|
||||||
|
def test_ensure_source_is_idempotent_and_rejects_caller_owned_unknown_sources() -> None:
|
||||||
|
grb = _registry("grb")
|
||||||
|
session = InMemorySession(grb)
|
||||||
|
|
||||||
|
assert SourceRegistryService.ensure_server_owned_source(session, "GRB") is grb
|
||||||
|
assert session.added == []
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
SourceRegistryService.ensure_server_owned_source(session, "caller_claimed_grb")
|
||||||
|
|
||||||
|
assert exc_info.value.code == "SOURCE_REGISTRY_ENTRY_NOT_FOUND"
|
||||||
|
|
||||||
|
|
||||||
|
def test_snapshot_is_checksum_bound_and_idempotent() -> None:
|
||||||
|
grb = _registry("grb")
|
||||||
|
session = InMemorySession(grb)
|
||||||
|
checksum = "a" * 64
|
||||||
|
|
||||||
|
snapshot = SourceRegistryService.record_snapshot(
|
||||||
|
session,
|
||||||
|
source_key="grb",
|
||||||
|
snapshot_key="2026-08-01-gbg",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
source_version="2026-08-01",
|
||||||
|
crs="EPSG:31370",
|
||||||
|
units="metres",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert snapshot.source_registry_id == grb.id
|
||||||
|
assert snapshot.checksum_sha256 == checksum
|
||||||
|
assert snapshot.ingest_status == "ingested"
|
||||||
|
assert (
|
||||||
|
SourceRegistryService.record_snapshot(
|
||||||
|
session,
|
||||||
|
source_key="grb",
|
||||||
|
snapshot_key="2026-08-01-gbg",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
)
|
||||||
|
is snapshot
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
SourceRegistryService.record_snapshot(
|
||||||
|
session,
|
||||||
|
source_key="grb",
|
||||||
|
snapshot_key="2026-08-01-gbg",
|
||||||
|
checksum_sha256="b" * 64,
|
||||||
|
)
|
||||||
|
assert exc_info.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT"
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as version_conflict:
|
||||||
|
SourceRegistryService.record_snapshot(
|
||||||
|
session,
|
||||||
|
source_key="grb",
|
||||||
|
snapshot_key="2026-08-01-gbg",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
source_version="2026-08-02",
|
||||||
|
)
|
||||||
|
assert version_conflict.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT"
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as invalid_checksum:
|
||||||
|
SourceRegistryService.record_snapshot(
|
||||||
|
session,
|
||||||
|
source_key="grb",
|
||||||
|
snapshot_key="bad-checksum",
|
||||||
|
checksum_sha256="not-a-checksum",
|
||||||
|
)
|
||||||
|
assert invalid_checksum.value.code == "SOURCE_SNAPSHOT_CHECKSUM_INVALID"
|
||||||
|
|
||||||
|
|
||||||
|
def test_governed_import_reuses_an_identical_snapshot_without_rewriting_fetched_at() -> None:
|
||||||
|
"""A second project may bind the same immutable source snapshot safely."""
|
||||||
|
|
||||||
|
grb = _registry("grb")
|
||||||
|
session = InMemorySession(grb)
|
||||||
|
checksum = "a" * 64
|
||||||
|
observed_at = None
|
||||||
|
metadata = {
|
||||||
|
"dataset_type": "vector",
|
||||||
|
"bounds_json": {"min_x": 4.0, "min_y": 50.0, "max_x": 4.1, "max_y": 50.1},
|
||||||
|
}
|
||||||
|
source_metadata = {"source_url": "https://example.test/grb", "units": "metres"}
|
||||||
|
|
||||||
|
# Dataset ingest keys are project-scoped, while a source snapshot is
|
||||||
|
# globally keyed by immutable source evidence. This represents the same
|
||||||
|
# source file arriving through two independently resumable imports.
|
||||||
|
project_one, project_two = uuid4(), uuid4()
|
||||||
|
assert (
|
||||||
|
DatasetService._ingest_key(
|
||||||
|
project_id=project_one,
|
||||||
|
source_key="grb",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
dataset_type="vector",
|
||||||
|
dataset_role="source",
|
||||||
|
area_id=None,
|
||||||
|
reference_layer_name=None,
|
||||||
|
source_version="2026-08-01",
|
||||||
|
)
|
||||||
|
!= DatasetService._ingest_key(
|
||||||
|
project_id=project_two,
|
||||||
|
source_key="grb",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
dataset_type="vector",
|
||||||
|
dataset_role="source",
|
||||||
|
area_id=None,
|
||||||
|
reference_layer_name=None,
|
||||||
|
source_version="2026-08-01",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
first_source, first_snapshot = DatasetService._record_snapshot(
|
||||||
|
session,
|
||||||
|
source_key="grb",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
source_version="2026-08-01",
|
||||||
|
observed_at=observed_at,
|
||||||
|
valid_from=None,
|
||||||
|
valid_to=None,
|
||||||
|
source_crs="EPSG:31370",
|
||||||
|
source_metadata=source_metadata,
|
||||||
|
metadata=metadata,
|
||||||
|
)
|
||||||
|
original_fetched_at = first_snapshot.fetched_at
|
||||||
|
|
||||||
|
replay_source, replay_snapshot = DatasetService._record_snapshot(
|
||||||
|
session,
|
||||||
|
source_key="grb",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
source_version="2026-08-01",
|
||||||
|
observed_at=observed_at,
|
||||||
|
valid_from=None,
|
||||||
|
valid_to=None,
|
||||||
|
source_crs="EPSG:31370",
|
||||||
|
source_metadata=source_metadata,
|
||||||
|
metadata=metadata,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert replay_source is first_source
|
||||||
|
assert replay_snapshot is first_snapshot
|
||||||
|
assert replay_snapshot.fetched_at == original_fetched_at
|
||||||
|
assert session.objects[SourceSnapshot] == [first_snapshot]
|
||||||
|
|
||||||
|
# Outside the governed replay path, a contradictory acquisition timestamp
|
||||||
|
# remains immutable evidence and is still rejected.
|
||||||
|
with pytest.raises(AppError) as fetched_at_conflict:
|
||||||
|
SourceRegistryService.record_snapshot(
|
||||||
|
session,
|
||||||
|
source_key="grb",
|
||||||
|
snapshot_key=first_snapshot.snapshot_key,
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
fetched_at=original_fetched_at + timedelta(seconds=1),
|
||||||
|
)
|
||||||
|
assert fetched_at_conflict.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT"
|
||||||
|
|
||||||
|
# Replay mode is narrow: a changed immutable evidence field still fails.
|
||||||
|
with pytest.raises(AppError) as conflict:
|
||||||
|
SourceRegistryService.record_snapshot(
|
||||||
|
session,
|
||||||
|
source_key="grb",
|
||||||
|
snapshot_key=first_snapshot.snapshot_key,
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
crs="EPSG:4326",
|
||||||
|
reuse_existing_snapshot=True,
|
||||||
|
)
|
||||||
|
assert conflict.value.code == "SOURCE_SNAPSHOT_IMMUTABILITY_CONFLICT"
|
||||||
|
|
||||||
|
|
||||||
|
def test_snapshot_schema_requires_a_canonical_sha256() -> None:
|
||||||
|
constraints = {
|
||||||
|
constraint.name: str(constraint.sqltext)
|
||||||
|
for constraint in SourceSnapshot.__table__.constraints
|
||||||
|
if isinstance(constraint, CheckConstraint)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert SourceSnapshot.__table__.c.checksum_sha256.nullable is False
|
||||||
|
assert "ck_source_snapshots_checksum_sha256" in constraints
|
||||||
|
assert (
|
||||||
|
"lower(checksum_sha256)" in constraints["ck_source_snapshots_checksum_sha256"]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_complete_provenance_binding_is_required_before_authoritative_validation() -> (
|
||||||
|
None
|
||||||
|
):
|
||||||
|
grb = _registry("grb")
|
||||||
|
snapshot = SourceSnapshot(
|
||||||
|
id=uuid4(),
|
||||||
|
source_registry_id=grb.id,
|
||||||
|
snapshot_key="governed-grb",
|
||||||
|
checksum_sha256="c" * 64,
|
||||||
|
ingest_status="ingested",
|
||||||
|
)
|
||||||
|
dataset = _dataset()
|
||||||
|
|
||||||
|
SourceRegistryService.bind_dataset_provenance(
|
||||||
|
dataset,
|
||||||
|
source=grb,
|
||||||
|
snapshot=snapshot,
|
||||||
|
data_contract_key="vector.grb.buildings",
|
||||||
|
data_contract_version="1.0.0",
|
||||||
|
validation_status="passed",
|
||||||
|
provenance_status="complete",
|
||||||
|
lineage_status="complete",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert SourceRegistryService.is_dataset_eligible_for_authoritative_validation(
|
||||||
|
dataset,
|
||||||
|
source=grb,
|
||||||
|
snapshot=snapshot,
|
||||||
|
task="building_validation",
|
||||||
|
)
|
||||||
|
|
||||||
|
osm = _registry("osm")
|
||||||
|
dataset.source_registry_id = osm.id
|
||||||
|
snapshot.source_registry_id = osm.id
|
||||||
|
assert not SourceRegistryService.is_dataset_eligible_for_authoritative_validation(
|
||||||
|
dataset,
|
||||||
|
source=osm,
|
||||||
|
snapshot=snapshot,
|
||||||
|
task="building_validation",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_lineage_and_quarantine_are_fail_closed_and_observable() -> None:
|
||||||
|
session = InMemorySession()
|
||||||
|
parent_id = uuid4()
|
||||||
|
child_id = uuid4()
|
||||||
|
edge = SourceRegistryService.record_lineage_edge(
|
||||||
|
session,
|
||||||
|
parent_dataset_id=parent_id,
|
||||||
|
child_dataset_id=child_id,
|
||||||
|
relation_type="derived_from",
|
||||||
|
transformation_name="vector_clip",
|
||||||
|
input_checksum_sha256="d" * 64,
|
||||||
|
output_checksum_sha256="e" * 64,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert isinstance(edge, DatasetLineageEdge)
|
||||||
|
assert (
|
||||||
|
SourceRegistryService.record_lineage_edge(
|
||||||
|
session,
|
||||||
|
parent_dataset_id=parent_id,
|
||||||
|
child_dataset_id=child_id,
|
||||||
|
relation_type="derived_from",
|
||||||
|
transformation_name="vector_clip",
|
||||||
|
input_checksum_sha256="d" * 64,
|
||||||
|
output_checksum_sha256="e" * 64,
|
||||||
|
)
|
||||||
|
is edge
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as self_reference:
|
||||||
|
SourceRegistryService.record_lineage_edge(
|
||||||
|
session,
|
||||||
|
parent_dataset_id=parent_id,
|
||||||
|
child_dataset_id=parent_id,
|
||||||
|
relation_type="derived_from",
|
||||||
|
transformation_name="vector_clip",
|
||||||
|
)
|
||||||
|
assert self_reference.value.code == "DATASET_LINEAGE_SELF_REFERENCE"
|
||||||
|
|
||||||
|
grandchild_id = uuid4()
|
||||||
|
SourceRegistryService.record_lineage_edge(
|
||||||
|
session,
|
||||||
|
parent_dataset_id=child_id,
|
||||||
|
child_dataset_id=grandchild_id,
|
||||||
|
relation_type="derived_from",
|
||||||
|
transformation_name="vector_buffer",
|
||||||
|
)
|
||||||
|
with pytest.raises(AppError) as cycle:
|
||||||
|
SourceRegistryService.record_lineage_edge(
|
||||||
|
session,
|
||||||
|
parent_dataset_id=grandchild_id,
|
||||||
|
child_dataset_id=parent_id,
|
||||||
|
relation_type="derived_from",
|
||||||
|
transformation_name="vector_union",
|
||||||
|
)
|
||||||
|
assert cycle.value.code == "DATASET_LINEAGE_CYCLE_DETECTED"
|
||||||
|
|
||||||
|
dataset = _dataset()
|
||||||
|
record = SourceRegistryService.quarantine_dataset(
|
||||||
|
session,
|
||||||
|
dataset=dataset,
|
||||||
|
stage="vector_ingest",
|
||||||
|
reason_code="CRS_UNVERIFIED",
|
||||||
|
details={"observed_crs": None},
|
||||||
|
)
|
||||||
|
assert isinstance(record, DatasetQuarantine)
|
||||||
|
assert dataset.status == "quarantined"
|
||||||
|
assert dataset.quarantine_status == "quarantined"
|
||||||
|
assert dataset.validation_status == "failed"
|
||||||
|
|
||||||
|
version_parent = _dataset()
|
||||||
|
version = DatasetVersion(id=uuid4(), dataset_id=version_parent.id, version=1)
|
||||||
|
version_session = InMemorySession(version_parent, version)
|
||||||
|
version_record = SourceRegistryService.quarantine_dataset(
|
||||||
|
version_session,
|
||||||
|
dataset_version=version,
|
||||||
|
stage="dataset_version_validation",
|
||||||
|
reason_code="CHECKSUM_MISMATCH",
|
||||||
|
)
|
||||||
|
assert version_record.dataset_id == version_parent.id
|
||||||
|
assert version_record.dataset_version_id == version.id
|
||||||
|
assert version_parent.status == "quarantined"
|
||||||
|
assert version_parent.quarantine_status == "quarantined"
|
||||||
|
assert version_parent.validation_status == "failed"
|
||||||
|
assert version_parent.provenance_status == "incomplete"
|
||||||
|
assert version_parent.lineage_status == "incomplete"
|
||||||
|
assert version.validation_status == "failed"
|
||||||
|
assert version.provenance_status == "incomplete"
|
||||||
|
|
||||||
|
snapshot = SourceSnapshot(
|
||||||
|
id=uuid4(),
|
||||||
|
source_registry_id=uuid4(),
|
||||||
|
snapshot_key="quarantined-source",
|
||||||
|
checksum_sha256="f" * 64,
|
||||||
|
ingest_status="ingested",
|
||||||
|
)
|
||||||
|
SourceRegistryService.quarantine_dataset(
|
||||||
|
session,
|
||||||
|
source_snapshot=snapshot,
|
||||||
|
stage="source_snapshot_validation",
|
||||||
|
reason_code="CHECKSUM_MISMATCH",
|
||||||
|
)
|
||||||
|
assert snapshot.ingest_status == "quarantined"
|
||||||
|
|
||||||
|
|
||||||
|
def test_quarantine_propagates_transitively_to_descendant_dataset_and_version_consumption_gates() -> (
|
||||||
|
None
|
||||||
|
):
|
||||||
|
"""A -> B -> C must fail closed when the governing A artifact is rejected."""
|
||||||
|
|
||||||
|
source = _registry("grb")
|
||||||
|
snapshot = SourceSnapshot(
|
||||||
|
id=uuid4(),
|
||||||
|
source_registry_id=source.id,
|
||||||
|
snapshot_key="transitive-quarantine-source",
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
freshness_status="current",
|
||||||
|
ingest_status="ingested",
|
||||||
|
)
|
||||||
|
|
||||||
|
def governed_dataset(name: str) -> Dataset:
|
||||||
|
dataset = _dataset()
|
||||||
|
dataset.name = name
|
||||||
|
dataset.source = "grb"
|
||||||
|
dataset.source_name = "grb"
|
||||||
|
dataset.dataset_role = "source"
|
||||||
|
dataset.checksum_sha256 = snapshot.checksum_sha256
|
||||||
|
dataset.source_registry_id = source.id
|
||||||
|
dataset.source_snapshot_id = snapshot.id
|
||||||
|
dataset.data_contract_key = "geointel.raster.geotiff"
|
||||||
|
dataset.data_contract_version = "1.0.0"
|
||||||
|
dataset.validation_status = "passed"
|
||||||
|
dataset.provenance_status = "complete"
|
||||||
|
dataset.lineage_status = "complete"
|
||||||
|
dataset.quarantine_status = "not_quarantined"
|
||||||
|
dataset.status = "ready"
|
||||||
|
dataset.source_registry = source
|
||||||
|
dataset.source_snapshot = snapshot
|
||||||
|
return dataset
|
||||||
|
|
||||||
|
parent = governed_dataset("parent.tif")
|
||||||
|
child = governed_dataset("child.tif")
|
||||||
|
grandchild = governed_dataset("grandchild.tif")
|
||||||
|
parent_version = DatasetVersion(
|
||||||
|
id=uuid4(),
|
||||||
|
dataset_id=parent.id,
|
||||||
|
version=1,
|
||||||
|
validation_status="passed",
|
||||||
|
provenance_status="complete",
|
||||||
|
lineage_status="complete",
|
||||||
|
)
|
||||||
|
child_version = DatasetVersion(
|
||||||
|
id=uuid4(),
|
||||||
|
dataset_id=child.id,
|
||||||
|
version=1,
|
||||||
|
validation_status="passed",
|
||||||
|
provenance_status="complete",
|
||||||
|
lineage_status="complete",
|
||||||
|
)
|
||||||
|
grandchild_version = DatasetVersion(
|
||||||
|
id=uuid4(),
|
||||||
|
dataset_id=grandchild.id,
|
||||||
|
version=1,
|
||||||
|
validation_status="passed",
|
||||||
|
provenance_status="complete",
|
||||||
|
lineage_status="complete",
|
||||||
|
)
|
||||||
|
session = InMemorySession(
|
||||||
|
parent,
|
||||||
|
child,
|
||||||
|
grandchild,
|
||||||
|
parent_version,
|
||||||
|
child_version,
|
||||||
|
grandchild_version,
|
||||||
|
)
|
||||||
|
SourceRegistryService.record_lineage_edge(
|
||||||
|
session,
|
||||||
|
parent_dataset_id=parent.id,
|
||||||
|
child_dataset_id=child.id,
|
||||||
|
relation_type="derived_from",
|
||||||
|
transformation_name="clip",
|
||||||
|
)
|
||||||
|
SourceRegistryService.record_lineage_edge(
|
||||||
|
session,
|
||||||
|
parent_dataset_id=child.id,
|
||||||
|
child_dataset_id=grandchild.id,
|
||||||
|
relation_type="derived_from",
|
||||||
|
transformation_name="buffer",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert (
|
||||||
|
DatasetConsumptionGate.evaluate(child, purpose="production_inference").eligible
|
||||||
|
is True
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
DatasetConsumptionGate.evaluate(
|
||||||
|
grandchild, purpose="production_inference"
|
||||||
|
).eligible
|
||||||
|
is True
|
||||||
|
)
|
||||||
|
|
||||||
|
SourceRegistryService.quarantine_dataset(
|
||||||
|
session,
|
||||||
|
dataset=parent,
|
||||||
|
stage="contract_validation",
|
||||||
|
reason_code="CHECKSUM_MISMATCH",
|
||||||
|
)
|
||||||
|
|
||||||
|
for dataset in (parent, child, grandchild):
|
||||||
|
decision = DatasetConsumptionGate.evaluate(
|
||||||
|
dataset, purpose="production_inference"
|
||||||
|
)
|
||||||
|
assert dataset.status == "quarantined"
|
||||||
|
assert dataset.quarantine_status == "quarantined"
|
||||||
|
assert dataset.validation_status == "failed"
|
||||||
|
assert dataset.provenance_status == "incomplete"
|
||||||
|
assert dataset.lineage_status == "incomplete"
|
||||||
|
assert decision.eligible is False
|
||||||
|
assert "dataset_quarantined" in decision.reasons
|
||||||
|
for dataset_version in (parent_version, child_version, grandchild_version):
|
||||||
|
assert dataset_version.validation_status == "failed"
|
||||||
|
assert dataset_version.provenance_status == "incomplete"
|
||||||
|
assert dataset_version.lineage_status == "incomplete"
|
||||||
|
|
||||||
|
|
||||||
|
def test_ingest_keys_are_scoped_and_migration_keeps_unknown_legacy_unbound() -> None:
|
||||||
|
project_id = uuid4()
|
||||||
|
dataset = _dataset()
|
||||||
|
dataset.project_id = project_id
|
||||||
|
dataset.ingest_key = "grb:2026-08-01:gbg:area-sha"
|
||||||
|
version = DatasetVersion(
|
||||||
|
id=uuid4(),
|
||||||
|
dataset_id=dataset.id,
|
||||||
|
ingest_key=dataset.ingest_key,
|
||||||
|
validation_status="not_validated",
|
||||||
|
provenance_status="incomplete",
|
||||||
|
lineage_status="incomplete",
|
||||||
|
)
|
||||||
|
session = InMemorySession(dataset, version)
|
||||||
|
|
||||||
|
assert (
|
||||||
|
SourceRegistryService.find_dataset_by_ingest_key(
|
||||||
|
session, project_id, dataset.ingest_key
|
||||||
|
)
|
||||||
|
is dataset
|
||||||
|
)
|
||||||
|
assert (
|
||||||
|
SourceRegistryService.find_dataset_version_by_ingest_key(
|
||||||
|
session, dataset.id, dataset.ingest_key
|
||||||
|
)
|
||||||
|
is version
|
||||||
|
)
|
||||||
|
with pytest.raises(AppError) as invalid_key:
|
||||||
|
SourceRegistryService.find_dataset_by_ingest_key(session, project_id, " ")
|
||||||
|
assert invalid_key.value.code == "INGEST_KEY_INVALID"
|
||||||
|
|
||||||
|
migration = (
|
||||||
|
Path(__file__).resolve().parents[1]
|
||||||
|
/ "alembic"
|
||||||
|
/ "versions"
|
||||||
|
/ "202608010001_source_registry_provenance.py"
|
||||||
|
).read_text(encoding="utf-8")
|
||||||
|
assert "uuid_generate_v5" not in migration
|
||||||
|
assert "__unregistered_legacy_source__" in migration
|
||||||
|
assert "uq_datasets_project_ingest_key" in migration
|
||||||
|
assert "uq_dataset_versions_dataset_ingest_key" in migration
|
||||||
|
|
||||||
|
|
||||||
|
def test_migration_contains_database_guards_for_snapshot_pairing_contract_lineage_and_quarantine() -> (
|
||||||
|
None
|
||||||
|
):
|
||||||
|
migration = (
|
||||||
|
Path(__file__).resolve().parents[1]
|
||||||
|
/ "alembic"
|
||||||
|
/ "versions"
|
||||||
|
/ "202608010001_source_registry_provenance.py"
|
||||||
|
).read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert "trg_datasets_snapshot_registry_guard" in migration
|
||||||
|
assert "trg_dataset_versions_snapshot_registry_guard" in migration
|
||||||
|
assert "trg_source_registry_write_guard" in migration
|
||||||
|
assert "trg_source_snapshots_evidence_immutable" in migration
|
||||||
|
assert "geointel_phase2_contract_report_guard" in migration
|
||||||
|
assert "trg_datasets_contract_report_guard" in migration
|
||||||
|
assert "trg_dataset_versions_contract_report_guard" in migration
|
||||||
|
assert "matching complete validation report" in migration
|
||||||
|
assert "geointel_phase2_lineage_cycle_guard" in migration
|
||||||
|
assert "trg_dataset_lineage_edges_cycle_guard" in migration
|
||||||
|
assert "geointel_phase2_lineage_edge_immutable_guard" in migration
|
||||||
|
assert "trg_dataset_lineage_edges_immutable" in migration
|
||||||
|
assert "WITH RECURSIVE descendants" in migration
|
||||||
|
assert "geointel_phase2_quarantine_lineage_descendants" in migration
|
||||||
|
assert "geointel_phase2_quarantine_state_guard" in migration
|
||||||
|
assert "trg_dataset_quarantines_state_guard" in migration
|
||||||
|
assert "accepted dataset artifact and provenance evidence is immutable" in migration
|
||||||
@@ -154,6 +154,7 @@ def test_guest_login_exposes_models_but_rejects_management_and_cross_project_req
|
|||||||
other_project = client.get("/api/v1/projects/00000000-0000-0000-0000-000000000999")
|
other_project = client.get("/api/v1/projects/00000000-0000-0000-0000-000000000999")
|
||||||
detection_models = client.get("/api/v1/detection/models")
|
detection_models = client.get("/api/v1/detection/models")
|
||||||
segmentation_models = client.get("/api/v1/segmentation/models")
|
segmentation_models = client.get("/api/v1/segmentation/models")
|
||||||
|
global_source_registry = client.get("/api/v1/source-registry/grb")
|
||||||
cross_project_runs = client.get(
|
cross_project_runs = client.get(
|
||||||
"/api/v1/detection/runs?project_id=00000000-0000-0000-0000-000000000999"
|
"/api/v1/detection/runs?project_id=00000000-0000-0000-0000-000000000999"
|
||||||
)
|
)
|
||||||
@@ -179,6 +180,8 @@ def test_guest_login_exposes_models_but_rejects_management_and_cross_project_req
|
|||||||
assert detection_models.status_code == 200
|
assert detection_models.status_code == 200
|
||||||
assert detection_models.json()["data"]["models"]
|
assert detection_models.json()["data"]["models"]
|
||||||
assert segmentation_models.status_code == 200
|
assert segmentation_models.status_code == 200
|
||||||
|
assert global_source_registry.status_code == 403
|
||||||
|
assert global_source_registry.json()["error"] == "GUEST_ROUTE_NOT_AVAILABLE"
|
||||||
assert cross_project_runs.status_code == 403
|
assert cross_project_runs.status_code == 403
|
||||||
assert cross_project_runs.json()["error"] == "GUEST_PROJECT_SCOPE_REQUIRED"
|
assert cross_project_runs.json()["error"] == "GUEST_PROJECT_SCOPE_REQUIRED"
|
||||||
assert cross_project_coverage.status_code == 403
|
assert cross_project_coverage.status_code == 403
|
||||||
@@ -186,7 +189,7 @@ def test_guest_login_exposes_models_but_rejects_management_and_cross_project_req
|
|||||||
|
|
||||||
|
|
||||||
def test_guest_session_tokens_fail_closed_without_a_project_scope(monkeypatch) -> None:
|
def test_guest_session_tokens_fail_closed_without_a_project_scope(monkeypatch) -> None:
|
||||||
client = auth_client(monkeypatch, guest_access=True)
|
auth_client(monkeypatch, guest_access=True)
|
||||||
settings = get_settings()
|
settings = get_settings()
|
||||||
|
|
||||||
try:
|
try:
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import importlib.util
|
import importlib.util
|
||||||
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
@@ -13,6 +14,94 @@ assert SPEC and SPEC.loader
|
|||||||
MODULE = importlib.util.module_from_spec(SPEC)
|
MODULE = importlib.util.module_from_spec(SPEC)
|
||||||
SPEC.loader.exec_module(MODULE)
|
SPEC.loader.exec_module(MODULE)
|
||||||
|
|
||||||
|
from training_release_manifest import create_training_release_manifest # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def write_fixture_manifest(path: Path) -> None:
|
||||||
|
policy = "geointel-training-source-eligibility/v1"
|
||||||
|
def eligible(sample_slug: str) -> dict[str, object]:
|
||||||
|
return {
|
||||||
|
"policy_version": policy,
|
||||||
|
"eligible": True,
|
||||||
|
"fixture_mode": True,
|
||||||
|
"raster": {
|
||||||
|
"eligible": True,
|
||||||
|
"reasons": [],
|
||||||
|
"evidence": {
|
||||||
|
"dataset_id": f"raster:{sample_slug}",
|
||||||
|
"checksum_sha256": "a" * 64,
|
||||||
|
"source_registry_id": "fixture-raster",
|
||||||
|
"source_snapshot_id": "fixture-raster-snapshot",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"reference": {
|
||||||
|
"eligible": True,
|
||||||
|
"reasons": [],
|
||||||
|
"evidence": {
|
||||||
|
"dataset_id": f"reference:{sample_slug}",
|
||||||
|
"checksum_sha256": "b" * 64,
|
||||||
|
"source_registry_id": "fixture-reference",
|
||||||
|
"source_snapshot_id": "fixture-reference-snapshot",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
path.write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"training_eligibility": {
|
||||||
|
"policy_version": policy,
|
||||||
|
"status": "eligible",
|
||||||
|
"fixture_mode": True,
|
||||||
|
},
|
||||||
|
"samples": [
|
||||||
|
{
|
||||||
|
"sample_slug": sample_slug,
|
||||||
|
"split": split,
|
||||||
|
"raster_dataset_id": f"raster:{sample_slug}",
|
||||||
|
"reference_dataset_id": f"reference:{sample_slug}",
|
||||||
|
"training_eligibility": eligible(sample_slug),
|
||||||
|
}
|
||||||
|
for sample_slug, split in (("fixture-train", "train"), ("fixture-val", "val"))
|
||||||
|
],
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
(path.parent / "corpus-freeze.json").write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"schema_version": 2,
|
||||||
|
"manifest_sha256": hashlib.sha256(path.read_bytes()).hexdigest(),
|
||||||
|
"immutable": True,
|
||||||
|
"training_eligibility_policy": policy,
|
||||||
|
"fixture_mode": True,
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def write_fixture_training_release(tmp_path: Path, manifest: Path) -> Path:
|
||||||
|
dataset_dir = tmp_path / "fixture-dataset"
|
||||||
|
for split, sample_slug in (("train", "fixture-train"), ("val", "fixture-val")):
|
||||||
|
image = dataset_dir / "images" / split / f"{sample_slug}.png"
|
||||||
|
label = dataset_dir / "labels" / split / f"{sample_slug}.txt"
|
||||||
|
image.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
label.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
image.write_bytes(split.encode("utf-8"))
|
||||||
|
label.write_text("0 0.5 0.5 0.2 0.2\n", encoding="utf-8")
|
||||||
|
yaml_path = dataset_dir / "dataset.yaml"
|
||||||
|
yaml_path.write_text(
|
||||||
|
f"path: {dataset_dir}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
create_training_release_manifest(
|
||||||
|
train_yaml=yaml_path,
|
||||||
|
corpus_manifest=manifest,
|
||||||
|
fixture_mode=True,
|
||||||
|
)
|
||||||
|
return yaml_path
|
||||||
|
|
||||||
|
|
||||||
def test_training_command_is_cuda_deterministic_and_bound_to_frozen_inputs(tmp_path: Path) -> None:
|
def test_training_command_is_cuda_deterministic_and_bound_to_frozen_inputs(tmp_path: Path) -> None:
|
||||||
command = MODULE.training_command(
|
command = MODULE.training_command(
|
||||||
@@ -78,6 +167,7 @@ def test_failed_iteration_builds_train_only_sampling_for_next_checkpoint(tmp_pat
|
|||||||
corpus_manifest=tmp_path / "manifest.json",
|
corpus_manifest=tmp_path / "manifest.json",
|
||||||
assessment=tmp_path / "assessment.json",
|
assessment=tmp_path / "assessment.json",
|
||||||
output_dir=tmp_path / "iteration-001" / "failure-driven-training",
|
output_dir=tmp_path / "iteration-001" / "failure-driven-training",
|
||||||
|
review_audit=tmp_path / "review-audit.json",
|
||||||
)
|
)
|
||||||
assert command[1].endswith("build_failure_driven_yolo_sampling.py")
|
assert command[1].endswith("build_failure_driven_yolo_sampling.py")
|
||||||
assert command[command.index("--summary") + 1].endswith("train-summary.json")
|
assert command[command.index("--summary") + 1].endswith("train-summary.json")
|
||||||
@@ -103,20 +193,23 @@ def test_dry_run_can_gate_existing_checkpoint_without_training(tmp_path: Path) -
|
|||||||
"status": "ok", "low_variance_positive_tile_count": 0,
|
"status": "ok", "low_variance_positive_tile_count": 0,
|
||||||
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
||||||
}))
|
}))
|
||||||
|
manifest = tmp_path / "manifest.json"
|
||||||
|
write_fixture_manifest(manifest)
|
||||||
|
train_yaml = write_fixture_training_release(tmp_path, manifest)
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
[
|
[
|
||||||
sys.executable, str(SCRIPT),
|
sys.executable, str(SCRIPT),
|
||||||
"--initial-model", str(tmp_path / "candidate.pt"),
|
"--initial-model", str(tmp_path / "candidate.pt"),
|
||||||
"--train-yaml", str(tmp_path / "dataset.yaml"),
|
"--train-yaml", str(train_yaml),
|
||||||
"--train-summary", str(tmp_path / "train-summary.json"),
|
"--train-summary", str(tmp_path / "train-summary.json"),
|
||||||
"--dataset-audit", str(audit),
|
"--dataset-audit", str(audit),
|
||||||
"--train-quality-audit", str(quality),
|
"--train-quality-audit", str(quality),
|
||||||
"--calibration-summary", str(tmp_path / "cal.json"),
|
"--calibration-summary", str(tmp_path / "cal.json"),
|
||||||
"--test-summary", str(tmp_path / "test.json"),
|
"--test-summary", str(tmp_path / "test.json"),
|
||||||
"--background-summary", str(tmp_path / "background.json"),
|
"--background-summary", str(tmp_path / "background.json"),
|
||||||
"--corpus-manifest", str(tmp_path / "manifest.json"),
|
"--corpus-manifest", str(manifest),
|
||||||
"--output-dir", str(tmp_path / "output"),
|
"--output-dir", str(tmp_path / "output"),
|
||||||
"--evaluate-initial-model", "--dry-run",
|
"--evaluate-initial-model", "--fixture-mode", "--dry-run",
|
||||||
], capture_output=True, text=True, check=False,
|
], capture_output=True, text=True, check=False,
|
||||||
)
|
)
|
||||||
assert result.returncode == 0
|
assert result.returncode == 0
|
||||||
@@ -139,6 +232,9 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
|
|||||||
"status": "ok", "low_variance_positive_tile_count": 0,
|
"status": "ok", "low_variance_positive_tile_count": 0,
|
||||||
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
||||||
}))
|
}))
|
||||||
|
manifest = tmp_path / "manifest.json"
|
||||||
|
write_fixture_manifest(manifest)
|
||||||
|
train_yaml = write_fixture_training_release(tmp_path, manifest)
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
[
|
[
|
||||||
sys.executable,
|
sys.executable,
|
||||||
@@ -146,7 +242,7 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
|
|||||||
"--initial-model",
|
"--initial-model",
|
||||||
str(tmp_path / "base.pt"),
|
str(tmp_path / "base.pt"),
|
||||||
"--train-yaml",
|
"--train-yaml",
|
||||||
str(tmp_path / "dataset.yaml"),
|
str(train_yaml),
|
||||||
"--train-summary",
|
"--train-summary",
|
||||||
str(tmp_path / "train-summary.json"),
|
str(tmp_path / "train-summary.json"),
|
||||||
"--dataset-audit",
|
"--dataset-audit",
|
||||||
@@ -160,9 +256,10 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
|
|||||||
"--background-summary",
|
"--background-summary",
|
||||||
str(tmp_path / "background.json"),
|
str(tmp_path / "background.json"),
|
||||||
"--corpus-manifest",
|
"--corpus-manifest",
|
||||||
str(tmp_path / "manifest.json"),
|
str(manifest),
|
||||||
"--output-dir",
|
"--output-dir",
|
||||||
str(tmp_path / "output"),
|
str(tmp_path / "output"),
|
||||||
|
"--fixture-mode",
|
||||||
],
|
],
|
||||||
capture_output=True,
|
capture_output=True,
|
||||||
text=True,
|
text=True,
|
||||||
@@ -172,7 +269,45 @@ def test_loop_refuses_failed_dataset_audit(tmp_path: Path) -> None:
|
|||||||
assert "Dataset audit is not eligible for training" in result.stderr
|
assert "Dataset audit is not eligible for training" in result.stderr
|
||||||
|
|
||||||
|
|
||||||
def test_pending_human_review_does_not_block_objective_training() -> None:
|
def test_loop_rejects_manifest_without_source_eligibility_before_cuda_training(tmp_path: Path) -> None:
|
||||||
|
audit = tmp_path / "audit.json"
|
||||||
|
audit.write_text(json.dumps({
|
||||||
|
"status": "needs_human_review", "failures": [],
|
||||||
|
"manifest_immutable": True, "spatial_leakage_status": "ok",
|
||||||
|
}))
|
||||||
|
quality = tmp_path / "quality.json"
|
||||||
|
quality.write_text(json.dumps({
|
||||||
|
"status": "ok", "low_variance_positive_tile_count": 0,
|
||||||
|
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
||||||
|
}))
|
||||||
|
manifest = tmp_path / "manifest.json"
|
||||||
|
manifest.write_text(json.dumps({"samples": [{"sample_slug": "unproven"}]}), encoding="utf-8")
|
||||||
|
|
||||||
|
result = subprocess.run(
|
||||||
|
[
|
||||||
|
sys.executable, str(SCRIPT),
|
||||||
|
"--initial-model", str(tmp_path / "candidate.pt"),
|
||||||
|
"--train-yaml", str(tmp_path / "dataset.yaml"),
|
||||||
|
"--train-summary", str(tmp_path / "train-summary.json"),
|
||||||
|
"--dataset-audit", str(audit),
|
||||||
|
"--train-quality-audit", str(quality),
|
||||||
|
"--calibration-summary", str(tmp_path / "cal.json"),
|
||||||
|
"--test-summary", str(tmp_path / "test.json"),
|
||||||
|
"--background-summary", str(tmp_path / "background.json"),
|
||||||
|
"--corpus-manifest", str(manifest),
|
||||||
|
"--output-dir", str(tmp_path / "output"),
|
||||||
|
"--evaluate-initial-model", "--dry-run",
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.returncode != 0
|
||||||
|
assert "manifest_training_eligibility_missing" in result.stderr
|
||||||
|
|
||||||
|
|
||||||
|
def test_pending_human_review_blocks_operational_training() -> None:
|
||||||
audit = {
|
audit = {
|
||||||
"status": "needs_human_review",
|
"status": "needs_human_review",
|
||||||
"failures": [],
|
"failures": [],
|
||||||
@@ -185,7 +320,58 @@ def test_pending_human_review_does_not_block_objective_training() -> None:
|
|||||||
"status": "ok", "low_variance_positive_tile_count": 0,
|
"status": "ok", "low_variance_positive_tile_count": 0,
|
||||||
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
||||||
}
|
}
|
||||||
assert MODULE.dataset_audit_failures(audit, quality) == []
|
failures = MODULE.dataset_audit_failures(audit, quality)
|
||||||
|
assert "unsupported audit status: needs_human_review" in failures
|
||||||
|
assert "review_complete_not_true" in failures
|
||||||
|
assert "accepted_human_review_evidence_missing" in failures
|
||||||
|
|
||||||
|
|
||||||
|
def test_fixture_mode_can_relax_review_only_after_fixture_manifest_gate() -> None:
|
||||||
|
audit = {
|
||||||
|
"status": "needs_human_review",
|
||||||
|
"failures": [],
|
||||||
|
"manifest_immutable": True,
|
||||||
|
"spatial_leakage_status": "ok",
|
||||||
|
"review_complete": False,
|
||||||
|
}
|
||||||
|
quality = {
|
||||||
|
"status": "ok", "low_variance_positive_tile_count": 0,
|
||||||
|
"label_stats": {"invalid_label_count": 0, "missing_label_file_count": 0},
|
||||||
|
}
|
||||||
|
assert MODULE.dataset_audit_failures(audit, quality, fixture_mode=True) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_operational_dataset_audit_must_be_the_one_bound_into_the_release(tmp_path: Path) -> None:
|
||||||
|
bound = tmp_path / "bound-audit.json"
|
||||||
|
other = tmp_path / "other-audit.json"
|
||||||
|
bound.write_text("{}", encoding="utf-8")
|
||||||
|
other.write_text("{}", encoding="utf-8")
|
||||||
|
release = {"human_review": {"audit_path": str(bound.resolve())}}
|
||||||
|
|
||||||
|
MODULE.assert_dataset_audit_bound_to_release(
|
||||||
|
release=release,
|
||||||
|
dataset_audit=bound,
|
||||||
|
fixture_mode=False,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
MODULE.assert_dataset_audit_bound_to_release(
|
||||||
|
release=release,
|
||||||
|
dataset_audit=other,
|
||||||
|
fixture_mode=False,
|
||||||
|
)
|
||||||
|
except MODULE.TrainingReleaseError as exc:
|
||||||
|
assert "does not match" in str(exc)
|
||||||
|
else:
|
||||||
|
raise AssertionError("unbound dataset audit was accepted")
|
||||||
|
|
||||||
|
|
||||||
|
def test_protected_assessment_feedback_is_terminal_and_cannot_seed_another_yaml() -> None:
|
||||||
|
assert MODULE.protected_feedback_roles(
|
||||||
|
{"status": "continue_training_loop", "test": {"aggregate": {}}, "background": None}
|
||||||
|
) == ["test"]
|
||||||
|
assert MODULE.protected_feedback_roles(
|
||||||
|
{"status": "continue_training_loop", "test": None, "background": {"aggregate": {}}}
|
||||||
|
) == ["background"]
|
||||||
|
|
||||||
|
|
||||||
def test_training_audit_still_fails_closed_on_automated_integrity_gates() -> None:
|
def test_training_audit_still_fails_closed_on_automated_integrity_gates() -> None:
|
||||||
|
|||||||
@@ -0,0 +1,449 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timedelta, timezone
|
||||||
|
from hashlib import sha256
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from shapely.geometry import box
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.services.data_contract_validation import (
|
||||||
|
AttributeRule,
|
||||||
|
BoundingBox,
|
||||||
|
ContractKind,
|
||||||
|
DataAssetValidationInput,
|
||||||
|
DataContract,
|
||||||
|
DataContractRegistry,
|
||||||
|
DataContractValidator,
|
||||||
|
FreshnessRules,
|
||||||
|
GeometryRecord,
|
||||||
|
GeometryRules,
|
||||||
|
LineageEvidence,
|
||||||
|
LineageRules,
|
||||||
|
RasterRules,
|
||||||
|
RequirementLevel,
|
||||||
|
Resolution,
|
||||||
|
ResolutionRules,
|
||||||
|
TransformationEvidence,
|
||||||
|
ValidationStatus,
|
||||||
|
build_default_data_contract_registry,
|
||||||
|
build_label_validation_input,
|
||||||
|
build_model_validation_input,
|
||||||
|
build_raster_ingest_input,
|
||||||
|
build_vector_ingest_input,
|
||||||
|
validate_registered_asset,
|
||||||
|
)
|
||||||
|
from app.services.data_quarantine_service import AssetUse, DataQuarantineService
|
||||||
|
|
||||||
|
|
||||||
|
FIXTURE_ROOT = Path(__file__).resolve().parents[2] / "tests" / "fixtures" / "data-contracts"
|
||||||
|
NOW = datetime(2026, 8, 1, 12, 0, tzinfo=timezone.utc)
|
||||||
|
CHECKSUM_A = "a" * 64
|
||||||
|
|
||||||
|
|
||||||
|
def _fixture_json(name: str) -> tuple[bytes, object]:
|
||||||
|
raw = (FIXTURE_ROOT / name).read_bytes()
|
||||||
|
return raw, json.loads(raw)
|
||||||
|
|
||||||
|
|
||||||
|
def _checksum(raw: bytes) -> str:
|
||||||
|
return sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _lineage_with_transform() -> LineageEvidence:
|
||||||
|
return LineageEvidence(
|
||||||
|
transformations=(
|
||||||
|
TransformationEvidence(
|
||||||
|
name="epsg31370-to-epsg4326",
|
||||||
|
version="1.0.0",
|
||||||
|
checksum_sha256=CHECKSUM_A,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _vector_input_from_fixture(name: str, *, source_crs: str = "EPSG:31370", storage_crs: str = "EPSG:4326") -> DataAssetValidationInput:
|
||||||
|
raw, payload = _fixture_json(name)
|
||||||
|
assert isinstance(payload, dict)
|
||||||
|
return build_vector_ingest_input(
|
||||||
|
asset_id=f"fixture:{name}",
|
||||||
|
source_crs=source_crs,
|
||||||
|
storage_crs=storage_crs,
|
||||||
|
feature_collection=payload,
|
||||||
|
checksum_sha256=_checksum(raw),
|
||||||
|
computed_checksum_sha256=_checksum(raw),
|
||||||
|
content=raw,
|
||||||
|
source_registry_id="source:digitaal-vlaanderen:grb",
|
||||||
|
source_snapshot_id="snapshot:grb:2026-07-31",
|
||||||
|
imported_at=NOW,
|
||||||
|
metadata={"license": "Open Data Licence", "provider": "Digitaal Vlaanderen"},
|
||||||
|
observed_at=NOW - timedelta(days=1),
|
||||||
|
source_version="2026.07.31",
|
||||||
|
lineage=_lineage_with_transform() if source_crs != storage_crs else LineageEvidence(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _issue_codes(report) -> set[str]:
|
||||||
|
return {issue.code for issue in report.issues}
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_vector_contract_accepts_transformed_geojson_with_complete_provenance() -> None:
|
||||||
|
report = validate_registered_asset(_vector_input_from_fixture("vector-building-valid.geojson"), now=NOW)
|
||||||
|
|
||||||
|
assert report.validation_status == ValidationStatus.PASSED
|
||||||
|
assert report.quarantine_status == "not_quarantined"
|
||||||
|
assert report.provenance_status == "complete"
|
||||||
|
assert report.lineage_status == "complete"
|
||||||
|
persisted = report.persistence_fields()
|
||||||
|
assert persisted["data_contract_key"] == "geointel.vector.geojson"
|
||||||
|
assert persisted["data_contract_version"] == "1.0.0"
|
||||||
|
assert persisted["validation_report_json"]["report_sha256"] == report.report_sha256
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_vector_contract_quarantines_lambert_coordinates_mislabelled_as_epsg4326() -> None:
|
||||||
|
report = validate_registered_asset(
|
||||||
|
_vector_input_from_fixture(
|
||||||
|
"vector-lambert-mislabelled-as-4326.geojson",
|
||||||
|
source_crs="EPSG:4326",
|
||||||
|
storage_crs="EPSG:4326",
|
||||||
|
),
|
||||||
|
now=NOW,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert report.validation_status == ValidationStatus.FAILED
|
||||||
|
assert report.quarantine_status == "quarantined"
|
||||||
|
assert "CRS_COORDINATE_DOMAIN_VIOLATION" in _issue_codes(report)
|
||||||
|
|
||||||
|
|
||||||
|
def test_vector_contract_checks_geometry_attributes_bounds_and_topology_fail_closed() -> None:
|
||||||
|
contract = DataContract(
|
||||||
|
key="test.vector.buildings",
|
||||||
|
version="1.0.0",
|
||||||
|
kind=ContractKind.VECTOR,
|
||||||
|
accepted_source_crs=frozenset({"EPSG:4326"}),
|
||||||
|
canonical_storage_crs="EPSG:4326",
|
||||||
|
spatial_domain=BoundingBox(2.0, 49.0, 7.0, 52.0),
|
||||||
|
require_bounds=True,
|
||||||
|
geometry_rules=GeometryRules(
|
||||||
|
allowed_geometry_types=frozenset({"Polygon"}),
|
||||||
|
attribute_rules=(AttributeRule("native_id", accepted_types=("integer",)),),
|
||||||
|
forbid_shared_area=True,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
raw = b"overlapping-vector"
|
||||||
|
asset = DataAssetValidationInput(
|
||||||
|
asset_id="vector:bad",
|
||||||
|
data_contract_key=contract.key,
|
||||||
|
data_contract_version=contract.version,
|
||||||
|
kind=ContractKind.VECTOR,
|
||||||
|
source_crs="EPSG:4326",
|
||||||
|
storage_crs="EPSG:4326",
|
||||||
|
bounds=BoundingBox(4.0, 51.0, 4.1, 51.1),
|
||||||
|
checksum_sha256=_checksum(raw),
|
||||||
|
computed_checksum_sha256=_checksum(raw),
|
||||||
|
content=raw,
|
||||||
|
geometry_records=(
|
||||||
|
GeometryRecord(box(4.0, 51.0, 4.05, 51.05), {"native_id": "wrong-type"}),
|
||||||
|
GeometryRecord(box(4.025, 51.025, 4.075, 51.075), {}),
|
||||||
|
),
|
||||||
|
source_registry_id="source:test",
|
||||||
|
source_snapshot_id="snapshot:test",
|
||||||
|
imported_at=NOW,
|
||||||
|
)
|
||||||
|
|
||||||
|
report = DataContractValidator.validate(contract, asset, now=NOW)
|
||||||
|
|
||||||
|
assert report.validation_status == ValidationStatus.FAILED
|
||||||
|
assert {"ATTRIBUTE_TYPE_INVALID", "ATTRIBUTE_REQUIRED", "TOPOLOGY_SHARED_AREA"} <= _issue_codes(report)
|
||||||
|
assert "BOUNDS_GEOMETRY_MISMATCH" in _issue_codes(report)
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_vector_contract_validates_replayable_large_partition_stream_without_materialising_geometry_list() -> None:
|
||||||
|
"""Regional imports may be large but remain fully schema/domain checked.
|
||||||
|
|
||||||
|
The default contract has no source-specific shared-area rule, so the
|
||||||
|
validator must make its bounds/schema passes over a replayable stream
|
||||||
|
without accumulating every Shapely geometry in memory. A stricter
|
||||||
|
source-specific contract can still opt into a bounded topology batch.
|
||||||
|
"""
|
||||||
|
|
||||||
|
class ReplayableRecords:
|
||||||
|
def __init__(self, count: int) -> None:
|
||||||
|
self.count = count
|
||||||
|
self.iterations = 0
|
||||||
|
|
||||||
|
def __iter__(self):
|
||||||
|
self.iterations += 1
|
||||||
|
for index in range(self.count):
|
||||||
|
yield GeometryRecord(
|
||||||
|
box(4.69, 51.09, 4.70, 51.10),
|
||||||
|
{"partition_feature": index},
|
||||||
|
)
|
||||||
|
|
||||||
|
raw = b"partitioned-vector-stream"
|
||||||
|
records = ReplayableRecords(12_000)
|
||||||
|
asset = DataAssetValidationInput(
|
||||||
|
asset_id="vector:partitioned-stream",
|
||||||
|
data_contract_key="geointel.vector.geojson",
|
||||||
|
data_contract_version="1.0.0",
|
||||||
|
kind=ContractKind.VECTOR,
|
||||||
|
source_crs="EPSG:4326",
|
||||||
|
storage_crs="EPSG:4326",
|
||||||
|
bounds=BoundingBox(4.69, 51.09, 4.70, 51.10),
|
||||||
|
checksum_sha256=_checksum(raw),
|
||||||
|
computed_checksum_sha256=_checksum(raw),
|
||||||
|
content=raw,
|
||||||
|
metadata={"license": "Open Data"},
|
||||||
|
geometry_records=records,
|
||||||
|
source_registry_id="source:grb",
|
||||||
|
source_snapshot_id="snapshot:grb:partitioned",
|
||||||
|
imported_at=NOW,
|
||||||
|
observed_at=NOW,
|
||||||
|
source_version="2026-08-01",
|
||||||
|
)
|
||||||
|
|
||||||
|
report = validate_registered_asset(asset, now=NOW)
|
||||||
|
|
||||||
|
assert report.validation_status == ValidationStatus.PASSED
|
||||||
|
assert records.iterations >= 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_raster_contract_accepts_explicit_units_and_quarantines_stale_bad_profile() -> None:
|
||||||
|
raw = b"raster-stage"
|
||||||
|
valid = build_raster_ingest_input(
|
||||||
|
asset_id="raster:valid",
|
||||||
|
source_crs="EPSG:31370",
|
||||||
|
storage_crs="EPSG:31370",
|
||||||
|
raster_profile={"width": 512, "height": 512, "band_count": 3, "dtype": ["uint8"]},
|
||||||
|
bounds=BoundingBox(193_277.5, 205_708.3, 193_777.5, 206_208.3),
|
||||||
|
resolution=Resolution(0.9765625, 0.9765625, "m"),
|
||||||
|
checksum_sha256=_checksum(raw),
|
||||||
|
computed_checksum_sha256=_checksum(raw),
|
||||||
|
content=raw,
|
||||||
|
source_registry_id="source:orthophoto",
|
||||||
|
source_snapshot_id="snapshot:orthophoto:2026.01",
|
||||||
|
imported_at=NOW,
|
||||||
|
metadata={"license": "Open Data"},
|
||||||
|
observed_at=None,
|
||||||
|
temporal_unknown_reason="latest mosaic has no per-pixel observation date",
|
||||||
|
source_version=None,
|
||||||
|
source_version_unknown_reason="provider did not publish an edition",
|
||||||
|
)
|
||||||
|
assert validate_registered_asset(valid, now=NOW).validation_status == ValidationStatus.PASSED
|
||||||
|
|
||||||
|
strict = DataContract(
|
||||||
|
key="test.raster.strict",
|
||||||
|
version="1.0.0",
|
||||||
|
kind=ContractKind.RASTER,
|
||||||
|
accepted_source_crs=frozenset({"EPSG:31370"}),
|
||||||
|
require_bounds=True,
|
||||||
|
raster_rules=RasterRules(allowed_band_counts=frozenset({3}), allowed_dtypes=frozenset({"uint8"})),
|
||||||
|
resolution_rules=ResolutionRules(allowed_units=frozenset({"m"}), min_x=0.2, max_x=1.0, min_y=0.2, max_y=1.0),
|
||||||
|
freshness_rules=FreshnessRules(observed_at=RequirementLevel.REQUIRED, max_age=timedelta(days=30)),
|
||||||
|
lineage_rules=LineageRules(require_transformation_when_crs_changes=False),
|
||||||
|
)
|
||||||
|
invalid = DataAssetValidationInput(
|
||||||
|
asset_id="raster:bad",
|
||||||
|
data_contract_key=strict.key,
|
||||||
|
data_contract_version=strict.version,
|
||||||
|
kind=ContractKind.RASTER,
|
||||||
|
source_crs="EPSG:31370",
|
||||||
|
storage_crs="EPSG:31370",
|
||||||
|
bounds=BoundingBox(100.0, 100.0, 200.0, 200.0),
|
||||||
|
checksum_sha256=_checksum(raw),
|
||||||
|
computed_checksum_sha256=_checksum(raw),
|
||||||
|
content=raw,
|
||||||
|
raster_profile={"width": 0, "height": 10, "band_count": 2, "dtype": ["float32"]},
|
||||||
|
resolution=Resolution(2.0, 0.1, "degree"),
|
||||||
|
source_registry_id="source:raster",
|
||||||
|
source_snapshot_id="snapshot:raster",
|
||||||
|
imported_at=NOW,
|
||||||
|
observed_at=NOW - timedelta(days=31),
|
||||||
|
)
|
||||||
|
report = DataContractValidator.validate(strict, invalid, now=NOW)
|
||||||
|
|
||||||
|
assert report.validation_status == ValidationStatus.FAILED
|
||||||
|
assert {
|
||||||
|
"RASTER_PROFILE_VALUE_INVALID",
|
||||||
|
"RASTER_BAND_COUNT_NOT_ALLOWED",
|
||||||
|
"RASTER_DTYPE_NOT_ALLOWED",
|
||||||
|
"RESOLUTION_UNIT_NOT_ALLOWED",
|
||||||
|
"RESOLUTION_OUT_OF_RANGE",
|
||||||
|
"FRESHNESS_EXCEEDED",
|
||||||
|
} <= _issue_codes(report)
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_label_and_model_contracts_validate_good_and_bad_fixtures() -> None:
|
||||||
|
valid_raw, valid_labels = _fixture_json("labels-valid.json")
|
||||||
|
invalid_raw, invalid_labels = _fixture_json("labels-invalid.json")
|
||||||
|
assert isinstance(valid_labels, list)
|
||||||
|
assert isinstance(invalid_labels, list)
|
||||||
|
lineage = LineageEvidence(upstream_asset_ids=("image:1",), upstream_checksums_sha256=(CHECKSUM_A,))
|
||||||
|
valid_label = build_label_validation_input(
|
||||||
|
asset_id="label:valid",
|
||||||
|
label_records=valid_labels,
|
||||||
|
checksum_sha256=_checksum(valid_raw),
|
||||||
|
computed_checksum_sha256=_checksum(valid_raw),
|
||||||
|
content=valid_raw,
|
||||||
|
source_registry_id="source:labels",
|
||||||
|
source_snapshot_id="snapshot:labels:1",
|
||||||
|
imported_at=NOW,
|
||||||
|
metadata={
|
||||||
|
"image_checksum_sha256": CHECKSUM_A,
|
||||||
|
"class_ontology_version": "buildings-v1",
|
||||||
|
"source_corpus_manifest_sha256": CHECKSUM_A,
|
||||||
|
},
|
||||||
|
temporal_unknown_reason="labels inherit image observation handling",
|
||||||
|
source_version_unknown_reason="label release is represented by its snapshot",
|
||||||
|
lineage=lineage,
|
||||||
|
)
|
||||||
|
valid_report = validate_registered_asset(valid_label, now=NOW)
|
||||||
|
assert valid_report.validation_status == ValidationStatus.PASSED
|
||||||
|
|
||||||
|
invalid_label = build_label_validation_input(
|
||||||
|
asset_id="label:invalid",
|
||||||
|
label_records=invalid_labels,
|
||||||
|
checksum_sha256=_checksum(invalid_raw),
|
||||||
|
computed_checksum_sha256=_checksum(invalid_raw),
|
||||||
|
content=invalid_raw,
|
||||||
|
source_registry_id="source:labels",
|
||||||
|
source_snapshot_id="snapshot:labels:1",
|
||||||
|
imported_at=NOW,
|
||||||
|
metadata={
|
||||||
|
"image_checksum_sha256": "not-a-sha256",
|
||||||
|
"class_ontology_version": "buildings-v1",
|
||||||
|
"source_corpus_manifest_sha256": CHECKSUM_A,
|
||||||
|
},
|
||||||
|
temporal_unknown_reason="labels inherit image observation handling",
|
||||||
|
source_version_unknown_reason="label release is represented by its snapshot",
|
||||||
|
lineage=lineage,
|
||||||
|
)
|
||||||
|
invalid_report = validate_registered_asset(invalid_label, now=NOW)
|
||||||
|
assert invalid_report.validation_status == ValidationStatus.FAILED
|
||||||
|
assert {
|
||||||
|
"LABEL_CLASS_ID_NOT_ALLOWED",
|
||||||
|
"LABEL_NORMALIZED_COORDINATE_INVALID",
|
||||||
|
"METADATA_CHECKSUM_INVALID",
|
||||||
|
} <= _issue_codes(invalid_report)
|
||||||
|
|
||||||
|
pure_background_raw = b""
|
||||||
|
pure_background_metadata = {
|
||||||
|
"image_checksum_sha256": CHECKSUM_A,
|
||||||
|
"class_ontology_version": "buildings-v1",
|
||||||
|
"source_corpus_manifest_sha256": CHECKSUM_A,
|
||||||
|
"label_mode": "pure_background",
|
||||||
|
"sample_slug": "forest-background-aoi",
|
||||||
|
"split": "train",
|
||||||
|
"raster_dataset_id": "dataset:raster:1",
|
||||||
|
"reference_dataset_id": "dataset:reference:1",
|
||||||
|
"review_decision": "accepted",
|
||||||
|
"reviewer_id": "reviewer@example.test",
|
||||||
|
"reviewed_at": "2026-08-01T11:00:00+00:00",
|
||||||
|
"review_artifact_sha256": CHECKSUM_A,
|
||||||
|
}
|
||||||
|
pure_background = build_label_validation_input(
|
||||||
|
asset_id="label:pure-background",
|
||||||
|
label_records=(),
|
||||||
|
label_mode="pure_background",
|
||||||
|
checksum_sha256=_checksum(pure_background_raw),
|
||||||
|
computed_checksum_sha256=_checksum(pure_background_raw),
|
||||||
|
content=pure_background_raw,
|
||||||
|
source_registry_id="source:labels",
|
||||||
|
source_snapshot_id="snapshot:labels:1",
|
||||||
|
imported_at=NOW,
|
||||||
|
metadata=pure_background_metadata,
|
||||||
|
temporal_unknown_reason="labels inherit image observation handling",
|
||||||
|
source_version_unknown_reason="label release is represented by its snapshot",
|
||||||
|
lineage=LineageEvidence(
|
||||||
|
upstream_asset_ids=("dataset:raster:1", "dataset:reference:1"),
|
||||||
|
upstream_checksums_sha256=(CHECKSUM_A, CHECKSUM_A),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert validate_registered_asset(pure_background, now=NOW).validation_status == ValidationStatus.PASSED
|
||||||
|
|
||||||
|
unmarked_empty = build_label_validation_input(
|
||||||
|
asset_id="label:unmarked-empty",
|
||||||
|
label_records=(),
|
||||||
|
checksum_sha256=_checksum(pure_background_raw),
|
||||||
|
computed_checksum_sha256=_checksum(pure_background_raw),
|
||||||
|
content=pure_background_raw,
|
||||||
|
source_registry_id="source:labels",
|
||||||
|
source_snapshot_id="snapshot:labels:1",
|
||||||
|
imported_at=NOW,
|
||||||
|
metadata={
|
||||||
|
"image_checksum_sha256": CHECKSUM_A,
|
||||||
|
"class_ontology_version": "buildings-v1",
|
||||||
|
"source_corpus_manifest_sha256": CHECKSUM_A,
|
||||||
|
},
|
||||||
|
temporal_unknown_reason="labels inherit image observation handling",
|
||||||
|
source_version_unknown_reason="label release is represented by its snapshot",
|
||||||
|
lineage=LineageEvidence(
|
||||||
|
upstream_asset_ids=("dataset:raster:1", "dataset:reference:1"),
|
||||||
|
upstream_checksums_sha256=(CHECKSUM_A, CHECKSUM_A),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
assert "PURE_BACKGROUND_MODE_REQUIRED" in _issue_codes(validate_registered_asset(unmarked_empty, now=NOW))
|
||||||
|
|
||||||
|
model_raw = b"model-asset"
|
||||||
|
model = build_model_validation_input(
|
||||||
|
asset_id="model:valid",
|
||||||
|
model_metadata={"model_format": "pytorch", "framework": "torch", "class_mapping": {"0": "building"}},
|
||||||
|
checksum_sha256=_checksum(model_raw),
|
||||||
|
computed_checksum_sha256=_checksum(model_raw),
|
||||||
|
content=model_raw,
|
||||||
|
source_registry_id="source:model-registry",
|
||||||
|
source_snapshot_id="snapshot:model:1",
|
||||||
|
imported_at=NOW,
|
||||||
|
source_version="candidate-1",
|
||||||
|
metadata={"training_manifest_sha256": CHECKSUM_A, "runtime_manifest_sha256": CHECKSUM_A},
|
||||||
|
lineage=lineage,
|
||||||
|
)
|
||||||
|
assert validate_registered_asset(model, now=NOW).validation_status == ValidationStatus.PASSED
|
||||||
|
|
||||||
|
|
||||||
|
def test_unknown_contract_and_quarantine_gate_are_deterministic_and_fail_closed() -> None:
|
||||||
|
unknown = DataAssetValidationInput(
|
||||||
|
asset_id="asset:unknown",
|
||||||
|
data_contract_key="does.not.exist",
|
||||||
|
data_contract_version="9.9.9",
|
||||||
|
kind=ContractKind.VECTOR,
|
||||||
|
)
|
||||||
|
report = DataContractRegistry().validate(unknown, now=NOW)
|
||||||
|
assert report.validation_status == ValidationStatus.FAILED
|
||||||
|
assert report.quarantine_status == "quarantined"
|
||||||
|
assert _issue_codes(report) == {"DATA_CONTRACT_UNKNOWN"}
|
||||||
|
|
||||||
|
first = DataQuarantineService.decide(report)
|
||||||
|
second = DataQuarantineService.decide(report)
|
||||||
|
assert first.idempotency_key == second.idempotency_key
|
||||||
|
assert first.reason_codes == ("DATA_CONTRACT_UNKNOWN",)
|
||||||
|
with pytest.raises(AppError, match="cannot enter this pipeline") as exc_info:
|
||||||
|
DataQuarantineService.require_eligible(first, use=AssetUse.PRODUCTION_INFERENCE)
|
||||||
|
assert exc_info.value.code == "DATASET_QUARANTINED"
|
||||||
|
assert exc_info.value.details["use"] == "production_inference"
|
||||||
|
|
||||||
|
clean_report = validate_registered_asset(_vector_input_from_fixture("vector-building-valid.geojson"), now=NOW)
|
||||||
|
release_request = DataQuarantineService.decide(clean_report, previous=first)
|
||||||
|
assert release_request.quarantine_status == "quarantined"
|
||||||
|
assert release_request.requires_explicit_release is True
|
||||||
|
assert release_request.reason_codes == ("QUARANTINE_RELEASE_REQUIRES_EXPLICIT_PERSISTENCE",)
|
||||||
|
|
||||||
|
|
||||||
|
def test_registry_requires_exact_contract_version_and_fingerprints_schema() -> None:
|
||||||
|
registry = build_default_data_contract_registry()
|
||||||
|
version_mismatch = _vector_input_from_fixture("vector-building-valid.geojson")
|
||||||
|
mismatched = DataAssetValidationInput(
|
||||||
|
**{**version_mismatch.__dict__, "data_contract_version": "2.0.0"},
|
||||||
|
)
|
||||||
|
|
||||||
|
report = registry.validate(mismatched, now=NOW)
|
||||||
|
assert report.validation_status == ValidationStatus.FAILED
|
||||||
|
assert "DATA_CONTRACT_UNKNOWN" in _issue_codes(report)
|
||||||
|
|
||||||
|
contract = registry.resolve("geointel.vector.geojson", "1.0.0")
|
||||||
|
assert contract is not None
|
||||||
|
direct_report = DataContractValidator.validate(contract, mismatched, now=NOW)
|
||||||
|
assert direct_report.validation_status == ValidationStatus.FAILED
|
||||||
|
assert "DATA_CONTRACT_IDENTITY_MISMATCH" in _issue_codes(direct_report)
|
||||||
@@ -0,0 +1,316 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from shapely.geometry import box
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.models import Dataset, SourceRegistry, SourceSnapshot
|
||||||
|
from app.services.coverage_registry_service import CoverageRegistryService, SOURCE_DEFINITIONS
|
||||||
|
import app.services.dataset_consumption_gate_service as gate_module
|
||||||
|
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||||
|
from app.services.export_service import ExportService
|
||||||
|
|
||||||
|
|
||||||
|
def _governed_dataset(
|
||||||
|
*,
|
||||||
|
source_key: str = "grb",
|
||||||
|
classification: str = "authoritative",
|
||||||
|
snapshot_freshness_status: str = "current",
|
||||||
|
) -> Dataset:
|
||||||
|
source_id = uuid4()
|
||||||
|
snapshot_id = uuid4()
|
||||||
|
checksum = "a" * 64
|
||||||
|
source = SourceRegistry(
|
||||||
|
id=source_id,
|
||||||
|
source_key=source_key,
|
||||||
|
display_name=f"{source_key} test source",
|
||||||
|
classification=classification,
|
||||||
|
authority_name="GeoIntel test authority",
|
||||||
|
authority_scope_json={"scope": "test"},
|
||||||
|
usage_policy_json={
|
||||||
|
"ground_truth_allowed": classification == "authoritative",
|
||||||
|
"validation_authority": {"building_validation": "primary"}
|
||||||
|
if classification == "authoritative"
|
||||||
|
else {},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
snapshot = SourceSnapshot(
|
||||||
|
id=snapshot_id,
|
||||||
|
source_registry_id=source_id,
|
||||||
|
snapshot_key="test-snapshot",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
freshness_status=snapshot_freshness_status,
|
||||||
|
ingest_status="ingested",
|
||||||
|
)
|
||||||
|
dataset = Dataset(
|
||||||
|
id=uuid4(),
|
||||||
|
project_id=uuid4(),
|
||||||
|
name="governed.tif",
|
||||||
|
dataset_type="raster",
|
||||||
|
source=source_key,
|
||||||
|
source_name=source_key,
|
||||||
|
dataset_role="source",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
source_registry_id=source_id,
|
||||||
|
source_snapshot_id=snapshot_id,
|
||||||
|
data_contract_key="geointel.raster.geotiff",
|
||||||
|
data_contract_version="1.0.0",
|
||||||
|
validation_status="passed",
|
||||||
|
provenance_status="complete",
|
||||||
|
lineage_status="not_applicable",
|
||||||
|
quarantine_status="not_quarantined",
|
||||||
|
status="ready",
|
||||||
|
)
|
||||||
|
dataset.source_registry = source
|
||||||
|
dataset.source_snapshot = snapshot
|
||||||
|
return dataset
|
||||||
|
|
||||||
|
|
||||||
|
def test_governed_dataset_passes_production_inference_and_authoritative_coverage() -> None:
|
||||||
|
dataset = _governed_dataset()
|
||||||
|
|
||||||
|
inference = DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
|
||||||
|
coverage = DatasetConsumptionGate.assert_eligible(dataset, purpose="authoritative_coverage")
|
||||||
|
|
||||||
|
assert inference.eligible is True
|
||||||
|
assert coverage.eligible is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("field", "value", "error_code"),
|
||||||
|
(
|
||||||
|
("provenance_status", "incomplete", "DATASET_PROVENANCE_INCOMPLETE"),
|
||||||
|
("validation_status", "failed", "DATASET_QUARANTINED"),
|
||||||
|
("quarantine_status", "quarantined", "DATASET_QUARANTINED"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
def test_explicit_unsafe_states_can_never_be_relaxed(field: str, value: str, error_code: str) -> None:
|
||||||
|
dataset = _governed_dataset()
|
||||||
|
setattr(dataset, field, value)
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
DatasetConsumptionGate.assert_eligible(
|
||||||
|
dataset,
|
||||||
|
purpose="production_inference",
|
||||||
|
fixture_mode=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert exc_info.value.code == error_code
|
||||||
|
assert field.replace("_status", "") in " ".join(exc_info.value.details["reasons"])
|
||||||
|
|
||||||
|
|
||||||
|
def test_legacy_fixture_can_support_fixture_qa_but_never_authoritative_coverage() -> None:
|
||||||
|
fixture = Dataset(
|
||||||
|
id=uuid4(),
|
||||||
|
project_id=uuid4(),
|
||||||
|
name="fixture.tif",
|
||||||
|
dataset_type="raster",
|
||||||
|
source="fixture",
|
||||||
|
)
|
||||||
|
|
||||||
|
qa = DatasetConsumptionGate.assert_eligible(fixture, purpose="quality_assessment")
|
||||||
|
with pytest.raises(AppError) as inference_error:
|
||||||
|
DatasetConsumptionGate.assert_eligible(
|
||||||
|
fixture,
|
||||||
|
purpose="production_inference",
|
||||||
|
fixture_mode=True,
|
||||||
|
)
|
||||||
|
with pytest.raises(AppError) as export_error:
|
||||||
|
DatasetConsumptionGate.assert_eligible(fixture, purpose="export")
|
||||||
|
with pytest.raises(AppError) as fixture_export_error:
|
||||||
|
DatasetConsumptionGate.assert_eligible(fixture, purpose="export", fixture_mode=True)
|
||||||
|
coverage = DatasetConsumptionGate.evaluate(fixture, purpose="authoritative_coverage")
|
||||||
|
|
||||||
|
assert qa.fixture_legacy_exception is True
|
||||||
|
assert inference_error.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||||
|
assert "fixture_qa_only" in inference_error.value.details["reasons"]
|
||||||
|
assert export_error.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||||
|
assert fixture_export_error.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||||
|
assert "fixture_qa_only" in fixture_export_error.value.details["reasons"]
|
||||||
|
assert coverage.eligible is False
|
||||||
|
assert "fixture_not_authoritative_coverage" in coverage.reasons
|
||||||
|
|
||||||
|
|
||||||
|
def test_unprovenanced_persistent_dataset_is_blocked(monkeypatch) -> None:
|
||||||
|
dataset = Dataset(
|
||||||
|
id=uuid4(),
|
||||||
|
project_id=uuid4(),
|
||||||
|
name="manual.tif",
|
||||||
|
dataset_type="raster",
|
||||||
|
source="manual_upload",
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(gate_module, "sa_inspect", lambda _dataset: SimpleNamespace(transient=False))
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
DatasetConsumptionGate.assert_eligible(
|
||||||
|
dataset,
|
||||||
|
purpose="production_inference",
|
||||||
|
fixture_mode=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||||
|
assert "phase2_provenance_missing" in exc_info.value.details["reasons"]
|
||||||
|
assert "fixture_source_required" in exc_info.value.details["reasons"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_transient_orm_test_double_can_only_bypass_missing_legacy_fields_for_qa() -> None:
|
||||||
|
transient = Dataset(
|
||||||
|
id=uuid4(),
|
||||||
|
project_id=uuid4(),
|
||||||
|
name="transient-test.tif",
|
||||||
|
dataset_type="raster",
|
||||||
|
source="manual_upload",
|
||||||
|
)
|
||||||
|
|
||||||
|
decision = DatasetConsumptionGate.assert_eligible(transient, purpose="quality_assessment")
|
||||||
|
coverage = DatasetConsumptionGate.evaluate(transient, purpose="authoritative_coverage")
|
||||||
|
with pytest.raises(AppError) as production_error:
|
||||||
|
DatasetConsumptionGate.assert_eligible(transient, purpose="production_inference")
|
||||||
|
|
||||||
|
assert decision.fixture_legacy_exception is True
|
||||||
|
assert production_error.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||||
|
assert coverage.eligible is False
|
||||||
|
assert "phase2_provenance_missing" in coverage.reasons
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("purpose", ("production_inference", "derived_processing", "export"))
|
||||||
|
def test_passed_manual_or_experimental_dataset_cannot_cross_production_boundary(purpose: str) -> None:
|
||||||
|
"""A syntactically valid manual upload remains experimental, never production-ready."""
|
||||||
|
|
||||||
|
manual = _governed_dataset(source_key="manual", classification="experimental")
|
||||||
|
manual.source = "manual_upload"
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
DatasetConsumptionGate.assert_eligible(manual, purpose=purpose) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||||
|
assert "experimental_source_not_allowed_for_purpose" in exc_info.value.details["reasons"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_reference_validation_requires_authoritative_ground_truth_reference() -> None:
|
||||||
|
reference = _governed_dataset()
|
||||||
|
reference.dataset_type = "vector"
|
||||||
|
reference.dataset_role = "reference"
|
||||||
|
|
||||||
|
decision = DatasetConsumptionGate.assert_eligible(
|
||||||
|
reference,
|
||||||
|
purpose="reference_validation",
|
||||||
|
reference_task="building_validation",
|
||||||
|
)
|
||||||
|
assert decision.eligible is True
|
||||||
|
|
||||||
|
reference.source_registry.classification = "corroborative"
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
DatasetConsumptionGate.assert_eligible(
|
||||||
|
reference,
|
||||||
|
purpose="reference_validation",
|
||||||
|
reference_task="building_validation",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||||
|
assert "reference_source_not_authoritative" in exc_info.value.details["reasons"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_pending_regional_building_authority_cannot_become_truth_without_approval() -> None:
|
||||||
|
reference = _governed_dataset(source_key="spw_picc", classification="authoritative")
|
||||||
|
reference.dataset_type = "vector"
|
||||||
|
reference.dataset_role = "reference"
|
||||||
|
reference.source_registry.authority_scope_json = {"zone": "Wallonia"}
|
||||||
|
reference.source_registry.usage_policy_json = {
|
||||||
|
"ground_truth_allowed": True,
|
||||||
|
"validation_authority": {"building_validation": "regional_primary_pending_contract"},
|
||||||
|
}
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
DatasetConsumptionGate.assert_eligible(
|
||||||
|
reference,
|
||||||
|
purpose="reference_validation",
|
||||||
|
reference_task="building_validation",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||||
|
assert "reference_task_authority_not_approved" in exc_info.value.details["reasons"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_source_snapshot_must_belong_to_the_dataset_source_registry() -> None:
|
||||||
|
dataset = _governed_dataset()
|
||||||
|
dataset.source_snapshot.source_registry_id = uuid4()
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
|
||||||
|
|
||||||
|
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||||
|
assert "source_snapshot_registry_mismatch" in exc_info.value.details["reasons"]
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("freshness_status", ("unknown", "review_required", "due", "stale"))
|
||||||
|
def test_non_consumable_source_snapshot_freshness_is_blocked_at_production_boundaries(
|
||||||
|
freshness_status: str,
|
||||||
|
) -> None:
|
||||||
|
dataset = _governed_dataset(snapshot_freshness_status=freshness_status)
|
||||||
|
|
||||||
|
for purpose in ("production_inference", "authoritative_coverage"):
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
DatasetConsumptionGate.assert_eligible(dataset, purpose=purpose) # type: ignore[arg-type]
|
||||||
|
|
||||||
|
assert exc_info.value.code == "DATASET_PROVENANCE_INCOMPLETE"
|
||||||
|
assert "source_snapshot_freshness_not_eligible" in exc_info.value.details["reasons"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_coverage_registry_ignores_explicitly_incomplete_materialization() -> None:
|
||||||
|
definition = next(item for item in SOURCE_DEFINITIONS if item.contract.source_name == "digitaal_vlaanderen")
|
||||||
|
unsafe_materialization = SimpleNamespace(
|
||||||
|
id=uuid4(),
|
||||||
|
status="ready",
|
||||||
|
source_name="grb",
|
||||||
|
validation_status="passed",
|
||||||
|
provenance_status="incomplete",
|
||||||
|
lineage_status="complete",
|
||||||
|
quarantine_status="not_quarantined",
|
||||||
|
)
|
||||||
|
|
||||||
|
matches, fully_covered = CoverageRegistryService._matching_datasets(
|
||||||
|
[unsafe_materialization],
|
||||||
|
definition,
|
||||||
|
"buildings",
|
||||||
|
"flanders",
|
||||||
|
box(4.0, 50.8, 4.1, 50.9),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert matches == []
|
||||||
|
assert fully_covered is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_vector_export_is_fail_closed_before_selection(monkeypatch) -> None:
|
||||||
|
dataset = _governed_dataset()
|
||||||
|
dataset.dataset_type = "vector"
|
||||||
|
dataset.status = "quarantined"
|
||||||
|
queried = False
|
||||||
|
|
||||||
|
class _Session:
|
||||||
|
@staticmethod
|
||||||
|
def get(model, item_id):
|
||||||
|
return dataset if model is Dataset and item_id == dataset.id else None
|
||||||
|
|
||||||
|
def _unexpected_selection(*_args, **_kwargs):
|
||||||
|
nonlocal queried
|
||||||
|
queried = True
|
||||||
|
raise AssertionError("unsafe dataset must be rejected before querying vector features")
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.export_service.VectorFeatureService.select_features_by_bbox",
|
||||||
|
_unexpected_selection,
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
ExportService.export_vector_selection_geojson(
|
||||||
|
_Session(),
|
||||||
|
dataset.id,
|
||||||
|
{"min_x": 4.0, "min_y": 50.0, "max_x": 4.1, "max_y": 50.1, "crs": "EPSG:4326"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert exc_info.value.code == "DATASET_QUARANTINED"
|
||||||
|
assert queried is False
|
||||||
@@ -74,6 +74,8 @@ def test_all_in_one_dockerfile_copies_operator_scripts_for_runtime_use() -> None
|
|||||||
"audit_operator_yolo_dataset_quality.py",
|
"audit_operator_yolo_dataset_quality.py",
|
||||||
"render_operator_yolo_label_qa_contact_sheets.py",
|
"render_operator_yolo_label_qa_contact_sheets.py",
|
||||||
"train_operator_yolo_detector.sh",
|
"train_operator_yolo_detector.sh",
|
||||||
|
"training_dataset_eligibility.py",
|
||||||
|
"training_release_manifest.py",
|
||||||
"verify_real_data_detection_qa_workflow.sh",
|
"verify_real_data_detection_qa_workflow.sh",
|
||||||
"run_detection_quality_matrix.sh",
|
"run_detection_quality_matrix.sh",
|
||||||
"run_multi_sample_detection_quality_matrix.sh",
|
"run_multi_sample_detection_quality_matrix.sh",
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ from __future__ import annotations
|
|||||||
import importlib.util
|
import importlib.util
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
SCRIPT = Path(__file__).parents[2] / "scripts" / "build_failure_driven_yolo_sampling.py"
|
SCRIPT = Path(__file__).parents[2] / "scripts" / "build_failure_driven_yolo_sampling.py"
|
||||||
SPEC = importlib.util.spec_from_file_location("failure_sampling", SCRIPT)
|
SPEC = importlib.util.spec_from_file_location("failure_sampling", SCRIPT)
|
||||||
@@ -22,7 +24,7 @@ def test_dataset_validation_source_preserves_manifest_path(tmp_path: Path):
|
|||||||
assert MODULE.dataset_validation_source(source) == "/data/source/internal-val.txt"
|
assert MODULE.dataset_validation_source(source) == "/data/source/internal-val.txt"
|
||||||
|
|
||||||
|
|
||||||
def test_sampling_repeats_only_failed_region_train_tiles() -> None:
|
def test_sampling_rejects_protected_test_and_background_feedback() -> None:
|
||||||
manifest = {
|
manifest = {
|
||||||
"samples": [
|
"samples": [
|
||||||
{"sample_slug": "train-fl", "split": "train", "region": "flanders"},
|
{"sample_slug": "train-fl", "split": "train", "region": "flanders"},
|
||||||
@@ -54,15 +56,10 @@ def test_sampling_repeats_only_failed_region_train_tiles() -> None:
|
|||||||
},
|
},
|
||||||
"background": {"pure_empty_false_positives": 2},
|
"background": {"pure_empty_false_positives": 2},
|
||||||
}
|
}
|
||||||
paths, metadata = MODULE.build_sampling(
|
with pytest.raises(ValueError, match="protected test/background evidence"):
|
||||||
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0
|
MODULE.build_sampling(
|
||||||
)
|
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0
|
||||||
assert paths.count(str(Path("/tmp/fl-pos.png").resolve())) == 3
|
)
|
||||||
assert paths.count(str(Path("/tmp/fl-neg.png").resolve())) == 4
|
|
||||||
assert paths.count(str(Path("/tmp/wa-pos.png").resolve())) == 1
|
|
||||||
assert not any("protected" in path for path in paths)
|
|
||||||
assert metadata["protected_samples_in_training"] == []
|
|
||||||
assert metadata["weak_recall_regions"] == ["flanders"]
|
|
||||||
|
|
||||||
|
|
||||||
def test_sampling_can_use_calibration_before_test_is_opened() -> None:
|
def test_sampling_can_use_calibration_before_test_is_opened() -> None:
|
||||||
@@ -171,7 +168,7 @@ def test_sampling_targets_failed_calibration_contexts_without_using_protected_ti
|
|||||||
assert metadata["recall_dominant_regions"] == ["flanders"]
|
assert metadata["recall_dominant_regions"] == ["flanders"]
|
||||||
|
|
||||||
|
|
||||||
def test_recall_dominance_does_not_suppress_negatives_when_background_gate_failed() -> None:
|
def test_sampling_rejects_background_feedback_after_a_protected_background_opening() -> None:
|
||||||
manifest = {"samples": [
|
manifest = {"samples": [
|
||||||
{"sample_slug": "positive", "split": "train", "region": "flanders", "context": "industrial"},
|
{"sample_slug": "positive", "split": "train", "region": "flanders", "context": "industrial"},
|
||||||
{"sample_slug": "negative", "split": "train", "region": "flanders", "context": "industrial-hard-negative"},
|
{"sample_slug": "negative", "split": "train", "region": "flanders", "context": "industrial-hard-negative"},
|
||||||
@@ -190,12 +187,10 @@ def test_recall_dominance_does_not_suppress_negatives_when_background_gate_faile
|
|||||||
"background": {"pure_empty_false_positives": 1},
|
"background": {"pure_empty_false_positives": 1},
|
||||||
}
|
}
|
||||||
|
|
||||||
paths, metadata = MODULE.build_sampling(
|
with pytest.raises(ValueError, match="protected background evidence"):
|
||||||
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0,
|
MODULE.build_sampling(
|
||||||
)
|
summary=summary, manifest=manifest, assessment=assessment, max_region_share=1.0,
|
||||||
|
)
|
||||||
assert paths.count(str(Path("/tmp/negative.png").resolve())) == 4
|
|
||||||
assert metadata["recall_dominant_regions"] == []
|
|
||||||
|
|
||||||
|
|
||||||
def test_region_cap_drops_only_repeats_and_preserves_every_unique_tile() -> None:
|
def test_region_cap_drops_only_repeats_and_preserves_every_unique_tile() -> None:
|
||||||
|
|||||||
@@ -0,0 +1,765 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from hashlib import sha256
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from pyproj import Transformer
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.models import (
|
||||||
|
Area,
|
||||||
|
Dataset,
|
||||||
|
DatasetQuarantine,
|
||||||
|
DatasetVersion,
|
||||||
|
Project,
|
||||||
|
SourceRegistry,
|
||||||
|
SourceSnapshot,
|
||||||
|
VectorFeature,
|
||||||
|
)
|
||||||
|
from app.services.dataset_service import DatasetService, _PartitionedGeoJsonRecords
|
||||||
|
from app.services.vector_operations_service import VectorOperationsService
|
||||||
|
|
||||||
|
|
||||||
|
class _Query:
|
||||||
|
def __init__(self, session: "_Session", model: type) -> None:
|
||||||
|
self.session = session
|
||||||
|
self.model = model
|
||||||
|
self.predicates = []
|
||||||
|
|
||||||
|
def filter(self, *predicates):
|
||||||
|
self.predicates.extend(predicates)
|
||||||
|
return self
|
||||||
|
|
||||||
|
def one_or_none(self):
|
||||||
|
matches = self._matches()
|
||||||
|
if len(matches) > 1:
|
||||||
|
raise AssertionError(
|
||||||
|
f"expected one {self.model.__name__}, found {len(matches)}"
|
||||||
|
)
|
||||||
|
return matches[0] if matches else None
|
||||||
|
|
||||||
|
def all(self):
|
||||||
|
return self._matches()
|
||||||
|
|
||||||
|
def _matches(self):
|
||||||
|
matches = list(self.session.rows.get(self.model, []))
|
||||||
|
for predicate in self.predicates:
|
||||||
|
field_name = predicate.left.key
|
||||||
|
expected = predicate.right.value
|
||||||
|
operator_name = getattr(predicate.operator, "__name__", "")
|
||||||
|
if operator_name == "in_op":
|
||||||
|
matches = [
|
||||||
|
item for item in matches if getattr(item, field_name) in expected
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
matches = [
|
||||||
|
item for item in matches if getattr(item, field_name) == expected
|
||||||
|
]
|
||||||
|
return matches
|
||||||
|
|
||||||
|
|
||||||
|
class _Session:
|
||||||
|
"""Small ORM-shaped harness that exercises the real governed path."""
|
||||||
|
|
||||||
|
def __init__(self, project: Project) -> None:
|
||||||
|
self.rows: dict[type, list[object]] = {Project: [project]}
|
||||||
|
self.commits = 0
|
||||||
|
self.rollbacks = 0
|
||||||
|
self.flushes = 0
|
||||||
|
|
||||||
|
def get(self, model: type, item_id: UUID):
|
||||||
|
return next(
|
||||||
|
(
|
||||||
|
item
|
||||||
|
for item in self.rows.get(model, [])
|
||||||
|
if getattr(item, "id", None) == item_id
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
|
||||||
|
def query(self, model: type) -> _Query:
|
||||||
|
return _Query(self, model)
|
||||||
|
|
||||||
|
def add(self, item: object) -> None:
|
||||||
|
if getattr(item, "id", None) is None:
|
||||||
|
setattr(item, "id", uuid4())
|
||||||
|
self.rows.setdefault(type(item), []).append(item)
|
||||||
|
|
||||||
|
def flush(self) -> None:
|
||||||
|
self.flushes += 1
|
||||||
|
|
||||||
|
def commit(self) -> None:
|
||||||
|
self.commits += 1
|
||||||
|
|
||||||
|
def rollback(self) -> None:
|
||||||
|
self.rollbacks += 1
|
||||||
|
|
||||||
|
def refresh(self, _item: object) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def expunge(self, _item: object) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _storage_info(tmp_path: Path, content: bytes) -> dict[str, object]:
|
||||||
|
path = tmp_path / "grb-buildings.geojson"
|
||||||
|
path.write_bytes(content)
|
||||||
|
return {
|
||||||
|
"storage_path": str(path),
|
||||||
|
"original_filename": path.name,
|
||||||
|
"stored_filename": path.name,
|
||||||
|
"content_type": "application/geo+json",
|
||||||
|
"size_bytes": len(content),
|
||||||
|
"checksum_sha256": sha256(content).hexdigest(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _valid_payload() -> bytes:
|
||||||
|
return json.dumps(
|
||||||
|
{
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"id": "gbg-1",
|
||||||
|
"properties": {"id": "gbg-1"},
|
||||||
|
"geometry": {
|
||||||
|
"type": "Polygon",
|
||||||
|
"coordinates": [
|
||||||
|
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _grb_payload_without_required_id() -> bytes:
|
||||||
|
return json.dumps(
|
||||||
|
{
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"crs": {"type": "name", "properties": {"name": "EPSG:4326"}},
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"properties": {"unrelated": "not a GRB identity"},
|
||||||
|
"geometry": {
|
||||||
|
"type": "Polygon",
|
||||||
|
"coordinates": [
|
||||||
|
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _lambert_grb_payload() -> tuple[bytes, tuple[float, float, float, float]]:
|
||||||
|
"""Create a valid GRB-shaped source artifact in its declared native CRS."""
|
||||||
|
|
||||||
|
longitude, latitude = 4.70, 51.10
|
||||||
|
max_longitude, max_latitude = 4.7001, 51.1001
|
||||||
|
to_lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
||||||
|
lambert_ring = [
|
||||||
|
to_lambert.transform(longitude, latitude),
|
||||||
|
to_lambert.transform(max_longitude, latitude),
|
||||||
|
to_lambert.transform(max_longitude, max_latitude),
|
||||||
|
to_lambert.transform(longitude, latitude),
|
||||||
|
]
|
||||||
|
return (
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"crs": {"type": "name", "properties": {"name": "EPSG:31370"}},
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"id": "GBG.lambert.1",
|
||||||
|
"properties": {"id": "GBG.lambert.1"},
|
||||||
|
"geometry": {"type": "Polygon", "coordinates": [lambert_ring]},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
).encode("utf-8"),
|
||||||
|
(longitude, latitude, max_longitude, max_latitude),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_governed_vector_import_persists_snapshot_contract_and_queryable_features(
|
||||||
|
monkeypatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
project = Project(id=uuid4(), name="Phase 2 governed ingest")
|
||||||
|
db = _Session(project)
|
||||||
|
raw = _valid_payload()
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||||
|
lambda **_kwargs: _storage_info(tmp_path, raw),
|
||||||
|
)
|
||||||
|
|
||||||
|
result = DatasetService.import_vector_bytes(
|
||||||
|
db,
|
||||||
|
project_id=project.id,
|
||||||
|
filename="grb-buildings.geojson",
|
||||||
|
content=raw,
|
||||||
|
source="grb_wfs",
|
||||||
|
source_name="grb",
|
||||||
|
dataset_role="reference",
|
||||||
|
reference_layer_name="buildings",
|
||||||
|
source_metadata={
|
||||||
|
"license": "Open data",
|
||||||
|
"source_url": "https://example.invalid/grb",
|
||||||
|
},
|
||||||
|
provenance_metadata={"adapter": "test"},
|
||||||
|
temporal_series_key="grb:2026-08",
|
||||||
|
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||||
|
source_version="2026-08-01",
|
||||||
|
temporal_granularity="snapshot",
|
||||||
|
)
|
||||||
|
|
||||||
|
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
|
||||||
|
snapshot = db.rows[SourceSnapshot][0]
|
||||||
|
source = db.rows[SourceRegistry][0]
|
||||||
|
assert result.status == "ready"
|
||||||
|
assert dataset.source_name == "grb"
|
||||||
|
assert dataset.source_registry_id == source.id
|
||||||
|
assert dataset.source_snapshot_id == snapshot.id
|
||||||
|
assert dataset.validation_status == "passed"
|
||||||
|
assert dataset.provenance_status == "complete"
|
||||||
|
assert dataset.lineage_status == "complete"
|
||||||
|
assert dataset.quarantine_status == "not_quarantined"
|
||||||
|
assert dataset.crs == "EPSG:4326"
|
||||||
|
assert snapshot.checksum_sha256 == sha256(raw).hexdigest()
|
||||||
|
assert len(db.rows[VectorFeature]) == 1
|
||||||
|
assert db.commits == 1
|
||||||
|
|
||||||
|
# A retry with identical governed evidence is idempotent and does not
|
||||||
|
# create a second source snapshot, dataset or vector feature.
|
||||||
|
repeated = DatasetService.import_vector_bytes(
|
||||||
|
db,
|
||||||
|
project_id=project.id,
|
||||||
|
filename="grb-buildings.geojson",
|
||||||
|
content=raw,
|
||||||
|
source="grb_wfs",
|
||||||
|
source_name="grb",
|
||||||
|
dataset_role="reference",
|
||||||
|
reference_layer_name="buildings",
|
||||||
|
source_metadata={"license": "Open data"},
|
||||||
|
provenance_metadata={"adapter": "test"},
|
||||||
|
temporal_series_key="grb:2026-08",
|
||||||
|
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||||
|
source_version="2026-08-01",
|
||||||
|
temporal_granularity="snapshot",
|
||||||
|
)
|
||||||
|
assert repeated.id == result.id
|
||||||
|
assert len(db.rows[Dataset]) == 1
|
||||||
|
assert len(db.rows[SourceSnapshot]) == 1
|
||||||
|
assert len(db.rows[VectorFeature]) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_governed_lambert_geojson_persists_canonical_consumption_bytes_and_provenance_evidence(
|
||||||
|
monkeypatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
"""Projected source bytes must never be the file that vector operations consume."""
|
||||||
|
|
||||||
|
project = Project(id=uuid4(), name="Canonical GeoJSON storage")
|
||||||
|
db = _Session(project)
|
||||||
|
raw, (longitude, latitude, max_longitude, max_latitude) = _lambert_grb_payload()
|
||||||
|
consumption_path = tmp_path / "consumption" / "grb-buildings.geojson"
|
||||||
|
provenance_path = tmp_path / "provenance" / "grb-buildings.geojson"
|
||||||
|
|
||||||
|
def _persist_dataset_file(**kwargs):
|
||||||
|
stored = kwargs["content"]
|
||||||
|
consumption_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
consumption_path.write_bytes(stored)
|
||||||
|
return _storage_info(consumption_path.parent, stored)
|
||||||
|
|
||||||
|
def _persist_file(storage_path, content, original_filename, content_type):
|
||||||
|
del storage_path, original_filename, content_type
|
||||||
|
provenance_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
provenance_path.write_bytes(content)
|
||||||
|
return _storage_info(provenance_path.parent, content)
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||||
|
_persist_dataset_file,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.dataset_service.StorageService.persist_file",
|
||||||
|
_persist_file,
|
||||||
|
)
|
||||||
|
|
||||||
|
result = DatasetService.import_vector_bytes(
|
||||||
|
db,
|
||||||
|
project_id=project.id,
|
||||||
|
filename="grb-lambert.geojson",
|
||||||
|
content=raw,
|
||||||
|
source="grb_wfs",
|
||||||
|
source_name="grb",
|
||||||
|
dataset_role="reference",
|
||||||
|
reference_layer_name="buildings",
|
||||||
|
source_metadata={"license": "Open data"},
|
||||||
|
provenance_metadata={"adapter": "test"},
|
||||||
|
temporal_series_key="grb:lambert:2026-08",
|
||||||
|
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||||
|
source_version="2026-08-01-lambert",
|
||||||
|
temporal_granularity="snapshot",
|
||||||
|
)
|
||||||
|
|
||||||
|
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
|
||||||
|
dataset_version = db.rows[DatasetVersion][0]
|
||||||
|
snapshot = db.rows[SourceSnapshot][0]
|
||||||
|
canonical_bytes = Path(str(dataset.storage_path)).read_bytes()
|
||||||
|
canonical_payload = json.loads(canonical_bytes)
|
||||||
|
source_artifact = dataset.provenance_metadata["source_artifact"]
|
||||||
|
|
||||||
|
assert result.status == "ready"
|
||||||
|
assert dataset.crs == "EPSG:4326"
|
||||||
|
assert canonical_payload["crs"]["properties"]["name"] == "EPSG:4326"
|
||||||
|
assert canonical_payload["features"][0]["geometry"]["coordinates"][0][0] == pytest.approx(
|
||||||
|
[longitude, latitude], abs=0.000001
|
||||||
|
)
|
||||||
|
assert sha256(canonical_bytes).hexdigest() == dataset.checksum_sha256
|
||||||
|
assert dataset_version.checksum_sha256 == dataset.checksum_sha256
|
||||||
|
assert snapshot.checksum_sha256 == dataset.checksum_sha256
|
||||||
|
assert source_artifact["retention"] == "provenance_evidence_only"
|
||||||
|
assert source_artifact["checksum_sha256"] == sha256(raw).hexdigest()
|
||||||
|
assert source_artifact["storage_path"] != dataset.storage_path
|
||||||
|
assert Path(source_artifact["storage_path"]).read_bytes() == raw
|
||||||
|
assert dataset.provenance_metadata["canonical_consumption_artifact"] == {
|
||||||
|
"checksum_sha256": dataset.checksum_sha256,
|
||||||
|
"crs": "EPSG:4326",
|
||||||
|
"storage_role": "dataset_consumption",
|
||||||
|
}
|
||||||
|
|
||||||
|
inspection = VectorOperationsService.inspect(db, dataset.id)
|
||||||
|
assert inspection.crs == "EPSG:4326"
|
||||||
|
assert inspection.bounds_json == {
|
||||||
|
"min_x": pytest.approx(longitude, abs=0.000001),
|
||||||
|
"min_y": pytest.approx(latitude, abs=0.000001),
|
||||||
|
"max_x": pytest.approx(max_longitude, abs=0.000001),
|
||||||
|
"max_y": pytest.approx(max_latitude, abs=0.000001),
|
||||||
|
}
|
||||||
|
response_payload = DatasetService.get_dataset_geojson(db, dataset.id)
|
||||||
|
assert response_payload["features"][0]["geometry"]["coordinates"][0][0] == pytest.approx(
|
||||||
|
[longitude, latitude], abs=0.000001
|
||||||
|
)
|
||||||
|
|
||||||
|
# The storage identity is enforced at the operation boundary too; a
|
||||||
|
# replacement with different canonical bytes is not silently processed.
|
||||||
|
Path(str(dataset.storage_path)).write_bytes(canonical_bytes + b"\n")
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
VectorOperationsService.inspect(db, dataset.id)
|
||||||
|
assert exc_info.value.code == "DATASET_STORAGE_CHECKSUM_MISMATCH"
|
||||||
|
|
||||||
|
|
||||||
|
def test_metadata_refresh_refuses_mutated_governed_artifact(
|
||||||
|
monkeypatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
"""A passed snapshot cannot be silently re-described from mutable storage."""
|
||||||
|
|
||||||
|
project = Project(id=uuid4(), name="Phase 2 immutable refresh")
|
||||||
|
db = _Session(project)
|
||||||
|
raw = _valid_payload()
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||||
|
lambda **_kwargs: _storage_info(tmp_path, raw),
|
||||||
|
)
|
||||||
|
result = DatasetService.import_vector_bytes(
|
||||||
|
db,
|
||||||
|
project_id=project.id,
|
||||||
|
filename="grb-buildings.geojson",
|
||||||
|
content=raw,
|
||||||
|
source="grb_wfs",
|
||||||
|
source_name="grb",
|
||||||
|
dataset_role="reference",
|
||||||
|
reference_layer_name="buildings",
|
||||||
|
source_metadata={"license": "Open data"},
|
||||||
|
provenance_metadata={"adapter": "test"},
|
||||||
|
temporal_series_key="grb:2026-08",
|
||||||
|
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||||
|
source_version="2026-08-01",
|
||||||
|
temporal_granularity="snapshot",
|
||||||
|
)
|
||||||
|
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
|
||||||
|
original_checksum = dataset.checksum_sha256
|
||||||
|
original_metadata = dict(dataset.metadata_json or {})
|
||||||
|
original_commit_count = db.commits
|
||||||
|
|
||||||
|
# Simulate an out-of-band storage replacement at the same path. The
|
||||||
|
# refresh endpoint must not parse it into an already-passed contract row.
|
||||||
|
Path(str(dataset.storage_path)).write_bytes(_grb_payload_without_required_id())
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
DatasetService.refresh_metadata(db, dataset.id)
|
||||||
|
|
||||||
|
assert exc_info.value.code == "GOVERNED_DATASET_REINGEST_REQUIRED"
|
||||||
|
assert dataset.status == "ready"
|
||||||
|
assert dataset.validation_status == "passed"
|
||||||
|
assert dataset.checksum_sha256 == original_checksum
|
||||||
|
assert dataset.metadata_json == original_metadata
|
||||||
|
assert db.commits == original_commit_count
|
||||||
|
|
||||||
|
|
||||||
|
def test_governed_import_quarantines_bad_artifacts_and_refuses_unknown_source(
|
||||||
|
monkeypatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
project = Project(id=uuid4(), name="Phase 2 quarantine")
|
||||||
|
db = _Session(project)
|
||||||
|
raw = b'{"type":"FeatureCollection","features":[]}'
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||||
|
lambda **_kwargs: _storage_info(tmp_path, raw),
|
||||||
|
)
|
||||||
|
|
||||||
|
quarantined = DatasetService.import_vector_bytes(
|
||||||
|
db,
|
||||||
|
project_id=project.id,
|
||||||
|
filename="empty.geojson",
|
||||||
|
content=raw,
|
||||||
|
source="grb_wfs",
|
||||||
|
source_name="grb",
|
||||||
|
dataset_role="reference",
|
||||||
|
reference_layer_name="buildings",
|
||||||
|
source_metadata={"license": "Open data"},
|
||||||
|
provenance_metadata={},
|
||||||
|
temporal_series_key="grb:2026-08-empty",
|
||||||
|
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||||
|
source_version="2026-08-01-empty",
|
||||||
|
temporal_granularity="snapshot",
|
||||||
|
)
|
||||||
|
assert quarantined.status == "quarantined"
|
||||||
|
assert quarantined.validation_status == "failed"
|
||||||
|
assert quarantined.quarantine_status == "quarantined"
|
||||||
|
assert len(db.rows[DatasetQuarantine]) == 1
|
||||||
|
assert db.rows[SourceSnapshot][0].ingest_status == "quarantined"
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
DatasetService.import_vector_bytes(
|
||||||
|
db,
|
||||||
|
project_id=project.id,
|
||||||
|
filename="unregistered.geojson",
|
||||||
|
content=_valid_payload(),
|
||||||
|
source="caller_controlled",
|
||||||
|
source_name="caller_claimed_grb",
|
||||||
|
dataset_role="reference",
|
||||||
|
reference_layer_name="buildings",
|
||||||
|
source_metadata={"license": "Open data"},
|
||||||
|
provenance_metadata={},
|
||||||
|
)
|
||||||
|
assert exc_info.value.code == "SOURCE_REGISTRY_ENTRY_NOT_FOUND"
|
||||||
|
|
||||||
|
|
||||||
|
def test_governed_grb_vector_quarantines_missing_server_owned_required_attribute(
|
||||||
|
monkeypatch, tmp_path: Path
|
||||||
|
) -> None:
|
||||||
|
project = Project(id=uuid4(), name="Phase 2 source schema")
|
||||||
|
db = _Session(project)
|
||||||
|
raw = _grb_payload_without_required_id()
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.dataset_service.StorageService.persist_dataset_file",
|
||||||
|
lambda **kwargs: _storage_info(tmp_path, kwargs["content"]),
|
||||||
|
)
|
||||||
|
|
||||||
|
quarantined = DatasetService.import_vector_bytes(
|
||||||
|
db,
|
||||||
|
project_id=project.id,
|
||||||
|
filename="grb-missing-id.geojson",
|
||||||
|
content=raw,
|
||||||
|
source="grb_wfs",
|
||||||
|
source_name="grb",
|
||||||
|
dataset_role="reference",
|
||||||
|
reference_layer_name="buildings",
|
||||||
|
source_metadata={"license": "Open data"},
|
||||||
|
provenance_metadata={"adapter": "test"},
|
||||||
|
temporal_series_key="grb:missing-id",
|
||||||
|
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||||
|
source_version="2026-08-01-missing-id",
|
||||||
|
temporal_granularity="snapshot",
|
||||||
|
)
|
||||||
|
|
||||||
|
dataset = next(item for item in db.rows[Dataset] if item.id == quarantined.id)
|
||||||
|
assert quarantined.status == "quarantined"
|
||||||
|
assert dataset.validation_status == "failed"
|
||||||
|
assert dataset.quarantine_status == "quarantined"
|
||||||
|
issue = dataset.validation_report_json["issues"][0]
|
||||||
|
assert issue["code"] == "SOURCE_SCHEMA_REQUIRED_ATTRIBUTE_MISSING"
|
||||||
|
assert issue["category"] == "source_schema"
|
||||||
|
assert len(db.rows[DatasetQuarantine]) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_partitioned_vector_ingest_is_idempotent_and_quarantines_noncanonical_partition_coordinates(
|
||||||
|
monkeypatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
project = Project(id=uuid4(), name="Partitioned governed ingest")
|
||||||
|
area = Area(id=uuid4(), project_id=project.id, name="Partitioned AOI")
|
||||||
|
db = _Session(project)
|
||||||
|
db.rows[Area] = [area]
|
||||||
|
|
||||||
|
feature = {
|
||||||
|
"type": "Feature",
|
||||||
|
"id": "GBG.1",
|
||||||
|
"properties": {"id": "GBG.1", "source_feature_id": "GBG.1"},
|
||||||
|
"geometry": {
|
||||||
|
"type": "Polygon",
|
||||||
|
"coordinates": [
|
||||||
|
[[4.69, 51.09], [4.70, 51.09], [4.70, 51.10], [4.69, 51.09]]
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
partition_payload = {"type": "FeatureCollection", "features": [feature]}
|
||||||
|
partition_path = tmp_path / "partition-01.geojson"
|
||||||
|
partition_path.write_text(json.dumps(partition_payload), encoding="utf-8")
|
||||||
|
artifact_payload = {
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"crs": "EPSG:4326",
|
||||||
|
"features": [feature],
|
||||||
|
}
|
||||||
|
artifact_path = tmp_path / "grb-partitioned.geojson"
|
||||||
|
artifact_raw = json.dumps(artifact_payload).encode("utf-8")
|
||||||
|
artifact_path.write_bytes(artifact_raw)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
|
||||||
|
lambda **_kwargs: _storage_info(tmp_path, artifact_raw),
|
||||||
|
)
|
||||||
|
|
||||||
|
result = DatasetService.import_partitioned_vector_artifact(
|
||||||
|
db,
|
||||||
|
project_id=project.id,
|
||||||
|
area_id=area.id,
|
||||||
|
artifact_path=artifact_path,
|
||||||
|
partition_paths=[partition_path],
|
||||||
|
original_filename="grb-partitioned.geojson",
|
||||||
|
source="operator_official_import",
|
||||||
|
dataset_role="reference",
|
||||||
|
source_name="grb",
|
||||||
|
reference_layer_name="buildings",
|
||||||
|
metadata_json={
|
||||||
|
"feature_count": 1,
|
||||||
|
"crs": "EPSG:4326",
|
||||||
|
"bounds_json": {
|
||||||
|
"min_x": 4.69,
|
||||||
|
"min_y": 51.09,
|
||||||
|
"max_x": 4.70,
|
||||||
|
"max_y": 51.10,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
source_metadata={"license": "Open data"},
|
||||||
|
provenance_metadata={
|
||||||
|
"artifact_sha256": sha256(artifact_raw).hexdigest(),
|
||||||
|
"partition_checksums": {
|
||||||
|
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
|
||||||
|
},
|
||||||
|
},
|
||||||
|
temporal_series_key="grb:partitioned:test",
|
||||||
|
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||||
|
source_version="2026-08-01",
|
||||||
|
)
|
||||||
|
|
||||||
|
dataset = next(item for item in db.rows[Dataset] if item.id == result.id)
|
||||||
|
assert result.status == "ready"
|
||||||
|
assert dataset.validation_status == "passed"
|
||||||
|
assert dataset.provenance_status == "complete"
|
||||||
|
assert dataset.source_name == "grb"
|
||||||
|
assert len(db.rows[SourceSnapshot]) == 1
|
||||||
|
assert len(db.rows[VectorFeature]) == 1
|
||||||
|
assert dataset.metadata_json["partitioned_geometry_audit"][
|
||||||
|
"partition_checksums_sha256"
|
||||||
|
] == {partition_path.name: sha256(partition_path.read_bytes()).hexdigest()}
|
||||||
|
assert dataset.provenance_metadata["partition_checksum_manifest_sha256"]
|
||||||
|
assert dataset.provenance_metadata["partitioned_artifact_binding_sha256"]
|
||||||
|
|
||||||
|
repeated = DatasetService.import_partitioned_vector_artifact(
|
||||||
|
db,
|
||||||
|
project_id=project.id,
|
||||||
|
area_id=area.id,
|
||||||
|
artifact_path=artifact_path,
|
||||||
|
partition_paths=[partition_path],
|
||||||
|
original_filename="grb-partitioned.geojson",
|
||||||
|
source="operator_official_import",
|
||||||
|
dataset_role="reference",
|
||||||
|
source_name="grb",
|
||||||
|
reference_layer_name="buildings",
|
||||||
|
metadata_json={
|
||||||
|
"feature_count": 1,
|
||||||
|
"crs": "EPSG:4326",
|
||||||
|
"bounds_json": {
|
||||||
|
"min_x": 4.69,
|
||||||
|
"min_y": 51.09,
|
||||||
|
"max_x": 4.70,
|
||||||
|
"max_y": 51.10,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
source_metadata={"license": "Open data"},
|
||||||
|
provenance_metadata={
|
||||||
|
"artifact_sha256": sha256(artifact_raw).hexdigest(),
|
||||||
|
"partition_checksums": {
|
||||||
|
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
|
||||||
|
},
|
||||||
|
},
|
||||||
|
temporal_series_key="grb:partitioned:test",
|
||||||
|
observed_at=datetime(2026, 8, 1, tzinfo=UTC),
|
||||||
|
source_version="2026-08-01",
|
||||||
|
)
|
||||||
|
assert repeated.id == result.id
|
||||||
|
assert len(db.rows[Dataset]) == 1
|
||||||
|
assert len(db.rows[VectorFeature]) == 1
|
||||||
|
|
||||||
|
lambert_feature = {
|
||||||
|
**feature,
|
||||||
|
"id": "GBG.lambert",
|
||||||
|
"properties": {"id": "GBG.lambert"},
|
||||||
|
"geometry": {
|
||||||
|
"type": "Polygon",
|
||||||
|
"coordinates": [
|
||||||
|
[[150000, 170000], [150010, 170000], [150010, 170010], [150000, 170000]]
|
||||||
|
],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
lambert_partition = tmp_path / "partition-lambert.geojson"
|
||||||
|
lambert_partition.write_text(
|
||||||
|
json.dumps({"type": "FeatureCollection", "features": [lambert_feature]}),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
lambert_artifact = tmp_path / "grb-lambert.geojson"
|
||||||
|
lambert_raw = json.dumps(
|
||||||
|
{"type": "FeatureCollection", "features": [lambert_feature]}
|
||||||
|
).encode("utf-8")
|
||||||
|
lambert_artifact.write_bytes(lambert_raw)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
|
||||||
|
lambda **_kwargs: _storage_info(tmp_path, lambert_raw),
|
||||||
|
)
|
||||||
|
|
||||||
|
quarantined = DatasetService.import_partitioned_vector_artifact(
|
||||||
|
db,
|
||||||
|
project_id=project.id,
|
||||||
|
area_id=area.id,
|
||||||
|
artifact_path=lambert_artifact,
|
||||||
|
partition_paths=[lambert_partition],
|
||||||
|
original_filename="grb-lambert.geojson",
|
||||||
|
source="operator_official_import",
|
||||||
|
dataset_role="reference",
|
||||||
|
source_name="grb",
|
||||||
|
reference_layer_name="buildings",
|
||||||
|
metadata_json={
|
||||||
|
"feature_count": 1,
|
||||||
|
"crs": "EPSG:4326",
|
||||||
|
"bounds_json": {
|
||||||
|
"min_x": 150000,
|
||||||
|
"min_y": 170000,
|
||||||
|
"max_x": 150010,
|
||||||
|
"max_y": 170010,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
source_metadata={"license": "Open data"},
|
||||||
|
provenance_metadata={
|
||||||
|
"artifact_sha256": sha256(lambert_raw).hexdigest(),
|
||||||
|
"partition_checksums": {
|
||||||
|
lambert_partition.name: sha256(
|
||||||
|
lambert_partition.read_bytes()
|
||||||
|
).hexdigest()
|
||||||
|
},
|
||||||
|
},
|
||||||
|
temporal_series_key="grb:partitioned:lambert",
|
||||||
|
observed_at=datetime(2026, 8, 2, tzinfo=UTC),
|
||||||
|
source_version="2026-08-02",
|
||||||
|
)
|
||||||
|
assert quarantined.status == "quarantined"
|
||||||
|
assert quarantined.validation_status == "failed"
|
||||||
|
assert quarantined.quarantine_status == "quarantined"
|
||||||
|
|
||||||
|
missing_manifest_partition = tmp_path / "partition-missing-manifest.geojson"
|
||||||
|
missing_manifest_partition.write_text(
|
||||||
|
json.dumps(partition_payload), encoding="utf-8"
|
||||||
|
)
|
||||||
|
missing_manifest_artifact = tmp_path / "grb-missing-manifest.geojson"
|
||||||
|
missing_manifest_raw = json.dumps(
|
||||||
|
{"type": "FeatureCollection", "features": [feature]}
|
||||||
|
).encode("utf-8")
|
||||||
|
missing_manifest_artifact.write_bytes(missing_manifest_raw)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.services.dataset_service.StorageService.persist_dataset_file_from_path",
|
||||||
|
lambda **_kwargs: _storage_info(tmp_path, missing_manifest_raw),
|
||||||
|
)
|
||||||
|
|
||||||
|
missing_manifest = DatasetService.import_partitioned_vector_artifact(
|
||||||
|
db,
|
||||||
|
project_id=project.id,
|
||||||
|
area_id=area.id,
|
||||||
|
artifact_path=missing_manifest_artifact,
|
||||||
|
partition_paths=[missing_manifest_partition],
|
||||||
|
original_filename="grb-missing-manifest.geojson",
|
||||||
|
source="operator_official_import",
|
||||||
|
dataset_role="reference",
|
||||||
|
source_name="grb",
|
||||||
|
reference_layer_name="buildings",
|
||||||
|
metadata_json={
|
||||||
|
"feature_count": 1,
|
||||||
|
"crs": "EPSG:4326",
|
||||||
|
"bounds_json": {
|
||||||
|
"min_x": 4.69,
|
||||||
|
"min_y": 51.09,
|
||||||
|
"max_x": 4.70,
|
||||||
|
"max_y": 51.10,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
source_metadata={"license": "Open data"},
|
||||||
|
provenance_metadata={
|
||||||
|
"artifact_sha256": sha256(missing_manifest_raw).hexdigest()
|
||||||
|
},
|
||||||
|
temporal_series_key="grb:partitioned:missing-manifest",
|
||||||
|
observed_at=datetime(2026, 8, 3, tzinfo=UTC),
|
||||||
|
source_version="2026-08-03",
|
||||||
|
)
|
||||||
|
assert missing_manifest.status == "quarantined"
|
||||||
|
assert (
|
||||||
|
missing_manifest.validation_report_json["issues"][0]["code"]
|
||||||
|
== "PARTITION_CHECKSUM_MANIFEST_REQUIRED"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_partitioned_geometry_audit_handles_more_than_generic_topology_limit_without_materializing_geometries(
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
feature_count = 10_001
|
||||||
|
partition_path = tmp_path / "large-partition.geojson"
|
||||||
|
partition_path.write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"id": f"GBG.{index}",
|
||||||
|
"properties": {"id": f"GBG.{index}"},
|
||||||
|
"geometry": {
|
||||||
|
"type": "Point",
|
||||||
|
"coordinates": [4.0 + index / 10_000_000, 51.0],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for index in range(feature_count)
|
||||||
|
],
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
audit = _PartitionedGeoJsonRecords(
|
||||||
|
[partition_path],
|
||||||
|
expected_feature_count=feature_count,
|
||||||
|
declared_partition_checksums={
|
||||||
|
partition_path.name: sha256(partition_path.read_bytes()).hexdigest()
|
||||||
|
},
|
||||||
|
).audit()
|
||||||
|
|
||||||
|
assert audit.feature_count == feature_count
|
||||||
|
assert audit.bounds_json["min_x"] == 4.0
|
||||||
|
assert audit.bounds_json["max_x"] > audit.bounds_json["min_x"]
|
||||||
|
assert audit.representative_record.geometry.geom_type == "MultiPoint"
|
||||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
|||||||
import json
|
import json
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
|
from hashlib import sha256
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
@@ -13,35 +14,114 @@ SCRIPT = Path(__file__).parents[2] / "scripts" / "build_grayscale_yolo_dataset.p
|
|||||||
|
|
||||||
def test_grayscale_builder_preserves_labels_and_split(tmp_path: Path) -> None:
|
def test_grayscale_builder_preserves_labels_and_split(tmp_path: Path) -> None:
|
||||||
source = tmp_path / "source"
|
source = tmp_path / "source"
|
||||||
(source / "images" / "train").mkdir(parents=True)
|
entries = []
|
||||||
(source / "labels" / "train").mkdir(parents=True)
|
for split, sample_slug, colour in (("train", "fixture-train", (255, 0, 0)), ("val", "fixture-val", (0, 255, 0))):
|
||||||
image = source / "images" / "train" / "tile.png"
|
image = source / "images" / split / f"{sample_slug}.png"
|
||||||
label = source / "labels" / "train" / "tile.txt"
|
label = source / "labels" / split / f"{sample_slug}.txt"
|
||||||
Image.new("RGB", (8, 8), (255, 0, 0)).save(image)
|
image.parent.mkdir(parents=True, exist_ok=True)
|
||||||
label.write_text("0 0.5 0.5 0.5 0.5\n", encoding="utf-8")
|
label.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
Image.new("RGB", (8, 8), colour).save(image)
|
||||||
|
label.write_text("0 0.5 0.5 0.5 0.5\n", encoding="utf-8")
|
||||||
|
entries.append((split, sample_slug, image, label))
|
||||||
|
manifest = source / "operator_samples_manifest.json"
|
||||||
|
policy = "geointel-training-source-eligibility/v1"
|
||||||
|
manifest.write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"immutable": True,
|
||||||
|
"training_eligibility": {"policy_version": policy, "status": "eligible", "fixture_mode": True},
|
||||||
|
"samples": [
|
||||||
|
{
|
||||||
|
"sample_slug": sample_slug,
|
||||||
|
"split": split,
|
||||||
|
"raster_dataset_id": f"raster:{sample_slug}",
|
||||||
|
"reference_dataset_id": f"reference:{sample_slug}",
|
||||||
|
"training_eligibility": {
|
||||||
|
"policy_version": policy,
|
||||||
|
"eligible": True,
|
||||||
|
"fixture_mode": True,
|
||||||
|
"raster": {"eligible": True, "reasons": [], "evidence": {"dataset_id": f"raster:{sample_slug}", "checksum_sha256": "a" * 64, "source_registry_id": "fixture-raster", "source_snapshot_id": "fixture-raster-snapshot"}},
|
||||||
|
"reference": {"eligible": True, "reasons": [], "evidence": {"dataset_id": f"reference:{sample_slug}", "checksum_sha256": "b" * 64, "source_registry_id": "fixture-reference", "source_snapshot_id": "fixture-reference-snapshot"}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for split, sample_slug, _image, _label in entries
|
||||||
|
],
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
(source / "corpus-freeze.json").write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"schema_version": 2,
|
||||||
|
"immutable": True,
|
||||||
|
"fixture_mode": True,
|
||||||
|
"training_eligibility_policy": policy,
|
||||||
|
"manifest_sha256": sha256(manifest.read_bytes()).hexdigest(),
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
dataset_yaml = source / "dataset.yaml"
|
||||||
|
dataset_yaml.write_text(
|
||||||
|
f"path: {source}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
release_script = SCRIPT.parent / "training_release_manifest.py"
|
||||||
|
subprocess.run(
|
||||||
|
[
|
||||||
|
sys.executable,
|
||||||
|
str(release_script),
|
||||||
|
"create",
|
||||||
|
"--train-yaml",
|
||||||
|
str(dataset_yaml),
|
||||||
|
"--corpus-manifest",
|
||||||
|
str(manifest),
|
||||||
|
"--fixture-mode",
|
||||||
|
],
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
release_path = dataset_yaml.with_name(dataset_yaml.name + ".geointel-training-release.json")
|
||||||
|
asset_path = dataset_yaml.with_name(dataset_yaml.name + ".geointel-training-assets.json")
|
||||||
|
assets = json.loads(asset_path.read_text(encoding="utf-8"))
|
||||||
summary = source / "yolo_tile_dataset_summary.json"
|
summary = source / "yolo_tile_dataset_summary.json"
|
||||||
summary.write_text(
|
summary.write_text(
|
||||||
json.dumps(
|
json.dumps(
|
||||||
{
|
{
|
||||||
|
"dataset_yaml": str(dataset_yaml.resolve()),
|
||||||
|
"training_release_manifest": str(release_path.resolve()),
|
||||||
|
"training_release_manifest_sha256": sha256(release_path.read_bytes()).hexdigest(),
|
||||||
|
"training_asset_manifest": str(asset_path.resolve()),
|
||||||
|
"source_manifest_sha256": sha256(manifest.read_bytes()).hexdigest(),
|
||||||
"tiles": [
|
"tiles": [
|
||||||
{
|
{"split": entry["split"], "image_path": entry["image_path"], "label_path": entry["label_path"]}
|
||||||
"split": "train",
|
for entry in assets["entries"]
|
||||||
"image_path": str(image),
|
],
|
||||||
"label_path": str(label),
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
),
|
),
|
||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
)
|
)
|
||||||
output = tmp_path / "gray"
|
output = tmp_path / "gray"
|
||||||
subprocess.run(
|
subprocess.run(
|
||||||
[sys.executable, str(SCRIPT), "--summary", str(summary), "--output-dir", str(output)],
|
[
|
||||||
|
sys.executable,
|
||||||
|
str(SCRIPT),
|
||||||
|
"--summary",
|
||||||
|
str(summary),
|
||||||
|
"--train-yaml",
|
||||||
|
str(dataset_yaml),
|
||||||
|
"--corpus-manifest",
|
||||||
|
str(manifest),
|
||||||
|
"--fixture-mode",
|
||||||
|
"--output-dir",
|
||||||
|
str(output),
|
||||||
|
],
|
||||||
check=True,
|
check=True,
|
||||||
)
|
)
|
||||||
converted = Image.open(output / "images" / "train" / "tile.png")
|
converted = Image.open(output / "images" / "train" / "fixture-train.png")
|
||||||
r, g, b = converted.getpixel((0, 0))
|
r, g, b = converted.getpixel((0, 0))
|
||||||
assert r == g == b
|
assert r == g == b
|
||||||
assert (output / "labels" / "train" / "tile.txt").read_text() == label.read_text()
|
assert (output / "labels" / "train" / "fixture-train.txt").read_text() == "0 0.5 0.5 0.5 0.5\n"
|
||||||
evidence = json.loads((output / "grayscale-dataset-evidence.json").read_text())
|
evidence = json.loads((output / "grayscale-dataset-evidence.json").read_text())
|
||||||
assert evidence["converted_tile_count"] == 1
|
assert evidence["converted_tile_count"] == 2
|
||||||
|
assert evidence["training_eligible"] is False
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from hashlib import sha256
|
||||||
import json
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
@@ -10,9 +11,10 @@ from fastapi.testclient import TestClient
|
|||||||
from app.core.config import Settings
|
from app.core.config import Settings
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.main import app
|
from app.main import app
|
||||||
from app.models import AnalysisRun, Dataset, Detection, Job, Project
|
from app.models import AnalysisRun, Dataset, Detection, Job, Project, SourceRegistry, SourceSnapshot
|
||||||
from app.services.detection_service import DetectionService
|
from app.services.detection_service import DetectionService
|
||||||
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
from app.services.model_asset_catalog_service import ModelAssetCatalogService
|
||||||
|
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||||
|
|
||||||
|
|
||||||
class FakeSession:
|
class FakeSession:
|
||||||
@@ -63,15 +65,47 @@ class MockYoloAdapter:
|
|||||||
def _project_and_raster_dataset():
|
def _project_and_raster_dataset():
|
||||||
project_id = uuid4()
|
project_id = uuid4()
|
||||||
dataset_id = uuid4()
|
dataset_id = uuid4()
|
||||||
|
source_registry_id = uuid4()
|
||||||
|
source_snapshot_id = uuid4()
|
||||||
|
checksum = "a" * 64
|
||||||
project = Project(id=project_id, name="Geel")
|
project = Project(id=project_id, name="Geel")
|
||||||
|
source_registry = SourceRegistry(
|
||||||
|
id=source_registry_id,
|
||||||
|
source_key="test-derived-raster",
|
||||||
|
display_name="Governed test-derived raster",
|
||||||
|
classification="derived",
|
||||||
|
authority_name="GeoIntel test fixture",
|
||||||
|
usage_policy_json={"ground_truth_allowed": False},
|
||||||
|
)
|
||||||
|
source_snapshot = SourceSnapshot(
|
||||||
|
id=source_snapshot_id,
|
||||||
|
source_registry_id=source_registry_id,
|
||||||
|
snapshot_key="test-derived-raster-v1",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
freshness_status="current",
|
||||||
|
ingest_status="ingested",
|
||||||
|
)
|
||||||
dataset = Dataset(
|
dataset = Dataset(
|
||||||
id=dataset_id,
|
id=dataset_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name="source.tif",
|
name="source.tif",
|
||||||
dataset_type="raster",
|
dataset_type="raster",
|
||||||
source="user_upload",
|
source="test-derived-raster",
|
||||||
|
source_name="test-derived-raster",
|
||||||
storage_path="storage/uploads/source.tif",
|
storage_path="storage/uploads/source.tif",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
source_registry_id=source_registry_id,
|
||||||
|
source_snapshot_id=source_snapshot_id,
|
||||||
|
data_contract_key="geointel.raster.geotiff",
|
||||||
|
data_contract_version="1.0.0",
|
||||||
|
validation_status="passed",
|
||||||
|
provenance_status="complete",
|
||||||
|
lineage_status="not_applicable",
|
||||||
|
quarantine_status="not_quarantined",
|
||||||
|
status="ready",
|
||||||
)
|
)
|
||||||
|
dataset.source_registry = source_registry
|
||||||
|
dataset.source_snapshot = source_snapshot
|
||||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||||
return db, project_id, dataset_id
|
return db, project_id, dataset_id
|
||||||
|
|
||||||
@@ -101,6 +135,72 @@ def _manifest(tmp_path: Path) -> Path:
|
|||||||
return manifest_path
|
return manifest_path
|
||||||
|
|
||||||
|
|
||||||
|
def _write_model_sidecar(
|
||||||
|
model_path: Path,
|
||||||
|
settings: Settings,
|
||||||
|
*,
|
||||||
|
db: FakeSession | None = None,
|
||||||
|
) -> None:
|
||||||
|
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
|
||||||
|
source_registry_id = uuid4()
|
||||||
|
source_snapshot_id = uuid4()
|
||||||
|
source_version = settings.yolo_model_version or "test-v1"
|
||||||
|
if db is not None:
|
||||||
|
source_registry = SourceRegistry(
|
||||||
|
id=source_registry_id,
|
||||||
|
source_key="model",
|
||||||
|
display_name="Governed test model artifact",
|
||||||
|
classification="experimental",
|
||||||
|
authority_name="GeoIntel test fixture",
|
||||||
|
freshness_status="current",
|
||||||
|
ingest_status="configured",
|
||||||
|
)
|
||||||
|
source_snapshot = SourceSnapshot(
|
||||||
|
id=source_snapshot_id,
|
||||||
|
source_registry_id=source_registry_id,
|
||||||
|
snapshot_key=f"model-{source_version}",
|
||||||
|
source_version=source_version,
|
||||||
|
checksum_sha256=model_sha256,
|
||||||
|
freshness_status="current",
|
||||||
|
ingest_status="ingested",
|
||||||
|
)
|
||||||
|
db.objects[(SourceRegistry, source_registry_id)] = source_registry
|
||||||
|
db.objects[(SourceSnapshot, source_snapshot_id)] = source_snapshot
|
||||||
|
payload = {
|
||||||
|
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
|
||||||
|
"data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"},
|
||||||
|
"model": {
|
||||||
|
"model_id": settings.yolo_model_id,
|
||||||
|
"task_type": "object_detection",
|
||||||
|
"sha256": model_sha256,
|
||||||
|
"model_format": "pytorch",
|
||||||
|
"framework": "ultralytics/pytorch",
|
||||||
|
"class_mapping": {"0": "building"},
|
||||||
|
"source_version": source_version,
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"source_registry_id": str(source_registry_id),
|
||||||
|
"source_snapshot_id": str(source_snapshot_id),
|
||||||
|
"source_registry_key": "model",
|
||||||
|
"source_snapshot_checksum_sha256": model_sha256,
|
||||||
|
},
|
||||||
|
"lineage": {
|
||||||
|
"upstream_asset_ids": ["test-training-corpus"],
|
||||||
|
"upstream_checksums_sha256": ["a" * 64],
|
||||||
|
"transformations": [
|
||||||
|
{"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"metadata": {"training_manifest_sha256": "c" * 64},
|
||||||
|
"imported_at": "2026-08-01T10:00:00+00:00",
|
||||||
|
}
|
||||||
|
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
|
||||||
|
RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text(
|
||||||
|
json.dumps(payload, sort_keys=True),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_model_asset_catalog_lists_supported_local_model_files(tmp_path: Path) -> None:
|
def test_model_asset_catalog_lists_supported_local_model_files(tmp_path: Path) -> None:
|
||||||
model_file = tmp_path / "building-detector.pt"
|
model_file = tmp_path / "building-detector.pt"
|
||||||
model_file.write_bytes(b"local model")
|
model_file.write_bytes(b"local model")
|
||||||
@@ -190,6 +290,7 @@ def test_detection_run_persists_selected_model_asset_parameters(tmp_path: Path)
|
|||||||
yolo_models_dir=str(tmp_path),
|
yolo_models_dir=str(tmp_path),
|
||||||
yolo_max_tiles=4,
|
yolo_max_tiles=4,
|
||||||
)
|
)
|
||||||
|
_write_model_sidecar(model_file, settings, db=db)
|
||||||
|
|
||||||
result = DetectionService.run_detection(
|
result = DetectionService.run_detection(
|
||||||
db=db,
|
db=db,
|
||||||
|
|||||||
@@ -0,0 +1,644 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
from hashlib import sha256
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
from app.models import Dataset, DatasetVersion, SourceRegistry, SourceSnapshot
|
||||||
|
from app.services.data_contract_validation import (
|
||||||
|
build_vector_ingest_input,
|
||||||
|
validate_registered_asset,
|
||||||
|
)
|
||||||
|
from app.services.demo_workflow_service import DemoWorkflowService
|
||||||
|
from app.services.derived_dataset_governance_service import (
|
||||||
|
DerivedDatasetGovernanceService,
|
||||||
|
)
|
||||||
|
from app.services.raster_operations_service import RasterOperationsService
|
||||||
|
from app.services.storage_service import StorageService
|
||||||
|
from app.services.vector_feature_service import VectorFeatureService
|
||||||
|
from app.services.vector_operations_service import VectorOperationsService
|
||||||
|
|
||||||
|
|
||||||
|
_CHECKSUM = "a" * 64
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeSession:
|
||||||
|
def __init__(self, rows=None) -> None:
|
||||||
|
self.rows = rows or {}
|
||||||
|
self.added = []
|
||||||
|
|
||||||
|
def get(self, model, row_id):
|
||||||
|
row = self.rows.get((model, row_id))
|
||||||
|
if row is not None:
|
||||||
|
return row
|
||||||
|
return next(
|
||||||
|
(
|
||||||
|
item
|
||||||
|
for item in self.added
|
||||||
|
if isinstance(item, model) and item.id == row_id
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
|
||||||
|
def add(self, row) -> None:
|
||||||
|
self.added.append(row)
|
||||||
|
|
||||||
|
def commit(self) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def refresh(self, row) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class _GovernedSession:
|
||||||
|
"""Small ORM-shaped session for the real governance branch.
|
||||||
|
|
||||||
|
Registry persistence is monkeypatched below; the test exercises the
|
||||||
|
service's orchestration and report decisions without needing PostGIS.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.added = []
|
||||||
|
self.flushes = 0
|
||||||
|
|
||||||
|
class _EmptyQuery:
|
||||||
|
def filter(self, *_args, **_kwargs):
|
||||||
|
return self
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def all():
|
||||||
|
return []
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def one_or_none():
|
||||||
|
return None
|
||||||
|
|
||||||
|
def query(self, *_args, **_kwargs):
|
||||||
|
# Governance now performs a bounded lineage traversal during quarantine.
|
||||||
|
# This focused harness intentionally has no persisted siblings/edges.
|
||||||
|
return self._EmptyQuery()
|
||||||
|
|
||||||
|
def add(self, row) -> None:
|
||||||
|
self.added.append(row)
|
||||||
|
|
||||||
|
def flush(self) -> None:
|
||||||
|
self.flushes += 1
|
||||||
|
|
||||||
|
|
||||||
|
def _governed_parent() -> Dataset:
|
||||||
|
source_id = uuid4()
|
||||||
|
snapshot_id = uuid4()
|
||||||
|
source = SourceRegistry(
|
||||||
|
id=source_id,
|
||||||
|
source_key="grb",
|
||||||
|
display_name="GRB parent fixture",
|
||||||
|
classification="authoritative",
|
||||||
|
authority_name="Digitaal Vlaanderen",
|
||||||
|
authority_scope_json={"zone": "Flanders"},
|
||||||
|
usage_policy_json={"ground_truth_allowed": True},
|
||||||
|
)
|
||||||
|
snapshot = SourceSnapshot(
|
||||||
|
id=snapshot_id,
|
||||||
|
source_registry_id=source_id,
|
||||||
|
snapshot_key="governed-parent",
|
||||||
|
checksum_sha256=_CHECKSUM,
|
||||||
|
freshness_status="current",
|
||||||
|
ingest_status="ingested",
|
||||||
|
)
|
||||||
|
dataset = Dataset(
|
||||||
|
id=uuid4(),
|
||||||
|
project_id=uuid4(),
|
||||||
|
name="governed.geojson",
|
||||||
|
dataset_type="vector",
|
||||||
|
source="grb",
|
||||||
|
source_name="grb",
|
||||||
|
status="ready",
|
||||||
|
checksum_sha256=_CHECKSUM,
|
||||||
|
data_contract_key="geointel.vector.geojson",
|
||||||
|
data_contract_version="1.0.0",
|
||||||
|
validation_status="passed",
|
||||||
|
provenance_status="complete",
|
||||||
|
lineage_status="not_applicable",
|
||||||
|
quarantine_status="not_quarantined",
|
||||||
|
source_registry_id=source_id,
|
||||||
|
source_snapshot_id=snapshot_id,
|
||||||
|
)
|
||||||
|
dataset.source_registry = source
|
||||||
|
dataset.source_snapshot = snapshot
|
||||||
|
return dataset
|
||||||
|
|
||||||
|
|
||||||
|
def test_lineage_evidence_quarantines_ungoverned_parent_without_inventing_a_checksum() -> (
|
||||||
|
None
|
||||||
|
):
|
||||||
|
parent = _governed_parent()
|
||||||
|
valid_lineage = DerivedDatasetGovernanceService._lineage_evidence(
|
||||||
|
parent, "vector.clip", {"area_id": "a"}
|
||||||
|
)
|
||||||
|
valid_report = validate_registered_asset(
|
||||||
|
build_vector_ingest_input(
|
||||||
|
asset_id="derived-valid",
|
||||||
|
source_crs="EPSG:4326",
|
||||||
|
storage_crs="EPSG:4326",
|
||||||
|
feature_collection={
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||||
|
"properties": {},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
checksum_sha256=_CHECKSUM,
|
||||||
|
computed_checksum_sha256=_CHECKSUM,
|
||||||
|
source_registry_id="derived-source",
|
||||||
|
source_snapshot_id="derived-snapshot",
|
||||||
|
imported_at=datetime.now(timezone.utc),
|
||||||
|
metadata={
|
||||||
|
"license": "internal derived artifact",
|
||||||
|
"bounds_json": {
|
||||||
|
"min_x": 5.0,
|
||||||
|
"min_y": 51.0,
|
||||||
|
"max_x": 5.0,
|
||||||
|
"max_y": 51.0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
temporal_unknown_reason="derived input has no precise observation timestamp",
|
||||||
|
source_version_unknown_reason="transform version is recorded separately",
|
||||||
|
lineage=valid_lineage,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert valid_report.validation_status.value == "passed"
|
||||||
|
assert valid_lineage.upstream_asset_ids == (str(parent.id),)
|
||||||
|
assert valid_lineage.upstream_checksums_sha256 == (_CHECKSUM,)
|
||||||
|
|
||||||
|
parent.validation_status = "not_validated"
|
||||||
|
rejected_lineage = DerivedDatasetGovernanceService._lineage_evidence(
|
||||||
|
parent, "vector.clip", {}
|
||||||
|
)
|
||||||
|
rejected_report = validate_registered_asset(
|
||||||
|
build_vector_ingest_input(
|
||||||
|
asset_id="derived-rejected",
|
||||||
|
source_crs="EPSG:4326",
|
||||||
|
storage_crs="EPSG:4326",
|
||||||
|
feature_collection={
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||||
|
"properties": {},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
checksum_sha256=_CHECKSUM,
|
||||||
|
computed_checksum_sha256=_CHECKSUM,
|
||||||
|
source_registry_id="derived-source",
|
||||||
|
source_snapshot_id="derived-snapshot",
|
||||||
|
imported_at=datetime.now(timezone.utc),
|
||||||
|
metadata={
|
||||||
|
"license": "internal derived artifact",
|
||||||
|
"bounds_json": {
|
||||||
|
"min_x": 5.0,
|
||||||
|
"min_y": 51.0,
|
||||||
|
"max_x": 5.0,
|
||||||
|
"max_y": 51.0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
temporal_unknown_reason="derived input has no precise observation timestamp",
|
||||||
|
source_version_unknown_reason="transform version is recorded separately",
|
||||||
|
lineage=rejected_lineage,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert rejected_lineage.upstream_checksums_sha256 == (
|
||||||
|
"parent_dataset_not_governed",
|
||||||
|
)
|
||||||
|
assert rejected_report.validation_status.value == "failed"
|
||||||
|
assert rejected_report.quarantine_status.value == "quarantined"
|
||||||
|
assert any(
|
||||||
|
issue.code == "UPSTREAM_CHECKSUM_FORMAT_INVALID"
|
||||||
|
for issue in rejected_report.issues
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_govern_vector_binds_snapshot_contract_and_lineage_before_marking_ready(
|
||||||
|
monkeypatch,
|
||||||
|
) -> None:
|
||||||
|
from app.services.source_registry_service import SourceRegistryService
|
||||||
|
|
||||||
|
db = _GovernedSession()
|
||||||
|
parent = _governed_parent()
|
||||||
|
parent_version_id = uuid4()
|
||||||
|
dataset = Dataset(
|
||||||
|
id=uuid4(),
|
||||||
|
project_id=parent.project_id,
|
||||||
|
name="derived.geojson",
|
||||||
|
dataset_type="vector",
|
||||||
|
source="operation:clip",
|
||||||
|
source_name="derived",
|
||||||
|
checksum_sha256=_CHECKSUM,
|
||||||
|
imported_at=datetime.now(timezone.utc),
|
||||||
|
crs="EPSG:4326",
|
||||||
|
metadata_json={
|
||||||
|
"bounds_json": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.0, "max_y": 51.0}
|
||||||
|
},
|
||||||
|
status="validating",
|
||||||
|
)
|
||||||
|
version = DatasetVersion(
|
||||||
|
id=uuid4(), dataset_id=dataset.id, version=1, checksum_sha256=_CHECKSUM
|
||||||
|
)
|
||||||
|
source = SimpleNamespace(id=uuid4(), license_name="internal derived artifact")
|
||||||
|
snapshot = SimpleNamespace(id=uuid4(), source_registry_id=source.id)
|
||||||
|
edges = []
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
SourceRegistryService,
|
||||||
|
"ensure_server_owned_source",
|
||||||
|
lambda *_args, **_kwargs: source,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
SourceRegistryService, "record_snapshot", lambda *_args, **_kwargs: snapshot
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
DerivedDatasetGovernanceService,
|
||||||
|
"_latest_parent_version_id",
|
||||||
|
lambda *_args: parent_version_id,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _bind(target, **kwargs):
|
||||||
|
target.source_registry_id = kwargs["source"].id
|
||||||
|
target.source_snapshot_id = kwargs["snapshot"].id
|
||||||
|
target.data_contract_key = kwargs["data_contract_key"]
|
||||||
|
target.data_contract_version = kwargs["data_contract_version"]
|
||||||
|
target.validation_status = kwargs["validation_status"]
|
||||||
|
target.provenance_status = kwargs["provenance_status"]
|
||||||
|
target.lineage_status = kwargs["lineage_status"]
|
||||||
|
return target
|
||||||
|
|
||||||
|
monkeypatch.setattr(SourceRegistryService, "bind_dataset_provenance", _bind)
|
||||||
|
monkeypatch.setattr(SourceRegistryService, "bind_dataset_version_provenance", _bind)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
SourceRegistryService,
|
||||||
|
"record_lineage_edge",
|
||||||
|
lambda *_args, **kwargs: edges.append(kwargs),
|
||||||
|
)
|
||||||
|
|
||||||
|
ready = DerivedDatasetGovernanceService.govern_vector(
|
||||||
|
db,
|
||||||
|
dataset=dataset,
|
||||||
|
dataset_version=version,
|
||||||
|
feature_collection={
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||||
|
"properties": {},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
source_key="derived",
|
||||||
|
operation="vector.clip",
|
||||||
|
parent_dataset=parent,
|
||||||
|
operation_parameters={"area_id": "a"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert ready is True
|
||||||
|
assert dataset.status == "ready"
|
||||||
|
assert dataset.validation_status == "passed"
|
||||||
|
assert dataset.provenance_status == "complete"
|
||||||
|
assert dataset.source_registry_id == source.id
|
||||||
|
assert version.source_snapshot_id == snapshot.id
|
||||||
|
assert db.flushes >= 1
|
||||||
|
assert edges[0]["parent_dataset_id"] == parent.id
|
||||||
|
assert edges[0]["parent_dataset_version_id"] == parent_version_id
|
||||||
|
assert edges[0]["child_dataset_version_id"] == version.id
|
||||||
|
|
||||||
|
|
||||||
|
def test_govern_vector_quarantines_output_when_parent_is_manual_or_experimental(
|
||||||
|
monkeypatch,
|
||||||
|
) -> None:
|
||||||
|
from app.services.source_registry_service import SourceRegistryService
|
||||||
|
|
||||||
|
db = _GovernedSession()
|
||||||
|
parent = _governed_parent()
|
||||||
|
parent.source = "manual"
|
||||||
|
parent.source_name = "manual"
|
||||||
|
parent.source_registry.source_key = "manual"
|
||||||
|
parent.source_registry.classification = "experimental"
|
||||||
|
parent.source_registry.usage_policy_json = {"ground_truth_allowed": False}
|
||||||
|
dataset = Dataset(
|
||||||
|
id=uuid4(),
|
||||||
|
project_id=parent.project_id,
|
||||||
|
name="manual-derived.geojson",
|
||||||
|
dataset_type="vector",
|
||||||
|
source="operation:clip",
|
||||||
|
source_name="derived",
|
||||||
|
checksum_sha256=_CHECKSUM,
|
||||||
|
imported_at=datetime.now(timezone.utc),
|
||||||
|
crs="EPSG:4326",
|
||||||
|
metadata_json={
|
||||||
|
"bounds_json": {"min_x": 5.0, "min_y": 51.0, "max_x": 5.0, "max_y": 51.0}
|
||||||
|
},
|
||||||
|
status="validating",
|
||||||
|
)
|
||||||
|
version = DatasetVersion(
|
||||||
|
id=uuid4(), dataset_id=dataset.id, version=1, checksum_sha256=_CHECKSUM
|
||||||
|
)
|
||||||
|
source = SimpleNamespace(id=uuid4(), license_name="internal derived artifact")
|
||||||
|
snapshot = SimpleNamespace(
|
||||||
|
id=uuid4(), source_registry_id=source.id, ingest_status="ingested"
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
SourceRegistryService,
|
||||||
|
"ensure_server_owned_source",
|
||||||
|
lambda *_args, **_kwargs: source,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
SourceRegistryService, "record_snapshot", lambda *_args, **_kwargs: snapshot
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
DerivedDatasetGovernanceService,
|
||||||
|
"_latest_parent_version_id",
|
||||||
|
lambda *_args: None,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
SourceRegistryService, "record_lineage_edge", lambda *_args, **_kwargs: None
|
||||||
|
)
|
||||||
|
|
||||||
|
ready = DerivedDatasetGovernanceService.govern_vector(
|
||||||
|
db,
|
||||||
|
dataset=dataset,
|
||||||
|
dataset_version=version,
|
||||||
|
feature_collection={
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||||
|
"properties": {},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
source_key="derived",
|
||||||
|
operation="vector.clip",
|
||||||
|
parent_dataset=parent,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert ready is False
|
||||||
|
assert dataset.status == "quarantined"
|
||||||
|
assert dataset.quarantine_status == "quarantined"
|
||||||
|
assert dataset.validation_status == "failed"
|
||||||
|
assert any(
|
||||||
|
issue["code"] == "PARENT_DATASET_NOT_ELIGIBLE_FOR_DERIVED_PROCESSING"
|
||||||
|
for issue in dataset.validation_report_json["issues"]
|
||||||
|
)
|
||||||
|
assert snapshot.ingest_status == "quarantined"
|
||||||
|
|
||||||
|
|
||||||
|
def test_vector_selection_uses_map_selection_registry_and_skips_features_when_quarantined(
|
||||||
|
monkeypatch, tmp_path
|
||||||
|
) -> None:
|
||||||
|
source = _governed_parent()
|
||||||
|
source.area_id = None
|
||||||
|
source.storage_path = str(tmp_path / "source.geojson")
|
||||||
|
db = _FakeSession({(Dataset, source.id): source})
|
||||||
|
output_path = tmp_path / "selection.geojson"
|
||||||
|
calls = []
|
||||||
|
persisted_features = []
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
VectorFeatureService,
|
||||||
|
"select_features_by_bbox",
|
||||||
|
lambda *_args, **_kwargs: {
|
||||||
|
"selection_bbox": {
|
||||||
|
"min_x": 4.9,
|
||||||
|
"min_y": 50.9,
|
||||||
|
"max_x": 5.2,
|
||||||
|
"max_y": 51.2,
|
||||||
|
"crs": "EPSG:4326",
|
||||||
|
},
|
||||||
|
"feature_count": 1,
|
||||||
|
"limit": 250,
|
||||||
|
"truncated": False,
|
||||||
|
"geojson": {
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"id": "source-feature",
|
||||||
|
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||||
|
"properties": {
|
||||||
|
"vector_feature_id": "source-feature",
|
||||||
|
"dataset_id": str(source.id),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def _persist_dataset_file(**kwargs):
|
||||||
|
output_path.write_bytes(kwargs["content"])
|
||||||
|
return {
|
||||||
|
"original_filename": kwargs["original_filename"],
|
||||||
|
"stored_filename": output_path.name,
|
||||||
|
"content_type": kwargs["content_type"],
|
||||||
|
"size_bytes": len(kwargs["content"]),
|
||||||
|
"checksum_sha256": _CHECKSUM,
|
||||||
|
"storage_path": str(output_path),
|
||||||
|
}
|
||||||
|
|
||||||
|
monkeypatch.setattr(StorageService, "persist_dataset_file", _persist_dataset_file)
|
||||||
|
|
||||||
|
def _quarantine(db, **kwargs):
|
||||||
|
calls.append(kwargs)
|
||||||
|
kwargs["dataset"].status = "quarantined"
|
||||||
|
return False
|
||||||
|
|
||||||
|
monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_vector", _quarantine)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
VectorFeatureService,
|
||||||
|
"persist_geojson_features",
|
||||||
|
lambda **kwargs: persisted_features.append(kwargs),
|
||||||
|
)
|
||||||
|
|
||||||
|
response = VectorOperationsService.derive_selection_dataset(
|
||||||
|
db=db,
|
||||||
|
dataset_id=source.id,
|
||||||
|
bbox={
|
||||||
|
"min_x": 4.9,
|
||||||
|
"min_y": 50.9,
|
||||||
|
"max_x": 5.2,
|
||||||
|
"max_y": 51.2,
|
||||||
|
"crs": "EPSG:4326",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status == "quarantined"
|
||||||
|
assert calls[0]["source_key"] == "map_selection"
|
||||||
|
assert calls[0]["parent_dataset"] is source
|
||||||
|
assert calls[0]["operation"] == "vector.selection"
|
||||||
|
assert persisted_features == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_vector_buffer_uses_projected_metres_instead_of_wgs84_degrees(
|
||||||
|
monkeypatch, tmp_path
|
||||||
|
) -> None:
|
||||||
|
source = _governed_parent()
|
||||||
|
source.storage_path = str(tmp_path / "source.geojson")
|
||||||
|
source.crs = "EPSG:4326"
|
||||||
|
Path(source.storage_path).write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"crs": "EPSG:4326",
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||||
|
"properties": {},
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
# A governed consumption artifact must carry the checksum of these exact
|
||||||
|
# bytes; vector operations deliberately refuse a stale fixture checksum.
|
||||||
|
source.checksum_sha256 = sha256(Path(source.storage_path).read_bytes()).hexdigest()
|
||||||
|
source.source_snapshot.checksum_sha256 = source.checksum_sha256
|
||||||
|
db = _FakeSession({(Dataset, source.id): source})
|
||||||
|
captured = {}
|
||||||
|
|
||||||
|
def _persist(**kwargs):
|
||||||
|
captured.update(kwargs)
|
||||||
|
return uuid4()
|
||||||
|
|
||||||
|
monkeypatch.setattr(VectorOperationsService, "_persist_derived_dataset", _persist)
|
||||||
|
VectorOperationsService.buffer(
|
||||||
|
db, source.id, distance_m=100.0, dissolve=False, output_name=None
|
||||||
|
)
|
||||||
|
|
||||||
|
coordinates = captured["feature_collection"]["features"][0]["geometry"][
|
||||||
|
"coordinates"
|
||||||
|
][0]
|
||||||
|
longitudes = [coordinate[0] for coordinate in coordinates]
|
||||||
|
latitudes = [coordinate[1] for coordinate in coordinates]
|
||||||
|
assert max(longitudes) - min(longitudes) < 0.01
|
||||||
|
assert max(latitudes) - min(latitudes) < 0.01
|
||||||
|
|
||||||
|
|
||||||
|
def test_raster_operation_uses_derived_registry_before_commit(
|
||||||
|
monkeypatch, tmp_path
|
||||||
|
) -> None:
|
||||||
|
source = _governed_parent()
|
||||||
|
source.dataset_type = "raster"
|
||||||
|
source.storage_path = str(tmp_path / "source.tif")
|
||||||
|
output_path = tmp_path / "derived.tif"
|
||||||
|
output_path.write_bytes(b"derived-raster")
|
||||||
|
db = _FakeSession()
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def _govern(db, **kwargs):
|
||||||
|
calls.append(kwargs)
|
||||||
|
kwargs["dataset"].status = "quarantined"
|
||||||
|
return False
|
||||||
|
|
||||||
|
monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_raster", _govern)
|
||||||
|
|
||||||
|
result = RasterOperationsService._persist_derived_dataset(
|
||||||
|
db,
|
||||||
|
source_dataset=source,
|
||||||
|
source_dataset_id=source.id,
|
||||||
|
operation="ndvi",
|
||||||
|
output_path=str(output_path),
|
||||||
|
output_name="derived.tif",
|
||||||
|
metadata={
|
||||||
|
"crs": "EPSG:31370",
|
||||||
|
"bounds": [100000.0, 100000.0, 100001.0, 100001.0],
|
||||||
|
"resolution": [1.0, 1.0],
|
||||||
|
"width": 1,
|
||||||
|
"height": 1,
|
||||||
|
"band_count": 1,
|
||||||
|
"dtype": ["float32"],
|
||||||
|
"operation_parameters": {"nir_band": 4, "red_band": 3},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
derived = next(item for item in db.added if isinstance(item, Dataset))
|
||||||
|
assert result == derived.id
|
||||||
|
assert derived.status == "quarantined"
|
||||||
|
assert derived.source_name == "derived"
|
||||||
|
assert calls[0]["source_key"] == "derived"
|
||||||
|
assert calls[0]["parent_dataset"] is source
|
||||||
|
assert calls[0]["operation"] == "raster.ndvi"
|
||||||
|
|
||||||
|
|
||||||
|
def test_demo_fixture_creation_is_governed_and_does_not_persist_features_when_rejected(
|
||||||
|
monkeypatch, tmp_path
|
||||||
|
) -> None:
|
||||||
|
project_id = uuid4()
|
||||||
|
area_id = uuid4()
|
||||||
|
db = _FakeSession()
|
||||||
|
calls = []
|
||||||
|
features = []
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
StorageService,
|
||||||
|
"persist_dataset_file",
|
||||||
|
lambda **kwargs: {
|
||||||
|
"storage_path": str(tmp_path / kwargs["original_filename"]),
|
||||||
|
"original_filename": kwargs["original_filename"],
|
||||||
|
"stored_filename": kwargs["original_filename"],
|
||||||
|
"content_type": kwargs["content_type"],
|
||||||
|
"size_bytes": len(kwargs["content"]),
|
||||||
|
"checksum_sha256": _CHECKSUM,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
def _quarantine(db, **kwargs):
|
||||||
|
calls.append(kwargs)
|
||||||
|
kwargs["dataset"].status = "quarantined"
|
||||||
|
return False
|
||||||
|
|
||||||
|
monkeypatch.setattr(DerivedDatasetGovernanceService, "govern_vector", _quarantine)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
VectorFeatureService,
|
||||||
|
"persist_geojson_features",
|
||||||
|
lambda **kwargs: features.append(kwargs),
|
||||||
|
)
|
||||||
|
payload = {
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"geometry": {"type": "Point", "coordinates": [5.0, 51.0]},
|
||||||
|
"properties": {},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
dataset = DemoWorkflowService._create_dataset(
|
||||||
|
db,
|
||||||
|
project_id=project_id,
|
||||||
|
area_id=area_id,
|
||||||
|
filename="fixture.geojson",
|
||||||
|
payload=payload,
|
||||||
|
raw=json.dumps(payload).encode("utf-8"),
|
||||||
|
role="source",
|
||||||
|
source_name="fixture",
|
||||||
|
reference_layer_name=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert dataset.status == "quarantined"
|
||||||
|
assert calls[0]["source_key"] == "fixture"
|
||||||
|
assert calls[0]["operation"] == "demo.fixture_vector"
|
||||||
|
assert features == []
|
||||||
@@ -1,11 +1,11 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
from hashlib import sha256
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from types import SimpleNamespace
|
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
from app.models import Area, Dataset
|
from app.models import Dataset, SourceRegistry, SourceSnapshot
|
||||||
from app.services.qa_service import QaService
|
from app.services.qa_service import QaService
|
||||||
|
|
||||||
|
|
||||||
@@ -46,6 +46,47 @@ def _write_features(path: Path, features: list[dict]) -> None:
|
|||||||
path.write_text(json.dumps({"type": "FeatureCollection", "features": features}), encoding="utf-8")
|
path.write_text(json.dumps({"type": "FeatureCollection", "features": features}), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _authoritative_reference(dataset: Dataset) -> Dataset:
|
||||||
|
"""Give QA reference fixtures the same durable authority proof as GRB."""
|
||||||
|
|
||||||
|
source_id = uuid4()
|
||||||
|
snapshot_id = uuid4()
|
||||||
|
checksum = sha256(Path(str(dataset.storage_path)).read_bytes()).hexdigest()
|
||||||
|
source = SourceRegistry(
|
||||||
|
id=source_id,
|
||||||
|
source_key="grb",
|
||||||
|
display_name="GRB test reference",
|
||||||
|
classification="authoritative",
|
||||||
|
authority_name="Digitaal Vlaanderen",
|
||||||
|
authority_scope_json={"zone": "Flanders"},
|
||||||
|
usage_policy_json={"ground_truth_allowed": True, "validation_authority": {"building_validation": "primary"}},
|
||||||
|
)
|
||||||
|
snapshot = SourceSnapshot(
|
||||||
|
id=snapshot_id,
|
||||||
|
source_registry_id=source_id,
|
||||||
|
snapshot_key=f"qa-grb-{dataset.id}",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
ingest_status="ingested",
|
||||||
|
freshness_status="current",
|
||||||
|
)
|
||||||
|
dataset.source = "grb"
|
||||||
|
dataset.source_name = "grb"
|
||||||
|
dataset.dataset_role = "reference"
|
||||||
|
dataset.status = "ready"
|
||||||
|
dataset.checksum_sha256 = checksum
|
||||||
|
dataset.source_registry_id = source_id
|
||||||
|
dataset.source_snapshot_id = snapshot_id
|
||||||
|
dataset.data_contract_key = "geointel.vector.geojson"
|
||||||
|
dataset.data_contract_version = "1.0.0"
|
||||||
|
dataset.validation_status = "passed"
|
||||||
|
dataset.provenance_status = "complete"
|
||||||
|
dataset.lineage_status = "complete"
|
||||||
|
dataset.quarantine_status = "not_quarantined"
|
||||||
|
dataset.source_registry = source
|
||||||
|
dataset.source_snapshot = snapshot
|
||||||
|
return dataset
|
||||||
|
|
||||||
|
|
||||||
def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None:
|
def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None:
|
||||||
project_id = uuid4()
|
project_id = uuid4()
|
||||||
candidate_id = uuid4()
|
candidate_id = uuid4()
|
||||||
@@ -66,7 +107,7 @@ def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None:
|
|||||||
crs="EPSG:4326",
|
crs="EPSG:4326",
|
||||||
metadata_json={"crs_assumed": False},
|
metadata_json={"crs_assumed": False},
|
||||||
)
|
)
|
||||||
reference = Dataset(
|
reference = _authoritative_reference(Dataset(
|
||||||
id=reference_id,
|
id=reference_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name="reference.geojson",
|
name="reference.geojson",
|
||||||
@@ -75,7 +116,7 @@ def test_qa_compare_candidate_with_reference_returns_metrics(tmp_path) -> None:
|
|||||||
storage_path=str(reference_path),
|
storage_path=str(reference_path),
|
||||||
crs="EPSG:4326",
|
crs="EPSG:4326",
|
||||||
metadata_json={"crs_assumed": False},
|
metadata_json={"crs_assumed": False},
|
||||||
)
|
))
|
||||||
|
|
||||||
result = QaService.compare_candidate_with_reference(
|
result = QaService.compare_candidate_with_reference(
|
||||||
db=FakeSession([candidate, reference]),
|
db=FakeSession([candidate, reference]),
|
||||||
@@ -117,7 +158,7 @@ def test_qa_compare_candidate_with_reference_returns_feature_level_evidence(tmp_
|
|||||||
crs="EPSG:4326",
|
crs="EPSG:4326",
|
||||||
metadata_json={"crs_assumed": False},
|
metadata_json={"crs_assumed": False},
|
||||||
)
|
)
|
||||||
reference = Dataset(
|
reference = _authoritative_reference(Dataset(
|
||||||
id=reference_id,
|
id=reference_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name="reference.geojson",
|
name="reference.geojson",
|
||||||
@@ -126,7 +167,7 @@ def test_qa_compare_candidate_with_reference_returns_feature_level_evidence(tmp_
|
|||||||
storage_path=str(reference_path),
|
storage_path=str(reference_path),
|
||||||
crs="EPSG:4326",
|
crs="EPSG:4326",
|
||||||
metadata_json={"crs_assumed": False},
|
metadata_json={"crs_assumed": False},
|
||||||
)
|
))
|
||||||
|
|
||||||
result = QaService.compare_candidate_with_reference(
|
result = QaService.compare_candidate_with_reference(
|
||||||
db=FakeSession([candidate, reference]),
|
db=FakeSession([candidate, reference]),
|
||||||
|
|||||||
@@ -49,6 +49,54 @@ def scope_area(name: str, geometry):
|
|||||||
return SimpleNamespace(name=name, geometry=geometry)
|
return SimpleNamespace(name=name, geometry=geometry)
|
||||||
|
|
||||||
|
|
||||||
|
def governed_materialization(
|
||||||
|
*,
|
||||||
|
source_name: str,
|
||||||
|
reference_layer_name: str | None,
|
||||||
|
source_metadata: dict[str, object],
|
||||||
|
dataset_id=None,
|
||||||
|
) -> SimpleNamespace:
|
||||||
|
"""Build a complete authoritative materialization for coverage tests.
|
||||||
|
|
||||||
|
Coverage is a production-facing statement. These fixtures must therefore
|
||||||
|
carry the same registry, immutable snapshot, checksum and freshness state
|
||||||
|
that a materialized official dataset needs in production.
|
||||||
|
"""
|
||||||
|
|
||||||
|
source_registry_id = uuid4()
|
||||||
|
source_snapshot_id = uuid4()
|
||||||
|
checksum_sha256 = "a" * 64
|
||||||
|
return SimpleNamespace(
|
||||||
|
id=dataset_id or uuid4(),
|
||||||
|
status="ready",
|
||||||
|
source=source_name,
|
||||||
|
source_name=source_name,
|
||||||
|
reference_layer_name=reference_layer_name,
|
||||||
|
source_metadata=dict(source_metadata),
|
||||||
|
checksum_sha256=checksum_sha256,
|
||||||
|
source_registry_id=source_registry_id,
|
||||||
|
source_snapshot_id=source_snapshot_id,
|
||||||
|
data_contract_key="geointel.vector.geojson",
|
||||||
|
data_contract_version="1.0.0",
|
||||||
|
validation_status="passed",
|
||||||
|
provenance_status="complete",
|
||||||
|
lineage_status="not_applicable",
|
||||||
|
quarantine_status="not_quarantined",
|
||||||
|
source_registry=SimpleNamespace(
|
||||||
|
source_key=source_name,
|
||||||
|
classification="authoritative",
|
||||||
|
authority_scope_json={"scope": "coverage test"},
|
||||||
|
usage_policy_json={},
|
||||||
|
),
|
||||||
|
source_snapshot=SimpleNamespace(
|
||||||
|
source_registry_id=source_registry_id,
|
||||||
|
checksum_sha256=checksum_sha256,
|
||||||
|
freshness_status="current",
|
||||||
|
ingest_status="ingested",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_coverage_catalog_uses_normalized_contracts_and_does_not_change_provider_registry() -> None:
|
def test_coverage_catalog_uses_normalized_contracts_and_does_not_change_provider_registry() -> None:
|
||||||
catalog = CoverageRegistryService.catalog()
|
catalog = CoverageRegistryService.catalog()
|
||||||
|
|
||||||
@@ -135,9 +183,8 @@ def test_coverage_resolver_only_reports_operational_for_materialized_ready_datas
|
|||||||
assert without_materialized.items[0].materialized_dataset_ids == []
|
assert without_materialized.items[0].materialized_dataset_ids == []
|
||||||
|
|
||||||
dataset_id = uuid4()
|
dataset_id = uuid4()
|
||||||
materialized = SimpleNamespace(
|
materialized = governed_materialization(
|
||||||
id=dataset_id,
|
dataset_id=dataset_id,
|
||||||
status="ready",
|
|
||||||
source_name="ngi_adminvector",
|
source_name="ngi_adminvector",
|
||||||
reference_layer_name="belgium_regions",
|
reference_layer_name="belgium_regions",
|
||||||
source_metadata={"coverage_zones": ["belgium", "flanders", "wallonia", "brussels"]},
|
source_metadata={"coverage_zones": ["belgium", "flanders", "wallonia", "brussels"]},
|
||||||
@@ -156,9 +203,8 @@ def test_coverage_resolver_only_reports_operational_for_materialized_ready_datas
|
|||||||
def test_statbel_population_materialization_does_not_masquerade_as_admin_data() -> None:
|
def test_statbel_population_materialization_does_not_masquerade_as_admin_data() -> None:
|
||||||
project_id = uuid4()
|
project_id = uuid4()
|
||||||
statbel_id = uuid4()
|
statbel_id = uuid4()
|
||||||
statbel = SimpleNamespace(
|
statbel = governed_materialization(
|
||||||
id=statbel_id,
|
dataset_id=statbel_id,
|
||||||
status="ready",
|
|
||||||
source_name="statbel",
|
source_name="statbel",
|
||||||
reference_layer_name="population",
|
reference_layer_name="population",
|
||||||
source_metadata={"coverage_zones": ["belgium", "flanders", "wallonia", "brussels"]},
|
source_metadata={"coverage_zones": ["belgium", "flanders", "wallonia", "brussels"]},
|
||||||
@@ -187,9 +233,8 @@ def test_statbel_population_materialization_does_not_masquerade_as_admin_data()
|
|||||||
def test_bounded_api_materialization_only_covers_its_persisted_bbox() -> None:
|
def test_bounded_api_materialization_only_covers_its_persisted_bbox() -> None:
|
||||||
project_id = uuid4()
|
project_id = uuid4()
|
||||||
dataset_id = uuid4()
|
dataset_id = uuid4()
|
||||||
dataset = SimpleNamespace(
|
dataset = governed_materialization(
|
||||||
id=dataset_id,
|
dataset_id=dataset_id,
|
||||||
status="ready",
|
|
||||||
source_name="spw_picc",
|
source_name="spw_picc",
|
||||||
reference_layer_name="buildings",
|
reference_layer_name="buildings",
|
||||||
source_metadata={
|
source_metadata={
|
||||||
@@ -230,8 +275,24 @@ def test_bounded_partition_union_can_be_operational() -> None:
|
|||||||
left_id = uuid4()
|
left_id = uuid4()
|
||||||
right_id = uuid4()
|
right_id = uuid4()
|
||||||
datasets = [
|
datasets = [
|
||||||
SimpleNamespace(id=left_id, status="ready", source_name="spw_picc", reference_layer_name="buildings", source_metadata={"coverage_zones": ["wallonia"], "bbox_epsg4326": [4.50, 50.50, 4.60, 50.60]}),
|
governed_materialization(
|
||||||
SimpleNamespace(id=right_id, status="ready", source_name="spw_picc", reference_layer_name="buildings", source_metadata={"coverage_zones": ["wallonia"], "bbox_epsg4326": [4.60, 50.50, 4.70, 50.60]}),
|
dataset_id=left_id,
|
||||||
|
source_name="spw_picc",
|
||||||
|
reference_layer_name="buildings",
|
||||||
|
source_metadata={
|
||||||
|
"coverage_zones": ["wallonia"],
|
||||||
|
"bbox_epsg4326": [4.50, 50.50, 4.60, 50.60],
|
||||||
|
},
|
||||||
|
),
|
||||||
|
governed_materialization(
|
||||||
|
dataset_id=right_id,
|
||||||
|
source_name="spw_picc",
|
||||||
|
reference_layer_name="buildings",
|
||||||
|
source_metadata={
|
||||||
|
"coverage_zones": ["wallonia"],
|
||||||
|
"bbox_epsg4326": [4.60, 50.50, 4.70, 50.60],
|
||||||
|
},
|
||||||
|
),
|
||||||
]
|
]
|
||||||
session = FakeSession(
|
session = FakeSession(
|
||||||
project=SimpleNamespace(id=project_id),
|
project=SimpleNamespace(id=project_id),
|
||||||
@@ -252,9 +313,7 @@ def test_spw_bathymetry_materialization_is_source_specific() -> None:
|
|||||||
scope_area("Wallonia", box(2.5, 49.5, 6.4, 50.8)),
|
scope_area("Wallonia", box(2.5, 49.5, 6.4, 50.8)),
|
||||||
]
|
]
|
||||||
selection = CoverageBBox(minx=4.851, miny=50.451, maxx=4.869, maxy=50.469)
|
selection = CoverageBBox(minx=4.851, miny=50.451, maxx=4.869, maxy=50.469)
|
||||||
spw_picc = SimpleNamespace(
|
spw_picc = governed_materialization(
|
||||||
id=uuid4(),
|
|
||||||
status="ready",
|
|
||||||
source_name="spw_picc",
|
source_name="spw_picc",
|
||||||
reference_layer_name="buildings",
|
reference_layer_name="buildings",
|
||||||
source_metadata={
|
source_metadata={
|
||||||
@@ -272,9 +331,8 @@ def test_spw_bathymetry_materialization_is_source_specific() -> None:
|
|||||||
assert without_bathymetry.items[0].materialized_dataset_ids == []
|
assert without_bathymetry.items[0].materialized_dataset_ids == []
|
||||||
|
|
||||||
bathymetry_id = uuid4()
|
bathymetry_id = uuid4()
|
||||||
bathymetry = SimpleNamespace(
|
bathymetry = governed_materialization(
|
||||||
id=bathymetry_id,
|
dataset_id=bathymetry_id,
|
||||||
status="ready",
|
|
||||||
source_name="spw_bathymetry",
|
source_name="spw_bathymetry",
|
||||||
reference_layer_name=None,
|
reference_layer_name=None,
|
||||||
source_metadata={
|
source_metadata={
|
||||||
@@ -309,9 +367,8 @@ def test_vha_bathymetry_profiles_are_operational_only_inside_the_persisted_selec
|
|||||||
assert without_profiles.items[0].source_names == ["vmm_vha_bathymetry_profiles"]
|
assert without_profiles.items[0].source_names == ["vmm_vha_bathymetry_profiles"]
|
||||||
|
|
||||||
profile_id = uuid4()
|
profile_id = uuid4()
|
||||||
profiles = SimpleNamespace(
|
profiles = governed_materialization(
|
||||||
id=profile_id,
|
dataset_id=profile_id,
|
||||||
status="ready",
|
|
||||||
source_name="vmm_vha_bathymetry_profiles",
|
source_name="vmm_vha_bathymetry_profiles",
|
||||||
reference_layer_name="bathymetry_profile_points",
|
reference_layer_name="bathymetry_profile_points",
|
||||||
source_metadata={
|
source_metadata={
|
||||||
@@ -360,9 +417,8 @@ def test_flemish_materialization_is_theme_specific() -> None:
|
|||||||
project_id = uuid4()
|
project_id = uuid4()
|
||||||
project = SimpleNamespace(id=project_id)
|
project = SimpleNamespace(id=project_id)
|
||||||
orthophoto_id = uuid4()
|
orthophoto_id = uuid4()
|
||||||
orthophoto = SimpleNamespace(
|
orthophoto = governed_materialization(
|
||||||
id=orthophoto_id,
|
dataset_id=orthophoto_id,
|
||||||
status="ready",
|
|
||||||
source_name="digitaal_vlaanderen_orthophoto",
|
source_name="digitaal_vlaanderen_orthophoto",
|
||||||
reference_layer_name="orthophoto",
|
reference_layer_name="orthophoto",
|
||||||
source_metadata={"coverage_zones": ["flanders"]},
|
source_metadata={"coverage_zones": ["flanders"]},
|
||||||
@@ -429,5 +485,5 @@ def test_frontend_prefers_materialized_national_workspace_and_resolves_drawn_bbo
|
|||||||
assert "externalApi.resolveCoverage" in coverage_hook
|
assert "externalApi.resolveCoverage" in coverage_hook
|
||||||
assert "coverage.outside_supported_scope" in map_workspace
|
assert "coverage.outside_supported_scope" in map_workspace
|
||||||
assert "coverageStatusLabel" in map_workspace
|
assert "coverageStatusLabel" in map_workspace
|
||||||
assert "coverageSelectionAvailable" in map_workspace
|
assert "activeThemeSupportsCurrentSelection" in map_workspace
|
||||||
assert "activeThemeAvailable && !regionalPartitionedThemeActive" in map_workspace
|
assert "activeThemeAvailable && !regionalPartitionedThemeActive" in map_workspace
|
||||||
|
|||||||
@@ -44,8 +44,10 @@ def _project_and_dataset():
|
|||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name="ortho.tif",
|
name="ortho.tif",
|
||||||
dataset_type="raster",
|
dataset_type="raster",
|
||||||
source="user_upload",
|
source="fixture",
|
||||||
|
source_name="fixture",
|
||||||
storage_path="storage/uploads/ortho.tif",
|
storage_path="storage/uploads/ortho.tif",
|
||||||
|
source_metadata={"fixture": True},
|
||||||
)
|
)
|
||||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||||
return db, project_id, dataset_id
|
return db, project_id, dataset_id
|
||||||
|
|||||||
@@ -0,0 +1,325 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from hashlib import sha256
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.core.errors import AppError
|
||||||
|
from app.models import DatasetQuarantine, SourceRegistry, SourceSnapshot
|
||||||
|
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||||
|
|
||||||
|
|
||||||
|
class FakeSession:
|
||||||
|
"""Explicit database double for production-runtime provenance tests."""
|
||||||
|
|
||||||
|
def __init__(self, objects: dict[tuple[type, object], object] | None = None) -> None:
|
||||||
|
self.objects = objects or {}
|
||||||
|
|
||||||
|
def get(self, model, item_id):
|
||||||
|
return self.objects.get((model, item_id))
|
||||||
|
|
||||||
|
|
||||||
|
def _write_sidecar(
|
||||||
|
model_path: Path,
|
||||||
|
*,
|
||||||
|
model_id: str = "yolo-configured",
|
||||||
|
task_type: str = "object_detection",
|
||||||
|
framework: str = "ultralytics/pytorch",
|
||||||
|
source_version: str = "test-v1",
|
||||||
|
source_registry_id: str | None = None,
|
||||||
|
source_snapshot_id: str | None = None,
|
||||||
|
) -> Path:
|
||||||
|
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
|
||||||
|
payload = {
|
||||||
|
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
|
||||||
|
"data_contract": {
|
||||||
|
"key": "geointel.model.pytorch",
|
||||||
|
"version": "1.0.0",
|
||||||
|
},
|
||||||
|
"model": {
|
||||||
|
"model_id": model_id,
|
||||||
|
"task_type": task_type,
|
||||||
|
"sha256": model_sha256,
|
||||||
|
"model_format": "pytorch",
|
||||||
|
"framework": framework,
|
||||||
|
"class_mapping": {"0": "building"},
|
||||||
|
"source_version": source_version,
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"source_registry_id": source_registry_id or str(uuid4()),
|
||||||
|
"source_snapshot_id": source_snapshot_id or str(uuid4()),
|
||||||
|
"source_registry_key": "model",
|
||||||
|
"source_snapshot_checksum_sha256": model_sha256,
|
||||||
|
},
|
||||||
|
"lineage": {
|
||||||
|
"upstream_asset_ids": ["training-corpus:test-v1"],
|
||||||
|
"upstream_checksums_sha256": ["a" * 64],
|
||||||
|
"transformations": [
|
||||||
|
{
|
||||||
|
"name": "pytorch-training",
|
||||||
|
"version": "1.0.0",
|
||||||
|
"checksum_sha256": "b" * 64,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"metadata": {
|
||||||
|
"training_manifest_sha256": "c" * 64,
|
||||||
|
},
|
||||||
|
"imported_at": "2026-08-01T10:00:00+00:00",
|
||||||
|
}
|
||||||
|
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
|
||||||
|
sidecar_path = RuntimeModelProvenanceService.manifest_path_for_model(model_path)
|
||||||
|
sidecar_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
|
||||||
|
return sidecar_path
|
||||||
|
|
||||||
|
|
||||||
|
def _governed_model_database(
|
||||||
|
*,
|
||||||
|
source_registry_id,
|
||||||
|
source_snapshot_id,
|
||||||
|
model_checksum: str,
|
||||||
|
source_version: str = "test-v1",
|
||||||
|
) -> tuple[FakeSession, SourceRegistry, SourceSnapshot]:
|
||||||
|
registry = SourceRegistry(
|
||||||
|
id=source_registry_id,
|
||||||
|
source_key="model",
|
||||||
|
display_name="Governed test model artifacts",
|
||||||
|
classification="experimental",
|
||||||
|
authority_name="GeoIntel test fixture",
|
||||||
|
freshness_status="current",
|
||||||
|
ingest_status="configured",
|
||||||
|
)
|
||||||
|
snapshot = SourceSnapshot(
|
||||||
|
id=source_snapshot_id,
|
||||||
|
source_registry_id=source_registry_id,
|
||||||
|
snapshot_key=f"model-{source_version}",
|
||||||
|
source_version=source_version,
|
||||||
|
checksum_sha256=model_checksum,
|
||||||
|
freshness_status="current",
|
||||||
|
ingest_status="ingested",
|
||||||
|
)
|
||||||
|
return (
|
||||||
|
FakeSession(
|
||||||
|
{
|
||||||
|
(SourceRegistry, source_registry_id): registry,
|
||||||
|
(SourceSnapshot, source_snapshot_id): snapshot,
|
||||||
|
}
|
||||||
|
),
|
||||||
|
registry,
|
||||||
|
snapshot,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_model_provenance_accepts_byte_bound_pytorch_sidecar_for_structural_preflight(tmp_path: Path) -> None:
|
||||||
|
model_path = tmp_path / "model.pt"
|
||||||
|
model_path.write_bytes(b"trusted local model bytes")
|
||||||
|
sidecar_path = _write_sidecar(model_path, source_version="v1")
|
||||||
|
|
||||||
|
evidence = RuntimeModelProvenanceService.validate_for_runtime(
|
||||||
|
model_path=model_path,
|
||||||
|
model_id="yolo-configured",
|
||||||
|
task_type="object_detection",
|
||||||
|
expected_model_version="v1",
|
||||||
|
allowed_frameworks=("ultralytics/pytorch",),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert evidence.model_sha256 == sha256(model_path.read_bytes()).hexdigest()
|
||||||
|
assert evidence.manifest_path == str(sidecar_path.resolve())
|
||||||
|
assert evidence.data_contract_key == "geointel.model.pytorch"
|
||||||
|
assert evidence.data_contract_version == "1.0.0"
|
||||||
|
assert len(evidence.validation_report_sha256) == 64
|
||||||
|
|
||||||
|
|
||||||
|
def test_production_runtime_requires_db_bound_model_source_snapshot(tmp_path: Path) -> None:
|
||||||
|
model_path = tmp_path / "model.pt"
|
||||||
|
model_path.write_bytes(b"governed local model bytes")
|
||||||
|
source_registry_id = uuid4()
|
||||||
|
source_snapshot_id = uuid4()
|
||||||
|
_write_sidecar(
|
||||||
|
model_path,
|
||||||
|
source_version="v1",
|
||||||
|
source_registry_id=str(source_registry_id),
|
||||||
|
source_snapshot_id=str(source_snapshot_id),
|
||||||
|
)
|
||||||
|
db, _, _ = _governed_model_database(
|
||||||
|
source_registry_id=source_registry_id,
|
||||||
|
source_snapshot_id=source_snapshot_id,
|
||||||
|
model_checksum=sha256(model_path.read_bytes()).hexdigest(),
|
||||||
|
source_version="v1",
|
||||||
|
)
|
||||||
|
|
||||||
|
evidence = RuntimeModelProvenanceService.validate_for_production_runtime(
|
||||||
|
db=db,
|
||||||
|
model_path=model_path,
|
||||||
|
model_id="yolo-configured",
|
||||||
|
task_type="object_detection",
|
||||||
|
expected_model_version="v1",
|
||||||
|
allowed_frameworks=("ultralytics/pytorch",),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert evidence.source_registry_id == str(source_registry_id)
|
||||||
|
assert evidence.source_snapshot_id == str(source_snapshot_id)
|
||||||
|
assert evidence.source_snapshot_checksum_sha256 == evidence.model_sha256
|
||||||
|
|
||||||
|
|
||||||
|
def test_production_runtime_rejects_missing_database_source_binding(tmp_path: Path) -> None:
|
||||||
|
model_path = tmp_path / "model.pt"
|
||||||
|
model_path.write_bytes(b"unbound model bytes")
|
||||||
|
_write_sidecar(model_path)
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
RuntimeModelProvenanceService.validate_for_production_runtime(
|
||||||
|
db=FakeSession(),
|
||||||
|
model_path=model_path,
|
||||||
|
model_id="yolo-configured",
|
||||||
|
task_type="object_detection",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert exc_info.value.code == "MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND"
|
||||||
|
|
||||||
|
|
||||||
|
def test_production_runtime_requires_a_database_session(tmp_path: Path) -> None:
|
||||||
|
model_path = tmp_path / "model.pt"
|
||||||
|
model_path.write_bytes(b"model bytes")
|
||||||
|
_write_sidecar(model_path)
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
RuntimeModelProvenanceService.validate_for_production_runtime(
|
||||||
|
db=None,
|
||||||
|
model_path=model_path,
|
||||||
|
model_id="yolo-configured",
|
||||||
|
task_type="object_detection",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert exc_info.value.code == "MODEL_PROVENANCE_DATABASE_REQUIRED"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("mutation", "expected_code"),
|
||||||
|
(
|
||||||
|
("registry_unsafe", "MODEL_PROVENANCE_SOURCE_REGISTRY_UNSAFE"),
|
||||||
|
("snapshot_registry_mismatch", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_REGISTRY_MISMATCH"),
|
||||||
|
("snapshot_missing", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_NOT_FOUND"),
|
||||||
|
("snapshot_quarantined", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_UNSAFE"),
|
||||||
|
("snapshot_checksum_mismatch", "MODEL_PROVENANCE_DATABASE_SNAPSHOT_CHECKSUM_MISMATCH"),
|
||||||
|
("active_quarantine", "MODEL_PROVENANCE_SOURCE_SNAPSHOT_QUARANTINED"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
def test_production_runtime_rejects_unsafe_or_inconsistent_database_snapshot(
|
||||||
|
tmp_path: Path,
|
||||||
|
mutation: str,
|
||||||
|
expected_code: str,
|
||||||
|
) -> None:
|
||||||
|
model_path = tmp_path / "model.pt"
|
||||||
|
model_path.write_bytes(b"governed model bytes")
|
||||||
|
source_registry_id = uuid4()
|
||||||
|
source_snapshot_id = uuid4()
|
||||||
|
_write_sidecar(
|
||||||
|
model_path,
|
||||||
|
source_registry_id=str(source_registry_id),
|
||||||
|
source_snapshot_id=str(source_snapshot_id),
|
||||||
|
)
|
||||||
|
db, registry, snapshot = _governed_model_database(
|
||||||
|
source_registry_id=source_registry_id,
|
||||||
|
source_snapshot_id=source_snapshot_id,
|
||||||
|
model_checksum=sha256(model_path.read_bytes()).hexdigest(),
|
||||||
|
)
|
||||||
|
if mutation == "registry_unsafe":
|
||||||
|
registry.ingest_status = "quarantined"
|
||||||
|
elif mutation == "snapshot_registry_mismatch":
|
||||||
|
snapshot.source_registry_id = uuid4()
|
||||||
|
elif mutation == "snapshot_missing":
|
||||||
|
db.objects.pop((SourceSnapshot, source_snapshot_id))
|
||||||
|
elif mutation == "snapshot_quarantined":
|
||||||
|
snapshot.ingest_status = "quarantined"
|
||||||
|
elif mutation == "snapshot_checksum_mismatch":
|
||||||
|
snapshot.checksum_sha256 = "f" * 64
|
||||||
|
elif mutation == "active_quarantine":
|
||||||
|
snapshot.quarantines = [
|
||||||
|
DatasetQuarantine(
|
||||||
|
source_snapshot_id=source_snapshot_id,
|
||||||
|
stage="test",
|
||||||
|
reason_code="test_active_quarantine",
|
||||||
|
status="quarantined",
|
||||||
|
)
|
||||||
|
]
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
RuntimeModelProvenanceService.validate_for_production_runtime(
|
||||||
|
db=db,
|
||||||
|
model_path=model_path,
|
||||||
|
model_id="yolo-configured",
|
||||||
|
task_type="object_detection",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert exc_info.value.code == expected_code
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_model_provenance_rejects_missing_sidecar(tmp_path: Path) -> None:
|
||||||
|
model_path = tmp_path / "model.pt"
|
||||||
|
model_path.write_bytes(b"unmanifested local model bytes")
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
RuntimeModelProvenanceService.validate_for_runtime(
|
||||||
|
model_path=model_path,
|
||||||
|
model_id="yolo-configured",
|
||||||
|
task_type="object_detection",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert exc_info.value.code == "MODEL_PROVENANCE_MANIFEST_MISSING"
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_model_provenance_rejects_model_bytes_tampered_after_manifest(tmp_path: Path) -> None:
|
||||||
|
model_path = tmp_path / "model.pt"
|
||||||
|
model_path.write_bytes(b"original local model bytes")
|
||||||
|
_write_sidecar(model_path)
|
||||||
|
model_path.write_bytes(b"tampered local model bytes")
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
RuntimeModelProvenanceService.validate_for_runtime(
|
||||||
|
model_path=model_path,
|
||||||
|
model_id="yolo-configured",
|
||||||
|
task_type="object_detection",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert exc_info.value.code == "MODEL_PROVENANCE_MODEL_CHECKSUM_MISMATCH"
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_model_provenance_rejects_tampered_manifest_contents(tmp_path: Path) -> None:
|
||||||
|
model_path = tmp_path / "model.pt"
|
||||||
|
model_path.write_bytes(b"original local model bytes")
|
||||||
|
sidecar_path = _write_sidecar(model_path)
|
||||||
|
payload = json.loads(sidecar_path.read_text(encoding="utf-8"))
|
||||||
|
payload["model"]["class_mapping"]["1"] = "road"
|
||||||
|
sidecar_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
RuntimeModelProvenanceService.validate_for_runtime(
|
||||||
|
model_path=model_path,
|
||||||
|
model_id="yolo-configured",
|
||||||
|
task_type="object_detection",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert exc_info.value.code == "MODEL_PROVENANCE_MANIFEST_CHECKSUM_MISMATCH"
|
||||||
|
|
||||||
|
|
||||||
|
def test_runtime_model_provenance_rejects_other_contract_even_if_structurally_valid(tmp_path: Path) -> None:
|
||||||
|
model_path = tmp_path / "model.pt"
|
||||||
|
model_path.write_bytes(b"local model bytes")
|
||||||
|
sidecar_path = _write_sidecar(model_path)
|
||||||
|
payload = json.loads(sidecar_path.read_text(encoding="utf-8"))
|
||||||
|
payload["data_contract"]["key"] = "geointel.vector.geojson"
|
||||||
|
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
|
||||||
|
sidecar_path.write_text(json.dumps(payload, sort_keys=True), encoding="utf-8")
|
||||||
|
|
||||||
|
with pytest.raises(AppError) as exc_info:
|
||||||
|
RuntimeModelProvenanceService.validate_for_runtime(
|
||||||
|
model_path=model_path,
|
||||||
|
model_id="yolo-configured",
|
||||||
|
task_type="object_detection",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert exc_info.value.code == "MODEL_PROVENANCE_MANIFEST_INVALID"
|
||||||
@@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from hashlib import sha256
|
||||||
import json
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
@@ -8,9 +9,10 @@ import pytest
|
|||||||
from geoalchemy2.shape import to_shape
|
from geoalchemy2.shape import to_shape
|
||||||
|
|
||||||
from app.core.config import Settings
|
from app.core.config import Settings
|
||||||
from app.models import Dataset, Project, Segmentation
|
from app.models import Dataset, Project, Segmentation, SourceRegistry, SourceSnapshot
|
||||||
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
|
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
|
||||||
from app.services.model_registry_service import ModelRegistryService
|
from app.services.model_registry_service import ModelRegistryService
|
||||||
|
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||||
from app.services.segmentation_service import SegmentationService
|
from app.services.segmentation_service import SegmentationService
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
@@ -80,18 +82,58 @@ class MissingDependencySegAdapter(AvailableSegAdapter):
|
|||||||
return False
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
class NeverLoadSegAdapter(AvailableSegAdapter):
|
||||||
|
load_calls = 0
|
||||||
|
|
||||||
|
def load_model(self, model_path: Path):
|
||||||
|
type(self).load_calls += 1
|
||||||
|
raise AssertionError("unmanifested weights must not reach adapter.load_model")
|
||||||
|
|
||||||
|
|
||||||
def _project_and_dataset(dataset_type: str = "raster"):
|
def _project_and_dataset(dataset_type: str = "raster"):
|
||||||
project_id = uuid4()
|
project_id = uuid4()
|
||||||
dataset_id = uuid4()
|
dataset_id = uuid4()
|
||||||
|
source_id = uuid4()
|
||||||
|
snapshot_id = uuid4()
|
||||||
|
checksum = "a" * 64
|
||||||
project = Project(id=project_id, name="Mol")
|
project = Project(id=project_id, name="Mol")
|
||||||
|
source = SourceRegistry(
|
||||||
|
id=source_id,
|
||||||
|
source_key="digitaal_vlaanderen_orthophoto",
|
||||||
|
display_name="Governed orthophoto test source",
|
||||||
|
classification="contextual",
|
||||||
|
authority_name="Digitaal Vlaanderen",
|
||||||
|
authority_scope_json={"zone": "Flanders", "role": "imagery"},
|
||||||
|
)
|
||||||
|
snapshot = SourceSnapshot(
|
||||||
|
id=snapshot_id,
|
||||||
|
source_registry_id=source_id,
|
||||||
|
snapshot_key="configured-segmentation-orthophoto",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
ingest_status="ingested",
|
||||||
|
freshness_status="current",
|
||||||
|
)
|
||||||
dataset = Dataset(
|
dataset = Dataset(
|
||||||
id=dataset_id,
|
id=dataset_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name="ortho.tif",
|
name="ortho.tif",
|
||||||
dataset_type=dataset_type,
|
dataset_type=dataset_type,
|
||||||
source="user_upload",
|
source="digitaal_vlaanderen_orthophoto",
|
||||||
|
source_name="digitaal_vlaanderen_orthophoto",
|
||||||
storage_path="storage/uploads/ortho.tif",
|
storage_path="storage/uploads/ortho.tif",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
source_registry_id=source_id,
|
||||||
|
source_snapshot_id=snapshot_id,
|
||||||
|
data_contract_key="geointel.raster.geotiff",
|
||||||
|
data_contract_version="1.0.0",
|
||||||
|
validation_status="passed",
|
||||||
|
provenance_status="complete",
|
||||||
|
lineage_status="not_applicable",
|
||||||
|
quarantine_status="not_quarantined",
|
||||||
|
status="ready",
|
||||||
)
|
)
|
||||||
|
dataset.source_registry = source
|
||||||
|
dataset.source_snapshot = snapshot
|
||||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||||
return db, project_id, dataset_id
|
return db, project_id, dataset_id
|
||||||
|
|
||||||
@@ -108,6 +150,102 @@ def _settings(tmp_path: Path, **overrides) -> Settings:
|
|||||||
return Settings(**values)
|
return Settings(**values)
|
||||||
|
|
||||||
|
|
||||||
|
def _write_model_sidecar(
|
||||||
|
model_path: Path,
|
||||||
|
*,
|
||||||
|
model_id: str,
|
||||||
|
framework: str,
|
||||||
|
source_version: str | None,
|
||||||
|
db: FakeSession | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Create explicit local test evidence; no production code creates sidecars."""
|
||||||
|
|
||||||
|
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
|
||||||
|
source_registry_id = uuid4()
|
||||||
|
source_snapshot_id = uuid4()
|
||||||
|
resolved_source_version = source_version or "test-v1"
|
||||||
|
if db is not None:
|
||||||
|
source_registry = SourceRegistry(
|
||||||
|
id=source_registry_id,
|
||||||
|
source_key="model",
|
||||||
|
display_name="Governed test model artifact",
|
||||||
|
classification="experimental",
|
||||||
|
authority_name="GeoIntel test fixture",
|
||||||
|
freshness_status="current",
|
||||||
|
ingest_status="configured",
|
||||||
|
)
|
||||||
|
source_snapshot = SourceSnapshot(
|
||||||
|
id=source_snapshot_id,
|
||||||
|
source_registry_id=source_registry_id,
|
||||||
|
snapshot_key=f"model-{model_id}-{resolved_source_version}",
|
||||||
|
source_version=resolved_source_version,
|
||||||
|
checksum_sha256=model_sha256,
|
||||||
|
freshness_status="current",
|
||||||
|
ingest_status="ingested",
|
||||||
|
)
|
||||||
|
db.objects[(SourceRegistry, source_registry_id)] = source_registry
|
||||||
|
db.objects[(SourceSnapshot, source_snapshot_id)] = source_snapshot
|
||||||
|
payload = {
|
||||||
|
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
|
||||||
|
"data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"},
|
||||||
|
"model": {
|
||||||
|
"model_id": model_id,
|
||||||
|
"task_type": "segmentation",
|
||||||
|
"sha256": model_sha256,
|
||||||
|
"model_format": "pytorch",
|
||||||
|
"framework": framework,
|
||||||
|
"class_mapping": {"0": "segment"},
|
||||||
|
"source_version": resolved_source_version,
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"source_registry_id": str(source_registry_id),
|
||||||
|
"source_snapshot_id": str(source_snapshot_id),
|
||||||
|
"source_registry_key": "model",
|
||||||
|
"source_snapshot_checksum_sha256": model_sha256,
|
||||||
|
},
|
||||||
|
"lineage": {
|
||||||
|
"upstream_asset_ids": ["test-training-corpus"],
|
||||||
|
"upstream_checksums_sha256": ["a" * 64],
|
||||||
|
"transformations": [
|
||||||
|
{"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"metadata": {"training_manifest_sha256": "c" * 64},
|
||||||
|
"imported_at": "2026-08-01T10:00:00+00:00",
|
||||||
|
}
|
||||||
|
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
|
||||||
|
RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text(
|
||||||
|
json.dumps(payload, sort_keys=True),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _write_configured_model_sidecars(
|
||||||
|
tmp_path: Path,
|
||||||
|
settings: Settings,
|
||||||
|
*,
|
||||||
|
include_yolo: bool = True,
|
||||||
|
include_sam: bool = True,
|
||||||
|
db: FakeSession | None = None,
|
||||||
|
) -> None:
|
||||||
|
if include_yolo:
|
||||||
|
_write_model_sidecar(
|
||||||
|
tmp_path / "seg.pt",
|
||||||
|
model_id=settings.yolo_seg_model_id,
|
||||||
|
framework="ultralytics/pytorch",
|
||||||
|
source_version=settings.yolo_seg_model_version,
|
||||||
|
db=db,
|
||||||
|
)
|
||||||
|
if include_sam:
|
||||||
|
_write_model_sidecar(
|
||||||
|
tmp_path / "sam.pt",
|
||||||
|
model_id=settings.sam_model_id,
|
||||||
|
framework="ultralytics/sam",
|
||||||
|
source_version=settings.sam_model_version,
|
||||||
|
db=db,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _manifest(tmp_path: Path, tile_count: int = 1) -> Path:
|
def _manifest(tmp_path: Path, tile_count: int = 1) -> Path:
|
||||||
tiles = []
|
tiles = []
|
||||||
for index in range(tile_count):
|
for index in range(tile_count):
|
||||||
@@ -172,10 +310,31 @@ def test_segmentation_models_report_dependency_unavailable(tmp_path: Path) -> No
|
|||||||
assert models["sam-configured"].status == "dependency_unavailable"
|
assert models["sam-configured"].status == "dependency_unavailable"
|
||||||
|
|
||||||
|
|
||||||
|
def test_segmentation_models_require_runtime_provenance_sidecars(tmp_path: Path) -> None:
|
||||||
|
(tmp_path / "seg.pt").write_bytes(b"unmanifested yolo segmentation weights")
|
||||||
|
(tmp_path / "sam.pt").write_bytes(b"unmanifested sam weights")
|
||||||
|
settings = _settings(tmp_path)
|
||||||
|
|
||||||
|
models = {
|
||||||
|
model.model_id: model
|
||||||
|
for model in ModelRegistryService.list_segmentation_model_capabilities(
|
||||||
|
settings=settings,
|
||||||
|
yolo_seg_adapter_class=AvailableSegAdapter,
|
||||||
|
sam_adapter_class=ClassAgnosticSamAdapter,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
assert models["yolo-seg-configured"].configured is False
|
||||||
|
assert models["yolo-seg-configured"].status == "contract_incomplete"
|
||||||
|
assert models["sam-configured"].configured is False
|
||||||
|
assert models["sam-configured"].status == "contract_incomplete"
|
||||||
|
|
||||||
|
|
||||||
def test_segmentation_models_report_configured_with_local_weights(tmp_path: Path) -> None:
|
def test_segmentation_models_report_configured_with_local_weights(tmp_path: Path) -> None:
|
||||||
(tmp_path / "seg.pt").write_bytes(b"weights")
|
(tmp_path / "seg.pt").write_bytes(b"weights")
|
||||||
(tmp_path / "sam.pt").write_bytes(b"weights")
|
(tmp_path / "sam.pt").write_bytes(b"weights")
|
||||||
settings = _settings(tmp_path)
|
settings = _settings(tmp_path)
|
||||||
|
_write_configured_model_sidecars(tmp_path, settings)
|
||||||
|
|
||||||
models = {
|
models = {
|
||||||
model.model_id: model
|
model.model_id: model
|
||||||
@@ -204,6 +363,7 @@ def test_configured_segmentation_requires_tile_manifest(tmp_path: Path) -> None:
|
|||||||
(tmp_path / "seg.pt").write_bytes(b"weights")
|
(tmp_path / "seg.pt").write_bytes(b"weights")
|
||||||
db, project_id, dataset_id = _project_and_dataset()
|
db, project_id, dataset_id = _project_and_dataset()
|
||||||
settings = _settings(tmp_path)
|
settings = _settings(tmp_path)
|
||||||
|
_write_configured_model_sidecars(tmp_path, settings, include_sam=False, db=db)
|
||||||
|
|
||||||
with pytest.raises(Exception) as exc_info:
|
with pytest.raises(Exception) as exc_info:
|
||||||
SegmentationService.run_segmentation(
|
SegmentationService.run_segmentation(
|
||||||
@@ -220,10 +380,60 @@ def test_configured_segmentation_requires_tile_manifest(tmp_path: Path) -> None:
|
|||||||
assert getattr(exc_info.value, "code", None) == "SEGMENTATION_TILE_MANIFEST_REQUIRED"
|
assert getattr(exc_info.value, "code", None) == "SEGMENTATION_TILE_MANIFEST_REQUIRED"
|
||||||
|
|
||||||
|
|
||||||
|
def test_configured_segmentation_fails_before_adapter_load_without_sidecar(tmp_path: Path) -> None:
|
||||||
|
(tmp_path / "seg.pt").write_bytes(b"unmanifested weights")
|
||||||
|
db, project_id, dataset_id = _project_and_dataset()
|
||||||
|
settings = _settings(tmp_path)
|
||||||
|
NeverLoadSegAdapter.load_calls = 0
|
||||||
|
|
||||||
|
response = SegmentationService.run_segmentation(
|
||||||
|
db=db,
|
||||||
|
project_id=project_id,
|
||||||
|
dataset_id=dataset_id,
|
||||||
|
model_id="yolo-seg-configured",
|
||||||
|
confidence_threshold=0.5,
|
||||||
|
tile_manifest_path=str(_manifest(tmp_path)),
|
||||||
|
settings=settings,
|
||||||
|
yolo_seg_adapter_class=NeverLoadSegAdapter,
|
||||||
|
sam_adapter_class=ClassAgnosticSamAdapter,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status == "failed"
|
||||||
|
assert response.error_code == "SEGMENTATION_MODEL_UNAVAILABLE"
|
||||||
|
assert NeverLoadSegAdapter.load_calls == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_configured_segmentation_rejects_unbound_model_snapshot_before_adapter_load(tmp_path: Path) -> None:
|
||||||
|
(tmp_path / "seg.pt").write_bytes(b"structurally valid but unbound weights")
|
||||||
|
db, project_id, dataset_id = _project_and_dataset()
|
||||||
|
settings = _settings(tmp_path)
|
||||||
|
# The sidecar passes catalog validation but its source registry/snapshot
|
||||||
|
# was never registered in this production-session fixture.
|
||||||
|
_write_configured_model_sidecars(tmp_path, settings, include_sam=False)
|
||||||
|
NeverLoadSegAdapter.load_calls = 0
|
||||||
|
|
||||||
|
response = SegmentationService.run_segmentation(
|
||||||
|
db=db,
|
||||||
|
project_id=project_id,
|
||||||
|
dataset_id=dataset_id,
|
||||||
|
model_id="yolo-seg-configured",
|
||||||
|
confidence_threshold=0.5,
|
||||||
|
tile_manifest_path=str(_manifest(tmp_path)),
|
||||||
|
settings=settings,
|
||||||
|
yolo_seg_adapter_class=NeverLoadSegAdapter,
|
||||||
|
sam_adapter_class=ClassAgnosticSamAdapter,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.status == "failed"
|
||||||
|
assert response.error_code == "MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND"
|
||||||
|
assert NeverLoadSegAdapter.load_calls == 0
|
||||||
|
|
||||||
|
|
||||||
def test_configured_yolo_seg_run_persists_georeferenced_masks(tmp_path: Path) -> None:
|
def test_configured_yolo_seg_run_persists_georeferenced_masks(tmp_path: Path) -> None:
|
||||||
(tmp_path / "seg.pt").write_bytes(b"weights")
|
(tmp_path / "seg.pt").write_bytes(b"weights")
|
||||||
db, project_id, dataset_id = _project_and_dataset()
|
db, project_id, dataset_id = _project_and_dataset()
|
||||||
settings = _settings(tmp_path)
|
settings = _settings(tmp_path)
|
||||||
|
_write_configured_model_sidecars(tmp_path, settings, include_sam=False, db=db)
|
||||||
manifest_path = _manifest(tmp_path)
|
manifest_path = _manifest(tmp_path)
|
||||||
|
|
||||||
response = SegmentationService.run_segmentation(
|
response = SegmentationService.run_segmentation(
|
||||||
@@ -255,12 +465,14 @@ def test_configured_yolo_seg_run_persists_georeferenced_masks(tmp_path: Path) ->
|
|||||||
assert segmentation.area_m2 is not None and segmentation.area_m2 > 0
|
assert segmentation.area_m2 is not None and segmentation.area_m2 > 0
|
||||||
assert segmentation.provenance_json["inference"] == "local"
|
assert segmentation.provenance_json["inference"] == "local"
|
||||||
assert segmentation.provenance_json["model_id"] == "yolo-seg-configured"
|
assert segmentation.provenance_json["model_id"] == "yolo-seg-configured"
|
||||||
|
assert segmentation.provenance_json["runtime_model_provenance"]["data_contract_key"] == "geointel.model.pytorch"
|
||||||
|
|
||||||
|
|
||||||
def test_configured_sam_run_is_class_agnostic(tmp_path: Path) -> None:
|
def test_configured_sam_run_is_class_agnostic(tmp_path: Path) -> None:
|
||||||
(tmp_path / "sam.pt").write_bytes(b"weights")
|
(tmp_path / "sam.pt").write_bytes(b"weights")
|
||||||
db, project_id, dataset_id = _project_and_dataset()
|
db, project_id, dataset_id = _project_and_dataset()
|
||||||
settings = _settings(tmp_path)
|
settings = _settings(tmp_path)
|
||||||
|
_write_configured_model_sidecars(tmp_path, settings, include_yolo=False, db=db)
|
||||||
manifest_path = _manifest(tmp_path)
|
manifest_path = _manifest(tmp_path)
|
||||||
|
|
||||||
response = SegmentationService.run_segmentation(
|
response = SegmentationService.run_segmentation(
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ from shapely.geometry import box
|
|||||||
|
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.main import app
|
from app.main import app
|
||||||
from app.models import Area, Dataset, Export
|
from app.models import Area, Dataset, Export, SourceRegistry, SourceSnapshot
|
||||||
from app.schemas.export import ExportCreateResponse
|
from app.schemas.export import ExportCreateResponse
|
||||||
from app.services.export_service import ExportService
|
from app.services.export_service import ExportService
|
||||||
from app.services.storage_service import StorageService
|
from app.services.storage_service import StorageService
|
||||||
@@ -45,18 +45,56 @@ class FakeSession:
|
|||||||
return row
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def _govern_fixture_dataset(dataset: Dataset) -> Dataset:
|
||||||
|
source_id = uuid4()
|
||||||
|
snapshot_id = uuid4()
|
||||||
|
checksum = "a" * 64
|
||||||
|
source = SourceRegistry(
|
||||||
|
id=source_id,
|
||||||
|
source_key="grb",
|
||||||
|
display_name="GRB map export test source",
|
||||||
|
classification="authoritative",
|
||||||
|
authority_name="Digitaal Vlaanderen",
|
||||||
|
authority_scope_json={"zone": "Flanders"},
|
||||||
|
usage_policy_json={"ground_truth_allowed": True},
|
||||||
|
)
|
||||||
|
snapshot = SourceSnapshot(
|
||||||
|
id=snapshot_id,
|
||||||
|
source_registry_id=source_id,
|
||||||
|
snapshot_key=f"map-export-grb-{dataset.id}",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
ingest_status="ingested",
|
||||||
|
freshness_status="current",
|
||||||
|
)
|
||||||
|
dataset.source = "grb"
|
||||||
|
dataset.source_name = "grb"
|
||||||
|
dataset.checksum_sha256 = checksum
|
||||||
|
dataset.source_registry_id = source_id
|
||||||
|
dataset.source_snapshot_id = snapshot_id
|
||||||
|
dataset.data_contract_key = "geointel.vector.geojson"
|
||||||
|
dataset.data_contract_version = "1.0.0"
|
||||||
|
dataset.validation_status = "passed"
|
||||||
|
dataset.provenance_status = "complete"
|
||||||
|
dataset.lineage_status = "not_applicable"
|
||||||
|
dataset.quarantine_status = "not_quarantined"
|
||||||
|
dataset.status = "ready"
|
||||||
|
dataset.source_registry = source
|
||||||
|
dataset.source_snapshot = snapshot
|
||||||
|
return dataset
|
||||||
|
|
||||||
|
|
||||||
def test_vector_selection_geojson_export_persists_handoff_artifact(tmp_path, monkeypatch) -> None:
|
def test_vector_selection_geojson_export_persists_handoff_artifact(tmp_path, monkeypatch) -> None:
|
||||||
project_id = uuid4()
|
project_id = uuid4()
|
||||||
dataset_id = uuid4()
|
dataset_id = uuid4()
|
||||||
export_path = tmp_path / "exports" / "selection.geojson"
|
export_path = tmp_path / "exports" / "selection.geojson"
|
||||||
dataset = Dataset(
|
dataset = _govern_fixture_dataset(Dataset(
|
||||||
id=dataset_id,
|
id=dataset_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name="candidate.geojson",
|
name="candidate.geojson",
|
||||||
dataset_type="vector",
|
dataset_type="vector",
|
||||||
source="fixture",
|
source="fixture",
|
||||||
status="ready",
|
status="ready",
|
||||||
)
|
))
|
||||||
db = FakeSession({(Dataset, dataset_id): dataset})
|
db = FakeSession({(Dataset, dataset_id): dataset})
|
||||||
selection_bbox = {"min_x": 4.9, "min_y": 50.9, "max_x": 5.2, "max_y": 51.2, "crs": "EPSG:4326"}
|
selection_bbox = {"min_x": 4.9, "min_y": 50.9, "max_x": 5.2, "max_y": 51.2, "crs": "EPSG:4326"}
|
||||||
selection_payload = {
|
selection_payload = {
|
||||||
@@ -135,14 +173,14 @@ def test_vector_selection_export_uses_exact_area_scope_when_requested(tmp_path,
|
|||||||
project_id = uuid4()
|
project_id = uuid4()
|
||||||
dataset_id = uuid4()
|
dataset_id = uuid4()
|
||||||
area_id = uuid4()
|
area_id = uuid4()
|
||||||
dataset = Dataset(
|
dataset = _govern_fixture_dataset(Dataset(
|
||||||
id=dataset_id,
|
id=dataset_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name="regional-buildings.geojson",
|
name="regional-buildings.geojson",
|
||||||
dataset_type="vector",
|
dataset_type="vector",
|
||||||
source="fixture",
|
source="fixture",
|
||||||
status="ready",
|
status="ready",
|
||||||
)
|
))
|
||||||
area_shape = box(5.0, 51.1, 5.2, 51.3)
|
area_shape = box(5.0, 51.1, 5.2, 51.3)
|
||||||
area_geometry = from_shape(area_shape, srid=4326)
|
area_geometry = from_shape(area_shape, srid=4326)
|
||||||
area = SimpleNamespace(id=area_id, project_id=project_id, name="Gemeente Mol", geometry=area_geometry)
|
area = SimpleNamespace(id=area_id, project_id=project_id, name="Gemeente Mol", geometry=area_geometry)
|
||||||
@@ -184,7 +222,7 @@ def test_area_constrained_bbox_uses_intersection_and_disables_full_area_fast_pat
|
|||||||
project_id = uuid4()
|
project_id = uuid4()
|
||||||
dataset_id = uuid4()
|
dataset_id = uuid4()
|
||||||
area_id = uuid4()
|
area_id = uuid4()
|
||||||
dataset = Dataset(
|
dataset = _govern_fixture_dataset(Dataset(
|
||||||
id=dataset_id,
|
id=dataset_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
area_id=area_id,
|
area_id=area_id,
|
||||||
@@ -193,7 +231,7 @@ def test_area_constrained_bbox_uses_intersection_and_disables_full_area_fast_pat
|
|||||||
source="fixture",
|
source="fixture",
|
||||||
source_metadata={"geometry_clipped_to_area": True},
|
source_metadata={"geometry_clipped_to_area": True},
|
||||||
status="ready",
|
status="ready",
|
||||||
)
|
))
|
||||||
area_shape = box(5.0, 51.0, 5.2, 51.2)
|
area_shape = box(5.0, 51.0, 5.2, 51.2)
|
||||||
area = SimpleNamespace(
|
area = SimpleNamespace(
|
||||||
id=area_id,
|
id=area_id,
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ from uuid import uuid4
|
|||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
from app.main import app
|
from app.main import app
|
||||||
from app.models import Dataset, VectorFeature
|
from app.models import Dataset
|
||||||
from app.schemas.dataset import DatasetCreateResponse
|
from app.schemas.dataset import DatasetCreateResponse
|
||||||
from app.services.storage_service import StorageService
|
from app.services.storage_service import StorageService
|
||||||
from app.services.vector_feature_service import VectorFeatureService
|
from app.services.vector_feature_service import VectorFeatureService
|
||||||
@@ -124,7 +124,7 @@ def test_vector_selection_derive_persists_queryable_derived_dataset(tmp_path, mo
|
|||||||
assert response.provenance_metadata["source_dataset_id"] == str(dataset_id)
|
assert response.provenance_metadata["source_dataset_id"] == str(dataset_id)
|
||||||
assert response.provenance_metadata["source_table"] == "vector_features"
|
assert response.provenance_metadata["source_table"] == "vector_features"
|
||||||
assert persisted_features[0]["dataset_id"] == derived.id
|
assert persisted_features[0]["dataset_id"] == derived.id
|
||||||
assert persisted_features[0]["commit"] is True
|
assert persisted_features[0]["commit"] is False
|
||||||
derived_payload = json.loads(output_path.read_text(encoding="utf-8"))
|
derived_payload = json.loads(output_path.read_text(encoding="utf-8"))
|
||||||
props = derived_payload["features"][0]["properties"]
|
props = derived_payload["features"][0]["properties"]
|
||||||
assert props["source_vector_feature_id"] == "source-row-1"
|
assert props["source_vector_feature_id"] == "source-row-1"
|
||||||
|
|||||||
@@ -75,5 +75,7 @@ def test_operator_yolo_train_smoke_script_contract() -> None:
|
|||||||
assert '"dataset_summary_sha256"' in script
|
assert '"dataset_summary_sha256"' in script
|
||||||
assert '"base_model_sha256"' in script
|
assert '"base_model_sha256"' in script
|
||||||
assert '"trained_model_sha256"' in script
|
assert '"trained_model_sha256"' in script
|
||||||
|
assert "training_release_manifest.py" in script
|
||||||
|
assert "verify" in script
|
||||||
assert "download" not in script.lower()
|
assert "download" not in script.lower()
|
||||||
assert "fixture_mode" not in script
|
assert "fixture_mode" not in script
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from hashlib import sha256
|
||||||
import json
|
import json
|
||||||
import subprocess
|
import subprocess
|
||||||
import sys
|
import sys
|
||||||
@@ -9,6 +10,7 @@ from fastapi.testclient import TestClient
|
|||||||
|
|
||||||
from app.core.config import Settings
|
from app.core.config import Settings
|
||||||
from app.main import app
|
from app.main import app
|
||||||
|
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||||
from app.services.yolo_preflight_service import YoloPreflightService
|
from app.services.yolo_preflight_service import YoloPreflightService
|
||||||
|
|
||||||
|
|
||||||
@@ -57,6 +59,43 @@ def _manifest(tmp_path: Path, tile_count: int = 1) -> Path:
|
|||||||
return manifest_path
|
return manifest_path
|
||||||
|
|
||||||
|
|
||||||
|
def _write_model_sidecar(model_path: Path, settings: Settings) -> None:
|
||||||
|
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
|
||||||
|
payload = {
|
||||||
|
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
|
||||||
|
"data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"},
|
||||||
|
"model": {
|
||||||
|
"model_id": settings.yolo_model_id,
|
||||||
|
"task_type": "object_detection",
|
||||||
|
"sha256": model_sha256,
|
||||||
|
"model_format": "pytorch",
|
||||||
|
"framework": "ultralytics/pytorch",
|
||||||
|
"class_mapping": {"0": "building"},
|
||||||
|
"source_version": settings.yolo_model_version or "test-v1",
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"source_registry_id": "11111111-1111-4111-8111-111111111111",
|
||||||
|
"source_snapshot_id": "22222222-2222-4222-8222-222222222222",
|
||||||
|
"source_registry_key": "model",
|
||||||
|
"source_snapshot_checksum_sha256": model_sha256,
|
||||||
|
},
|
||||||
|
"lineage": {
|
||||||
|
"upstream_asset_ids": ["test-training-corpus"],
|
||||||
|
"upstream_checksums_sha256": ["a" * 64],
|
||||||
|
"transformations": [
|
||||||
|
{"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"metadata": {"training_manifest_sha256": "c" * 64},
|
||||||
|
"imported_at": "2026-08-01T10:00:00+00:00",
|
||||||
|
}
|
||||||
|
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
|
||||||
|
RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text(
|
||||||
|
json.dumps(payload, sort_keys=True),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_yolo_preflight_reports_disabled_without_loading_model(tmp_path: Path, monkeypatch) -> None:
|
def test_yolo_preflight_reports_disabled_without_loading_model(tmp_path: Path, monkeypatch) -> None:
|
||||||
monkeypatch.setenv("YOLO_CONFIG_DIR", str(tmp_path / "ultralytics"))
|
monkeypatch.setenv("YOLO_CONFIG_DIR", str(tmp_path / "ultralytics"))
|
||||||
|
|
||||||
@@ -99,9 +138,11 @@ def test_yolo_preflight_validates_model_and_manifest_without_importing_yolo(tmp_
|
|||||||
model_path = tmp_path / "model.pt"
|
model_path = tmp_path / "model.pt"
|
||||||
model_path.write_bytes(b"weights")
|
model_path.write_bytes(b"weights")
|
||||||
manifest_path = _manifest(tmp_path, tile_count=2)
|
manifest_path = _manifest(tmp_path, tile_count=2)
|
||||||
|
settings = Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4)
|
||||||
|
_write_model_sidecar(model_path, settings)
|
||||||
|
|
||||||
result = YoloPreflightService.run(
|
result = YoloPreflightService.run(
|
||||||
settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4),
|
settings=settings,
|
||||||
tile_manifest_path=str(manifest_path),
|
tile_manifest_path=str(manifest_path),
|
||||||
yolo_adapter_class=AvailableAdapter,
|
yolo_adapter_class=AvailableAdapter,
|
||||||
)
|
)
|
||||||
@@ -109,6 +150,7 @@ def test_yolo_preflight_validates_model_and_manifest_without_importing_yolo(tmp_
|
|||||||
assert result["status"] == "ready"
|
assert result["status"] == "ready"
|
||||||
assert result["checks"]["dependencies_available"] is True
|
assert result["checks"]["dependencies_available"] is True
|
||||||
assert result["checks"]["model_file_exists"] is True
|
assert result["checks"]["model_file_exists"] is True
|
||||||
|
assert result["checks"]["model_provenance_valid"] is True
|
||||||
assert result["checks"]["manifest_valid"] is True
|
assert result["checks"]["manifest_valid"] is True
|
||||||
assert result["tile_count"] == 2
|
assert result["tile_count"] == 2
|
||||||
assert result["will_download_models"] is False
|
assert result["will_download_models"] is False
|
||||||
@@ -120,9 +162,11 @@ def test_yolo_preflight_marks_assumed_dependencies_in_runtime_details(tmp_path:
|
|||||||
model_path = tmp_path / "model.pt"
|
model_path = tmp_path / "model.pt"
|
||||||
model_path.write_bytes(b"weights")
|
model_path.write_bytes(b"weights")
|
||||||
manifest_path = _manifest(tmp_path, tile_count=1)
|
manifest_path = _manifest(tmp_path, tile_count=1)
|
||||||
|
settings = Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4)
|
||||||
|
_write_model_sidecar(model_path, settings)
|
||||||
|
|
||||||
result = YoloPreflightService.run(
|
result = YoloPreflightService.run(
|
||||||
settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4),
|
settings=settings,
|
||||||
tile_manifest_path=str(manifest_path),
|
tile_manifest_path=str(manifest_path),
|
||||||
yolo_adapter_class=MissingDependencyAdapter,
|
yolo_adapter_class=MissingDependencyAdapter,
|
||||||
assume_dependencies=True,
|
assume_dependencies=True,
|
||||||
@@ -138,9 +182,11 @@ def test_yolo_preflight_can_explicitly_smoke_load_local_model(tmp_path: Path) ->
|
|||||||
model_path = tmp_path / "model.pt"
|
model_path = tmp_path / "model.pt"
|
||||||
model_path.write_bytes(b"weights")
|
model_path.write_bytes(b"weights")
|
||||||
manifest_path = _manifest(tmp_path)
|
manifest_path = _manifest(tmp_path)
|
||||||
|
settings = Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4)
|
||||||
|
_write_model_sidecar(model_path, settings)
|
||||||
|
|
||||||
result = YoloPreflightService.run(
|
result = YoloPreflightService.run(
|
||||||
settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4),
|
settings=settings,
|
||||||
tile_manifest_path=str(manifest_path),
|
tile_manifest_path=str(manifest_path),
|
||||||
yolo_adapter_class=AvailableAdapter,
|
yolo_adapter_class=AvailableAdapter,
|
||||||
check_model_load=True,
|
check_model_load=True,
|
||||||
@@ -156,9 +202,11 @@ def test_yolo_preflight_can_explicitly_smoke_load_local_model(tmp_path: Path) ->
|
|||||||
def test_yolo_preflight_reports_explicit_model_load_failure(tmp_path: Path) -> None:
|
def test_yolo_preflight_reports_explicit_model_load_failure(tmp_path: Path) -> None:
|
||||||
model_path = tmp_path / "model.pt"
|
model_path = tmp_path / "model.pt"
|
||||||
model_path.write_bytes(b"weights")
|
model_path.write_bytes(b"weights")
|
||||||
|
settings = Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4)
|
||||||
|
_write_model_sidecar(model_path, settings)
|
||||||
|
|
||||||
result = YoloPreflightService.run(
|
result = YoloPreflightService.run(
|
||||||
settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4),
|
settings=settings,
|
||||||
tile_manifest_path=str(_manifest(tmp_path)),
|
tile_manifest_path=str(_manifest(tmp_path)),
|
||||||
yolo_adapter_class=FailingLoadAdapter,
|
yolo_adapter_class=FailingLoadAdapter,
|
||||||
check_model_load=True,
|
check_model_load=True,
|
||||||
@@ -173,6 +221,7 @@ def test_yolo_preflight_script_outputs_json(tmp_path: Path) -> None:
|
|||||||
model_path = tmp_path / "model.pt"
|
model_path = tmp_path / "model.pt"
|
||||||
model_path.write_bytes(b"weights")
|
model_path.write_bytes(b"weights")
|
||||||
manifest_path = _manifest(tmp_path)
|
manifest_path = _manifest(tmp_path)
|
||||||
|
_write_model_sidecar(model_path, Settings(yolo_enabled=True, yolo_model_path=str(model_path)))
|
||||||
|
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
[
|
[
|
||||||
@@ -201,6 +250,7 @@ def test_yolo_preflight_script_uses_environment_configuration(tmp_path: Path, mo
|
|||||||
model_path = tmp_path / "model.pt"
|
model_path = tmp_path / "model.pt"
|
||||||
model_path.write_bytes(b"weights")
|
model_path.write_bytes(b"weights")
|
||||||
manifest_path = _manifest(tmp_path)
|
manifest_path = _manifest(tmp_path)
|
||||||
|
_write_model_sidecar(model_path, Settings(yolo_enabled=True, yolo_model_path=str(model_path), yolo_max_tiles=4))
|
||||||
monkeypatch.setenv("YOLO_ENABLED", "true")
|
monkeypatch.setenv("YOLO_ENABLED", "true")
|
||||||
monkeypatch.setenv("YOLO_MODEL_PATH", str(model_path))
|
monkeypatch.setenv("YOLO_MODEL_PATH", str(model_path))
|
||||||
monkeypatch.setenv("YOLO_MAX_TILES", "4")
|
monkeypatch.setenv("YOLO_MAX_TILES", "4")
|
||||||
@@ -227,6 +277,21 @@ def test_yolo_preflight_script_uses_environment_configuration(tmp_path: Path, mo
|
|||||||
assert payload["max_tiles"] == 4
|
assert payload["max_tiles"] == 4
|
||||||
|
|
||||||
|
|
||||||
|
def test_yolo_preflight_refuses_unmanifested_local_weights(tmp_path: Path) -> None:
|
||||||
|
model_path = tmp_path / "model.pt"
|
||||||
|
model_path.write_bytes(b"unmanifested weights")
|
||||||
|
|
||||||
|
result = YoloPreflightService.run(
|
||||||
|
settings=Settings(yolo_enabled=True, yolo_model_path=str(model_path)),
|
||||||
|
tile_manifest_path=str(_manifest(tmp_path)),
|
||||||
|
yolo_adapter_class=AvailableAdapter,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["status"] == "contract_incomplete"
|
||||||
|
assert result["checks"]["model_provenance_valid"] is False
|
||||||
|
assert result["error_code"] == "MODEL_PROVENANCE_MANIFEST_MISSING"
|
||||||
|
|
||||||
|
|
||||||
def test_yolo_preflight_script_rejects_assumed_dependencies_for_model_load(tmp_path: Path) -> None:
|
def test_yolo_preflight_script_rejects_assumed_dependencies_for_model_load(tmp_path: Path) -> None:
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
[
|
[
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from fastapi.testclient import TestClient
|
|||||||
|
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.main import app
|
from app.main import app
|
||||||
from app.models import Area, Dataset, Export, Project, QualityCheck
|
from app.models import Area, Dataset, Export, Project, QualityCheck, SourceRegistry, SourceSnapshot
|
||||||
from app.schemas.export import ExportCreateResponse
|
from app.schemas.export import ExportCreateResponse
|
||||||
from app.services.export_service import ExportService
|
from app.services.export_service import ExportService
|
||||||
from app.services.storage_service import StorageService
|
from app.services.storage_service import StorageService
|
||||||
@@ -66,13 +66,57 @@ class FakeSession:
|
|||||||
return row
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def _govern_fixture_dataset(dataset: Dataset) -> Dataset:
|
||||||
|
"""Give an export fixture a governed authoritative source identity.
|
||||||
|
|
||||||
|
Export is a production boundary: test data must model a source that could
|
||||||
|
cross it, rather than using the deliberately QA-only ``fixture`` source.
|
||||||
|
"""
|
||||||
|
|
||||||
|
source_id = uuid4()
|
||||||
|
snapshot_id = uuid4()
|
||||||
|
checksum = "a" * 64
|
||||||
|
source = SourceRegistry(
|
||||||
|
id=source_id,
|
||||||
|
source_key="grb",
|
||||||
|
display_name="GRB export test source",
|
||||||
|
classification="authoritative",
|
||||||
|
authority_name="Digitaal Vlaanderen",
|
||||||
|
authority_scope_json={"zone": "Flanders"},
|
||||||
|
usage_policy_json={"ground_truth_allowed": True},
|
||||||
|
)
|
||||||
|
snapshot = SourceSnapshot(
|
||||||
|
id=snapshot_id,
|
||||||
|
source_registry_id=source_id,
|
||||||
|
snapshot_key=f"export-grb-{dataset.id}",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
ingest_status="ingested",
|
||||||
|
freshness_status="current",
|
||||||
|
)
|
||||||
|
dataset.source = "grb"
|
||||||
|
dataset.source_name = "grb"
|
||||||
|
dataset.checksum_sha256 = checksum
|
||||||
|
dataset.source_registry_id = source_id
|
||||||
|
dataset.source_snapshot_id = snapshot_id
|
||||||
|
dataset.data_contract_key = "geointel.vector.geojson"
|
||||||
|
dataset.data_contract_version = "1.0.0"
|
||||||
|
dataset.validation_status = "passed"
|
||||||
|
dataset.provenance_status = "complete"
|
||||||
|
dataset.lineage_status = "not_applicable"
|
||||||
|
dataset.quarantine_status = "not_quarantined"
|
||||||
|
dataset.status = "ready"
|
||||||
|
dataset.source_registry = source
|
||||||
|
dataset.source_snapshot = snapshot
|
||||||
|
return dataset
|
||||||
|
|
||||||
|
|
||||||
def test_dataset_geojson_export_persists_export_and_writes_artifact(tmp_path, monkeypatch) -> None:
|
def test_dataset_geojson_export_persists_export_and_writes_artifact(tmp_path, monkeypatch) -> None:
|
||||||
project_id = uuid4()
|
project_id = uuid4()
|
||||||
dataset_id = uuid4()
|
dataset_id = uuid4()
|
||||||
dataset_path = tmp_path / "input.geojson"
|
dataset_path = tmp_path / "input.geojson"
|
||||||
dataset_path.write_text(json.dumps({"type": "FeatureCollection", "features": []}), encoding="utf-8")
|
dataset_path.write_text(json.dumps({"type": "FeatureCollection", "features": []}), encoding="utf-8")
|
||||||
export_path = tmp_path / "exports" / "buildings.geojson"
|
export_path = tmp_path / "exports" / "buildings.geojson"
|
||||||
dataset = Dataset(
|
dataset = _govern_fixture_dataset(Dataset(
|
||||||
id=dataset_id,
|
id=dataset_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name="buildings.geojson",
|
name="buildings.geojson",
|
||||||
@@ -80,7 +124,7 @@ def test_dataset_geojson_export_persists_export_and_writes_artifact(tmp_path, mo
|
|||||||
source="fixture",
|
source="fixture",
|
||||||
storage_path=str(dataset_path),
|
storage_path=str(dataset_path),
|
||||||
status="ready",
|
status="ready",
|
||||||
)
|
))
|
||||||
db = FakeSession({(Dataset, dataset_id): dataset})
|
db = FakeSession({(Dataset, dataset_id): dataset})
|
||||||
monkeypatch.setattr(StorageService, "dataset_export_path", lambda *_args: str(export_path))
|
monkeypatch.setattr(StorageService, "dataset_export_path", lambda *_args: str(export_path))
|
||||||
|
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ from app.core.config import Settings
|
|||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.db.session import get_db
|
from app.db.session import get_db
|
||||||
from app.main import app
|
from app.main import app
|
||||||
from app.models import Area, Dataset, DatasetVersion, Job, Project
|
from app.models import Area, Dataset, DatasetVersion, Job, Project, SourceRegistry, SourceSnapshot
|
||||||
from app.schemas.orthophoto import OrthophotoAcquireRequest
|
from app.schemas.orthophoto import OrthophotoAcquireRequest
|
||||||
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
|
from app.services.orthophoto_acquisition_service import OrthophotoAcquisitionService
|
||||||
|
|
||||||
@@ -41,6 +41,15 @@ class FakeSession:
|
|||||||
def add(self, row):
|
def add(self, row):
|
||||||
self.added.append(row)
|
self.added.append(row)
|
||||||
|
|
||||||
|
def flush(self):
|
||||||
|
# The governed importer persists source identities and immutable
|
||||||
|
# snapshots before the Dataset. Mirror the database-generated UUIDs
|
||||||
|
# so this harness exercises that Phase 2 path rather than the legacy
|
||||||
|
# no-registry fallback.
|
||||||
|
for row in self.added:
|
||||||
|
if getattr(row, "id", None) is None:
|
||||||
|
row.id = uuid4()
|
||||||
|
|
||||||
def commit(self):
|
def commit(self):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -50,13 +59,23 @@ class FakeSession:
|
|||||||
def refresh(self, row):
|
def refresh(self, row):
|
||||||
return row
|
return row
|
||||||
|
|
||||||
def query(self, _model):
|
def query(self, model):
|
||||||
return FakeQuery(self.query_result)
|
rows = [
|
||||||
|
row
|
||||||
|
for (row_model, _row_id), row in self.rows.items()
|
||||||
|
if row_model is model and isinstance(row, model)
|
||||||
|
]
|
||||||
|
rows.extend(row for row in self.added if isinstance(row, model))
|
||||||
|
if isinstance(self.query_result, model):
|
||||||
|
rows.append(self.query_result)
|
||||||
|
elif isinstance(self.query_result, list):
|
||||||
|
rows.extend(row for row in self.query_result if isinstance(row, model))
|
||||||
|
return FakeQuery(rows)
|
||||||
|
|
||||||
|
|
||||||
class FakeQuery:
|
class FakeQuery:
|
||||||
def __init__(self, result):
|
def __init__(self, results):
|
||||||
self.result = result
|
self.results = list(results)
|
||||||
|
|
||||||
def filter(self, *_args):
|
def filter(self, *_args):
|
||||||
return self
|
return self
|
||||||
@@ -65,7 +84,10 @@ class FakeQuery:
|
|||||||
return self
|
return self
|
||||||
|
|
||||||
def first(self):
|
def first(self):
|
||||||
return self.result
|
return self.results[0] if self.results else None
|
||||||
|
|
||||||
|
def one_or_none(self):
|
||||||
|
return self.first()
|
||||||
|
|
||||||
|
|
||||||
class FakeImageResponse:
|
class FakeImageResponse:
|
||||||
@@ -222,12 +244,23 @@ def test_regional_orthophoto_products_bind_provider_and_governed_scope(
|
|||||||
)
|
)
|
||||||
|
|
||||||
dataset = next(row for row in db.added if isinstance(row, Dataset))
|
dataset = next(row for row in db.added if isinstance(row, Dataset))
|
||||||
|
source = next(row for row in db.added if isinstance(row, SourceRegistry))
|
||||||
|
snapshot = next(row for row in db.added if isinstance(row, SourceSnapshot))
|
||||||
assert result["provider"] == provider
|
assert result["provider"] == provider
|
||||||
assert result["layer"] == layer
|
assert result["layer"] == layer
|
||||||
assert dataset.source_name == provider
|
assert dataset.source_name == provider
|
||||||
assert dataset.source_metadata["coverage_zone"] == coverage_zone
|
assert dataset.source_metadata["coverage_zone"] == coverage_zone
|
||||||
assert dataset.source_metadata["license_note"]
|
assert dataset.source_metadata["license_note"]
|
||||||
assert dataset.provenance_metadata["request_url"].startswith(prepared["product"].wms_url)
|
assert dataset.provenance_metadata["request_url"].startswith(prepared["product"].wms_url)
|
||||||
|
assert dataset.source_registry_id == source.id
|
||||||
|
assert dataset.source_snapshot_id == snapshot.id
|
||||||
|
assert dataset.validation_status == "passed"
|
||||||
|
assert dataset.provenance_status == "complete"
|
||||||
|
assert dataset.quarantine_status == "not_quarantined"
|
||||||
|
assert snapshot.source_registry_id == source.id
|
||||||
|
assert snapshot.checksum_sha256 == dataset.checksum_sha256
|
||||||
|
assert snapshot.ingest_status == "ingested"
|
||||||
|
assert snapshot.freshness_status == "current"
|
||||||
|
|
||||||
prepared = OrthophotoAcquisitionService._prepared_request(_selection_payload(product_key="1971"), settings)
|
prepared = OrthophotoAcquisitionService._prepared_request(_selection_payload(product_key="1971"), settings)
|
||||||
assert prepared["params"]["LAYERS"] == "OKZPAN71VL"
|
assert prepared["params"]["LAYERS"] == "OKZPAN71VL"
|
||||||
@@ -291,6 +324,8 @@ def test_orthophoto_acquisition_persists_georeferenced_raster_and_provenance(tmp
|
|||||||
assert len(datasets) == 1
|
assert len(datasets) == 1
|
||||||
assert len(versions) == 1
|
assert len(versions) == 1
|
||||||
dataset = datasets[0]
|
dataset = datasets[0]
|
||||||
|
source = next(row for row in db.added if isinstance(row, SourceRegistry))
|
||||||
|
snapshot = next(row for row in db.added if isinstance(row, SourceSnapshot))
|
||||||
assert result["output_dataset_id"] == str(dataset.id)
|
assert result["output_dataset_id"] == str(dataset.id)
|
||||||
assert result["reused"] is False
|
assert result["reused"] is False
|
||||||
assert dataset.project_id == project_id
|
assert dataset.project_id == project_id
|
||||||
@@ -301,6 +336,15 @@ def test_orthophoto_acquisition_persists_georeferenced_raster_and_provenance(tmp
|
|||||||
assert dataset.crs == "EPSG:31370"
|
assert dataset.crs == "EPSG:31370"
|
||||||
assert dataset.provenance_metadata["acquisition"] == "explicit_bounded_map_selection"
|
assert dataset.provenance_metadata["acquisition"] == "explicit_bounded_map_selection"
|
||||||
assert dataset.provenance_metadata["request_hash"] == prepared["request_hash"]
|
assert dataset.provenance_metadata["request_hash"] == prepared["request_hash"]
|
||||||
|
assert dataset.source_registry_id == source.id
|
||||||
|
assert dataset.source_snapshot_id == snapshot.id
|
||||||
|
assert dataset.validation_status == "passed"
|
||||||
|
assert dataset.provenance_status == "complete"
|
||||||
|
assert dataset.lineage_status == "not_applicable"
|
||||||
|
assert dataset.quarantine_status == "not_quarantined"
|
||||||
|
assert snapshot.source_registry_id == source.id
|
||||||
|
assert snapshot.checksum_sha256 == dataset.checksum_sha256
|
||||||
|
assert snapshot.freshness_status == "current"
|
||||||
assert dataset.source_metadata["attribution"].startswith("Bron: Orthofotomozaiek Vlaanderen")
|
assert dataset.source_metadata["attribution"].startswith("Bron: Orthofotomozaiek Vlaanderen")
|
||||||
assert dataset.storage_path is not None
|
assert dataset.storage_path is not None
|
||||||
with rasterio.open(dataset.storage_path) as stored:
|
with rasterio.open(dataset.storage_path) as stored:
|
||||||
|
|||||||
@@ -17,7 +17,7 @@ from app.core.config import Settings
|
|||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.db.session import get_db
|
from app.db.session import get_db
|
||||||
from app.main import app
|
from app.main import app
|
||||||
from app.models import Area, Dataset, DatasetVersion, Job, Project
|
from app.models import Area, Dataset, DatasetVersion, Job, Project, SourceRegistry, SourceSnapshot
|
||||||
from app.schemas.dhmv import DhmvAcquireRequest, TerrainPartitionSelectionRequest, TerrainSelectionRequest
|
from app.schemas.dhmv import DhmvAcquireRequest, TerrainPartitionSelectionRequest, TerrainSelectionRequest
|
||||||
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
|
from app.services.dhmv_acquisition_service import DhmvAcquisitionService
|
||||||
from app.services.terrain_analysis_service import TerrainAnalysisService
|
from app.services.terrain_analysis_service import TerrainAnalysisService
|
||||||
@@ -27,8 +27,8 @@ ROOT = Path(__file__).resolve().parents[2]
|
|||||||
|
|
||||||
|
|
||||||
class FakeQuery:
|
class FakeQuery:
|
||||||
def __init__(self, result=None):
|
def __init__(self, results=None):
|
||||||
self.result = result
|
self.results = list(results or [])
|
||||||
|
|
||||||
def filter(self, *_args):
|
def filter(self, *_args):
|
||||||
return self
|
return self
|
||||||
@@ -37,10 +37,13 @@ class FakeQuery:
|
|||||||
return self
|
return self
|
||||||
|
|
||||||
def first(self):
|
def first(self):
|
||||||
return self.result
|
return self.results[0] if self.results else None
|
||||||
|
|
||||||
|
def one_or_none(self):
|
||||||
|
return self.first()
|
||||||
|
|
||||||
def all(self):
|
def all(self):
|
||||||
return self.result if isinstance(self.result, list) else []
|
return list(self.results)
|
||||||
|
|
||||||
|
|
||||||
class FakeSession:
|
class FakeSession:
|
||||||
@@ -58,6 +61,14 @@ class FakeSession:
|
|||||||
def add(self, row):
|
def add(self, row):
|
||||||
self.added.append(row)
|
self.added.append(row)
|
||||||
|
|
||||||
|
def flush(self):
|
||||||
|
# Exercise the governed source/snapshot import path with database-like
|
||||||
|
# primary-key assignment instead of silently falling back to legacy
|
||||||
|
# fixture behavior.
|
||||||
|
for row in self.added:
|
||||||
|
if getattr(row, "id", None) is None:
|
||||||
|
row.id = uuid4()
|
||||||
|
|
||||||
def commit(self):
|
def commit(self):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -67,8 +78,18 @@ class FakeSession:
|
|||||||
def refresh(self, row):
|
def refresh(self, row):
|
||||||
return row
|
return row
|
||||||
|
|
||||||
def query(self, _model):
|
def query(self, model):
|
||||||
return FakeQuery(self.query_result)
|
rows = [
|
||||||
|
row
|
||||||
|
for (row_model, _row_id), row in self.rows.items()
|
||||||
|
if row_model is model and isinstance(row, model)
|
||||||
|
]
|
||||||
|
rows.extend(row for row in self.added if isinstance(row, model))
|
||||||
|
if isinstance(self.query_result, model):
|
||||||
|
rows.append(self.query_result)
|
||||||
|
elif isinstance(self.query_result, list):
|
||||||
|
rows.extend(row for row in self.query_result if isinstance(row, model))
|
||||||
|
return FakeQuery(rows)
|
||||||
|
|
||||||
|
|
||||||
class FakeResponse:
|
class FakeResponse:
|
||||||
@@ -318,6 +339,8 @@ def test_dhmv_acquisition_clips_validates_and_persists_via_dataset_service(tmp_p
|
|||||||
|
|
||||||
dataset = next(item for item in db.added if isinstance(item, Dataset))
|
dataset = next(item for item in db.added if isinstance(item, Dataset))
|
||||||
version = next(item for item in db.added if isinstance(item, DatasetVersion))
|
version = next(item for item in db.added if isinstance(item, DatasetVersion))
|
||||||
|
source = next(item for item in db.added if isinstance(item, SourceRegistry))
|
||||||
|
snapshot = next(item for item in db.added if isinstance(item, SourceSnapshot))
|
||||||
assert result["output_dataset_id"] == str(dataset.id)
|
assert result["output_dataset_id"] == str(dataset.id)
|
||||||
assert dataset.source_name == "digitaal_vlaanderen_dhmv"
|
assert dataset.source_name == "digitaal_vlaanderen_dhmv"
|
||||||
assert dataset.area_id == area_id
|
assert dataset.area_id == area_id
|
||||||
@@ -331,6 +354,16 @@ def test_dhmv_acquisition_clips_validates_and_persists_via_dataset_service(tmp_p
|
|||||||
assert dataset.provenance_metadata["water_depth_available"] is False
|
assert dataset.provenance_metadata["water_depth_available"] is False
|
||||||
assert dataset.provenance_metadata["water_volume_available"] is False
|
assert dataset.provenance_metadata["water_volume_available"] is False
|
||||||
assert len(dataset.provenance_metadata["response_sha256"]) == 64
|
assert len(dataset.provenance_metadata["response_sha256"]) == 64
|
||||||
|
assert dataset.source_registry_id == source.id
|
||||||
|
assert dataset.source_snapshot_id == snapshot.id
|
||||||
|
assert dataset.validation_status == "passed"
|
||||||
|
assert dataset.provenance_status == "complete"
|
||||||
|
assert dataset.lineage_status == "not_applicable"
|
||||||
|
assert dataset.quarantine_status == "not_quarantined"
|
||||||
|
assert snapshot.source_registry_id == source.id
|
||||||
|
assert snapshot.checksum_sha256 == dataset.checksum_sha256
|
||||||
|
assert snapshot.ingest_status == "ingested"
|
||||||
|
assert snapshot.freshness_status == "current"
|
||||||
with rasterio.open(dataset.storage_path) as stored:
|
with rasterio.open(dataset.storage_path) as stored:
|
||||||
assert stored.crs.to_epsg() == 31370
|
assert stored.crs.to_epsg() == 31370
|
||||||
assert stored.count == 1
|
assert stored.count == 1
|
||||||
|
|||||||
@@ -1,8 +1,9 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import json
|
import json
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from hashlib import sha256
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from types import SimpleNamespace
|
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@@ -10,12 +11,13 @@ from fastapi.testclient import TestClient
|
|||||||
from pydantic import ValidationError
|
from pydantic import ValidationError
|
||||||
|
|
||||||
from app.main import app
|
from app.main import app
|
||||||
from app.models import Dataset, Export
|
from app.models import Dataset, Export, SourceRegistry, SourceSnapshot
|
||||||
from app.schemas.export import ExportCreateResponse, MapResultExportRequest
|
from app.schemas.export import ExportCreateResponse, MapResultExportRequest
|
||||||
from app.schemas.project import ProjectRead
|
from app.schemas.project import ProjectRead
|
||||||
from app.services.export_service import ExportService
|
from app.services.export_service import ExportService
|
||||||
from app.services.project_service import ProjectService
|
from app.services.project_service import ProjectService
|
||||||
from app.services.storage_service import StorageService
|
from app.services.storage_service import StorageService
|
||||||
|
from app.services.source_registry_service import SourceRegistryService
|
||||||
from app.services.temporal_analysis_service import TemporalAnalysisService
|
from app.services.temporal_analysis_service import TemporalAnalysisService
|
||||||
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
|
from app.services.thematic_raster_analysis_service import ThematicRasterAnalysisService
|
||||||
|
|
||||||
@@ -48,6 +50,71 @@ def bbox_payload() -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def governed_dataset(
|
||||||
|
*,
|
||||||
|
project_id,
|
||||||
|
dataset_id,
|
||||||
|
name: str,
|
||||||
|
dataset_type: str,
|
||||||
|
source_key: str,
|
||||||
|
dataset_role: str = "source",
|
||||||
|
) -> Dataset:
|
||||||
|
"""Build an in-memory stand-in for a passed governed dataset.
|
||||||
|
|
||||||
|
Map-result export is an operational consumption boundary. These tests
|
||||||
|
must therefore model the same source registry/snapshot, checksum and
|
||||||
|
passed-contract evidence supplied by a real adapter rather than relying
|
||||||
|
on an old transient Dataset fixture.
|
||||||
|
"""
|
||||||
|
|
||||||
|
source = SourceRegistry(
|
||||||
|
id=uuid4(),
|
||||||
|
**SourceRegistryService.definition_for(source_key).as_model_values(),
|
||||||
|
)
|
||||||
|
checksum = sha256(f"{dataset_id}:{source_key}:{dataset_type}".encode("utf-8")).hexdigest()
|
||||||
|
snapshot = SourceSnapshot(
|
||||||
|
id=uuid4(),
|
||||||
|
source_registry_id=source.id,
|
||||||
|
snapshot_key=f"test:{source_key}:{checksum}",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
fetched_at=datetime.now(UTC),
|
||||||
|
crs="EPSG:31370",
|
||||||
|
units=source.default_units,
|
||||||
|
spatial_resolution_json={"x": 1.0, "y": 1.0, "unit": "m"},
|
||||||
|
temporal_coverage_json={"status": "test-fixture"},
|
||||||
|
geographic_coverage_json={"zone": "Flanders"},
|
||||||
|
observed_schema_json={"dataset_type": dataset_type},
|
||||||
|
freshness_status="current",
|
||||||
|
ingest_status="ingested",
|
||||||
|
known_limitations_json=["In-memory governed fixture used only by this export test."],
|
||||||
|
snapshot_metadata_json={"fixture_mode": True},
|
||||||
|
)
|
||||||
|
return Dataset(
|
||||||
|
id=dataset_id,
|
||||||
|
project_id=project_id,
|
||||||
|
name=name,
|
||||||
|
dataset_type=dataset_type,
|
||||||
|
source="governed test fixture",
|
||||||
|
dataset_role=dataset_role,
|
||||||
|
source_name=source.source_key,
|
||||||
|
source_registry_id=source.id,
|
||||||
|
source_snapshot_id=snapshot.id,
|
||||||
|
source_registry=source,
|
||||||
|
source_snapshot=snapshot,
|
||||||
|
data_contract_key=("geointel.raster.geotiff" if dataset_type == "raster" else "geointel.vector.geojson"),
|
||||||
|
data_contract_version="1.0.0",
|
||||||
|
validation_status="passed",
|
||||||
|
provenance_status="complete",
|
||||||
|
lineage_status="not_applicable",
|
||||||
|
quarantine_status="not_quarantined",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
metadata_json={"fixture_mode": True},
|
||||||
|
source_metadata={"fixture_mode": True, "source_registry_key": source.source_key},
|
||||||
|
provenance_metadata={"fixture_mode": True, "source_snapshot_id": str(snapshot.id)},
|
||||||
|
status="ready",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_map_result_export_request_requires_a_complete_target() -> None:
|
def test_map_result_export_request_requires_a_complete_target() -> None:
|
||||||
with pytest.raises(ValidationError):
|
with pytest.raises(ValidationError):
|
||||||
MapResultExportRequest(project_id=uuid4(), mode="current", bbox=bbox_payload())
|
MapResultExportRequest(project_id=uuid4(), mode="current", bbox=bbox_payload())
|
||||||
@@ -67,13 +134,12 @@ def test_current_vector_map_result_uses_authoritative_selection_export(monkeypat
|
|||||||
project_id = uuid4()
|
project_id = uuid4()
|
||||||
dataset_id = uuid4()
|
dataset_id = uuid4()
|
||||||
area_id = uuid4()
|
area_id = uuid4()
|
||||||
dataset = Dataset(
|
dataset = governed_dataset(
|
||||||
id=dataset_id,
|
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
|
dataset_id=dataset_id,
|
||||||
name="buildings.geojson",
|
name="buildings.geojson",
|
||||||
dataset_type="vector",
|
dataset_type="vector",
|
||||||
source="fixture",
|
source_key="grb",
|
||||||
status="ready",
|
|
||||||
)
|
)
|
||||||
db = FakeSession({(Dataset, dataset_id): dataset})
|
db = FakeSession({(Dataset, dataset_id): dataset})
|
||||||
expected = ExportCreateResponse(
|
expected = ExportCreateResponse(
|
||||||
@@ -109,14 +175,12 @@ def test_current_vector_map_result_uses_authoritative_selection_export(monkeypat
|
|||||||
def test_partitioned_vector_map_result_uses_governed_partition_export(monkeypatch) -> None:
|
def test_partitioned_vector_map_result_uses_governed_partition_export(monkeypatch) -> None:
|
||||||
project_id = uuid4()
|
project_id = uuid4()
|
||||||
dataset_id = uuid4()
|
dataset_id = uuid4()
|
||||||
dataset = Dataset(
|
dataset = governed_dataset(
|
||||||
id=dataset_id,
|
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
|
dataset_id=dataset_id,
|
||||||
name="vha-municipality.geojson",
|
name="vha-municipality.geojson",
|
||||||
dataset_type="vector",
|
dataset_type="vector",
|
||||||
source="VHA",
|
source_key="vmm_vha_bathymetry_profiles",
|
||||||
source_name="vmm_vha_bathymetry_profiles",
|
|
||||||
status="ready",
|
|
||||||
)
|
)
|
||||||
db = FakeSession({(Dataset, dataset_id): dataset})
|
db = FakeSession({(Dataset, dataset_id): dataset})
|
||||||
expected = ExportCreateResponse(
|
expected = ExportCreateResponse(
|
||||||
@@ -159,14 +223,12 @@ def test_partitioned_vector_map_result_uses_governed_partition_export(monkeypatc
|
|||||||
def test_raster_map_result_is_recomputed_and_persisted(tmp_path, monkeypatch) -> None:
|
def test_raster_map_result_is_recomputed_and_persisted(tmp_path, monkeypatch) -> None:
|
||||||
project_id = uuid4()
|
project_id = uuid4()
|
||||||
dataset_id = uuid4()
|
dataset_id = uuid4()
|
||||||
dataset = Dataset(
|
dataset = governed_dataset(
|
||||||
id=dataset_id,
|
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
|
dataset_id=dataset_id,
|
||||||
name="space-occupation.tif",
|
name="space-occupation.tif",
|
||||||
dataset_type="raster",
|
dataset_type="raster",
|
||||||
source="official",
|
source_key="department_omgeving_thematic_raster",
|
||||||
source_name="department_omgeving_thematic_raster",
|
|
||||||
status="ready",
|
|
||||||
)
|
)
|
||||||
db = FakeSession({(Dataset, dataset_id): dataset})
|
db = FakeSession({(Dataset, dataset_id): dataset})
|
||||||
export_path = tmp_path / "space-occupation-analysis.json"
|
export_path = tmp_path / "space-occupation-analysis.json"
|
||||||
@@ -209,7 +271,26 @@ def test_evolution_map_result_is_recomputed_and_persisted(tmp_path, monkeypatch)
|
|||||||
project_id = uuid4()
|
project_id = uuid4()
|
||||||
earlier_id = uuid4()
|
earlier_id = uuid4()
|
||||||
later_id = uuid4()
|
later_id = uuid4()
|
||||||
db = FakeSession()
|
earlier_dataset = governed_dataset(
|
||||||
|
project_id=project_id,
|
||||||
|
dataset_id=earlier_id,
|
||||||
|
name="forest-earlier.geojson",
|
||||||
|
dataset_type="vector",
|
||||||
|
source_key="inbo_bwk_natura2000",
|
||||||
|
)
|
||||||
|
later_dataset = governed_dataset(
|
||||||
|
project_id=project_id,
|
||||||
|
dataset_id=later_id,
|
||||||
|
name="forest-later.geojson",
|
||||||
|
dataset_type="vector",
|
||||||
|
source_key="inbo_bwk_natura2000",
|
||||||
|
)
|
||||||
|
db = FakeSession(
|
||||||
|
{
|
||||||
|
(Dataset, earlier_id): earlier_dataset,
|
||||||
|
(Dataset, later_id): later_dataset,
|
||||||
|
}
|
||||||
|
)
|
||||||
export_path = tmp_path / "forest-evolution.json"
|
export_path = tmp_path / "forest-evolution.json"
|
||||||
captured: dict = {}
|
captured: dict = {}
|
||||||
|
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ from uuid import uuid4
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from geoalchemy2.shape import to_shape
|
from geoalchemy2.shape import to_shape
|
||||||
|
from pyproj import Transformer
|
||||||
|
|
||||||
from app.api.routes.qa import compare_candidate_with_reference
|
from app.api.routes.qa import compare_candidate_with_reference
|
||||||
from app.models import Dataset, Metric, Project, QualityCheck, VectorFeature
|
from app.models import Dataset, Metric, Project, QualityCheck, VectorFeature
|
||||||
@@ -117,6 +118,54 @@ def test_vector_feature_service_normalizes_source_z_coordinates_to_canonical_2d(
|
|||||||
assert to_shape(persisted[0].geometry).has_z is False
|
assert to_shape(persisted[0].geometry).has_z is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_vector_feature_service_transforms_declared_source_crs_before_epsg4326_storage() -> None:
|
||||||
|
to_lambert = Transformer.from_crs("EPSG:4326", "EPSG:31370", always_xy=True)
|
||||||
|
x, y = to_lambert.transform(4.7, 51.1)
|
||||||
|
db = FakeSession()
|
||||||
|
|
||||||
|
persisted = VectorFeatureService.persist_geojson_features(
|
||||||
|
db=db,
|
||||||
|
dataset_id=uuid4(),
|
||||||
|
payload={
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"id": "lambert-point",
|
||||||
|
"properties": {},
|
||||||
|
"geometry": {"type": "Point", "coordinates": [x, y]},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
source_crs="EPSG:31370",
|
||||||
|
)
|
||||||
|
|
||||||
|
geometry = to_shape(persisted[0].geometry)
|
||||||
|
assert geometry.x == pytest.approx(4.7, abs=0.000001)
|
||||||
|
assert geometry.y == pytest.approx(51.1, abs=0.000001)
|
||||||
|
|
||||||
|
|
||||||
|
def test_vector_feature_service_rejects_invalid_declared_source_crs() -> None:
|
||||||
|
with pytest.raises(Exception) as exc_info:
|
||||||
|
VectorFeatureService.persist_geojson_features(
|
||||||
|
db=FakeSession(),
|
||||||
|
dataset_id=uuid4(),
|
||||||
|
payload={
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"properties": {},
|
||||||
|
"geometry": {"type": "Point", "coordinates": [4.7, 51.1]},
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
source_crs="EPSG:not-a-crs",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert getattr(exc_info.value, "code", None) == "INVALID_DATASET_CRS"
|
||||||
|
|
||||||
|
|
||||||
def test_dataset_upload_persists_vector_features(monkeypatch, tmp_path) -> None:
|
def test_dataset_upload_persists_vector_features(monkeypatch, tmp_path) -> None:
|
||||||
project_id = uuid4()
|
project_id = uuid4()
|
||||||
db = FakeSession(objects={(Project, project_id): Project(id=project_id, name="Geel")})
|
db = FakeSession(objects={(Project, project_id): Project(id=project_id, name="Geel")})
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ def _project_and_dataset(dataset_type: str = "raster"):
|
|||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name="source.tif",
|
name="source.tif",
|
||||||
dataset_type=dataset_type,
|
dataset_type=dataset_type,
|
||||||
source="user_upload",
|
source="test",
|
||||||
storage_path="storage/uploads/source.tif",
|
storage_path="storage/uploads/source.tif",
|
||||||
)
|
)
|
||||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from hashlib import sha256
|
||||||
import json
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import sys
|
import sys
|
||||||
@@ -10,10 +11,11 @@ import pytest
|
|||||||
|
|
||||||
from app.core.config import Settings
|
from app.core.config import Settings
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project
|
from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project, SourceRegistry, SourceSnapshot
|
||||||
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
|
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
|
||||||
from app.services.detection_service import DetectionService
|
from app.services.detection_service import DetectionService
|
||||||
from app.services.model_registry_service import ModelRegistryService
|
from app.services.model_registry_service import ModelRegistryService
|
||||||
|
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
|
||||||
from app.services.yolo_adapter import YoloDetectionAdapter
|
from app.services.yolo_adapter import YoloDetectionAdapter
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[2]
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
@@ -79,6 +81,14 @@ class MockYoloAdapter:
|
|||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
class NeverLoadUnboundModelAdapter(MockYoloAdapter):
|
||||||
|
load_calls = 0
|
||||||
|
|
||||||
|
def load_model(self, model_path: Path):
|
||||||
|
type(self).load_calls += 1
|
||||||
|
raise AssertionError("unbound model provenance must be rejected before adapter.load_model")
|
||||||
|
|
||||||
|
|
||||||
class MixedCaseYoloAdapter(MockYoloAdapter):
|
class MixedCaseYoloAdapter(MockYoloAdapter):
|
||||||
def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict]:
|
def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict]:
|
||||||
return [
|
return [
|
||||||
@@ -141,15 +151,47 @@ class ExplodingPredictModel:
|
|||||||
def _project_and_dataset(dataset_type: str = "raster"):
|
def _project_and_dataset(dataset_type: str = "raster"):
|
||||||
project_id = uuid4()
|
project_id = uuid4()
|
||||||
dataset_id = uuid4()
|
dataset_id = uuid4()
|
||||||
|
source_registry_id = uuid4()
|
||||||
|
source_snapshot_id = uuid4()
|
||||||
|
checksum = "a" * 64
|
||||||
project = Project(id=project_id, name="Geel")
|
project = Project(id=project_id, name="Geel")
|
||||||
|
source_registry = SourceRegistry(
|
||||||
|
id=source_registry_id,
|
||||||
|
source_key="test-derived-raster",
|
||||||
|
display_name="Governed test-derived raster",
|
||||||
|
classification="derived",
|
||||||
|
authority_name="GeoIntel test fixture",
|
||||||
|
usage_policy_json={"ground_truth_allowed": False},
|
||||||
|
)
|
||||||
|
source_snapshot = SourceSnapshot(
|
||||||
|
id=source_snapshot_id,
|
||||||
|
source_registry_id=source_registry_id,
|
||||||
|
snapshot_key="test-derived-raster-v1",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
freshness_status="current",
|
||||||
|
ingest_status="ingested",
|
||||||
|
)
|
||||||
dataset = Dataset(
|
dataset = Dataset(
|
||||||
id=dataset_id,
|
id=dataset_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name="source.tif",
|
name="source.tif",
|
||||||
dataset_type=dataset_type,
|
dataset_type=dataset_type,
|
||||||
source="user_upload",
|
source="test-derived-raster",
|
||||||
|
source_name="test-derived-raster",
|
||||||
storage_path="storage/uploads/source.tif",
|
storage_path="storage/uploads/source.tif",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
source_registry_id=source_registry_id,
|
||||||
|
source_snapshot_id=source_snapshot_id,
|
||||||
|
data_contract_key="geointel.raster.geotiff",
|
||||||
|
data_contract_version="1.0.0",
|
||||||
|
validation_status="passed",
|
||||||
|
provenance_status="complete",
|
||||||
|
lineage_status="not_applicable",
|
||||||
|
quarantine_status="not_quarantined",
|
||||||
|
status="ready",
|
||||||
)
|
)
|
||||||
|
dataset.source_registry = source_registry
|
||||||
|
dataset.source_snapshot = source_snapshot
|
||||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||||
return db, project_id, dataset_id
|
return db, project_id, dataset_id
|
||||||
|
|
||||||
@@ -165,6 +207,74 @@ def _settings(tmp_path: Path, **overrides) -> Settings:
|
|||||||
return Settings(**values)
|
return Settings(**values)
|
||||||
|
|
||||||
|
|
||||||
|
def _write_model_sidecar(
|
||||||
|
model_path: Path,
|
||||||
|
settings: Settings,
|
||||||
|
*,
|
||||||
|
db: FakeSession | None = None,
|
||||||
|
) -> None:
|
||||||
|
"""Create explicit test-only evidence; production never self-generates it."""
|
||||||
|
|
||||||
|
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
|
||||||
|
source_registry_id = uuid4()
|
||||||
|
source_snapshot_id = uuid4()
|
||||||
|
source_version = settings.yolo_model_version or "test-v1"
|
||||||
|
if db is not None:
|
||||||
|
source_registry = SourceRegistry(
|
||||||
|
id=source_registry_id,
|
||||||
|
source_key="model",
|
||||||
|
display_name="Governed test model artifact",
|
||||||
|
classification="experimental",
|
||||||
|
authority_name="GeoIntel test fixture",
|
||||||
|
freshness_status="current",
|
||||||
|
ingest_status="configured",
|
||||||
|
)
|
||||||
|
source_snapshot = SourceSnapshot(
|
||||||
|
id=source_snapshot_id,
|
||||||
|
source_registry_id=source_registry_id,
|
||||||
|
snapshot_key=f"model-{source_version}",
|
||||||
|
source_version=source_version,
|
||||||
|
checksum_sha256=model_sha256,
|
||||||
|
freshness_status="current",
|
||||||
|
ingest_status="ingested",
|
||||||
|
)
|
||||||
|
db.objects[(SourceRegistry, source_registry_id)] = source_registry
|
||||||
|
db.objects[(SourceSnapshot, source_snapshot_id)] = source_snapshot
|
||||||
|
payload = {
|
||||||
|
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
|
||||||
|
"data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"},
|
||||||
|
"model": {
|
||||||
|
"model_id": settings.yolo_model_id,
|
||||||
|
"task_type": "object_detection",
|
||||||
|
"sha256": model_sha256,
|
||||||
|
"model_format": "pytorch",
|
||||||
|
"framework": "ultralytics/pytorch",
|
||||||
|
"class_mapping": {"0": "building"},
|
||||||
|
"source_version": source_version,
|
||||||
|
},
|
||||||
|
"source": {
|
||||||
|
"source_registry_id": str(source_registry_id),
|
||||||
|
"source_snapshot_id": str(source_snapshot_id),
|
||||||
|
"source_registry_key": "model",
|
||||||
|
"source_snapshot_checksum_sha256": model_sha256,
|
||||||
|
},
|
||||||
|
"lineage": {
|
||||||
|
"upstream_asset_ids": ["test-training-corpus"],
|
||||||
|
"upstream_checksums_sha256": ["a" * 64],
|
||||||
|
"transformations": [
|
||||||
|
{"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"metadata": {"training_manifest_sha256": "c" * 64},
|
||||||
|
"imported_at": "2026-08-01T10:00:00+00:00",
|
||||||
|
}
|
||||||
|
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
|
||||||
|
RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text(
|
||||||
|
json.dumps(payload, sort_keys=True),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def _manifest(tmp_path: Path, tile_count: int = 1) -> Path:
|
def _manifest(tmp_path: Path, tile_count: int = 1) -> Path:
|
||||||
tiles = []
|
tiles = []
|
||||||
for index in range(tile_count):
|
for index in range(tile_count):
|
||||||
@@ -223,10 +333,28 @@ def test_yolo_configured_model_reports_dependency_unavailable(tmp_path: Path) ->
|
|||||||
assert model.status == "dependency_unavailable"
|
assert model.status == "dependency_unavailable"
|
||||||
|
|
||||||
|
|
||||||
|
def test_yolo_configured_model_requires_a_runtime_provenance_sidecar(tmp_path: Path) -> None:
|
||||||
|
model_path = tmp_path / "model.pt"
|
||||||
|
model_path.write_bytes(b"unmanifested local weights")
|
||||||
|
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
||||||
|
|
||||||
|
model = ModelRegistryService.get_model_capability(
|
||||||
|
"yolo-configured",
|
||||||
|
settings=settings,
|
||||||
|
yolo_adapter_class=AvailableAdapter,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert model is not None
|
||||||
|
assert model.configured is False
|
||||||
|
assert model.status == "contract_incomplete"
|
||||||
|
assert "sidecar" in model.limitation_message
|
||||||
|
|
||||||
|
|
||||||
def test_yolo_configured_model_reports_configured_with_local_model_and_dependencies(tmp_path: Path) -> None:
|
def test_yolo_configured_model_reports_configured_with_local_model_and_dependencies(tmp_path: Path) -> None:
|
||||||
model_path = tmp_path / "model.pt"
|
model_path = tmp_path / "model.pt"
|
||||||
model_path.write_bytes(b"local weights")
|
model_path.write_bytes(b"local weights")
|
||||||
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
||||||
|
_write_model_sidecar(model_path, settings)
|
||||||
|
|
||||||
model = ModelRegistryService.get_model_capability("yolo-configured", settings=settings, yolo_adapter_class=AvailableAdapter)
|
model = ModelRegistryService.get_model_capability("yolo-configured", settings=settings, yolo_adapter_class=AvailableAdapter)
|
||||||
|
|
||||||
@@ -306,11 +434,37 @@ def test_yolo_run_requires_tile_manifest_path(tmp_path: Path) -> None:
|
|||||||
assert getattr(exc_info.value, "code", None) == "DETECTION_TILE_MANIFEST_REQUIRED"
|
assert getattr(exc_info.value, "code", None) == "DETECTION_TILE_MANIFEST_REQUIRED"
|
||||||
|
|
||||||
|
|
||||||
|
def test_yolo_run_fails_closed_before_adapter_load_without_sidecar(tmp_path: Path) -> None:
|
||||||
|
db, project_id, dataset_id = _project_and_dataset()
|
||||||
|
model_path = tmp_path / "model.pt"
|
||||||
|
model_path.write_bytes(b"unmanifested local weights")
|
||||||
|
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
||||||
|
|
||||||
|
# AvailableAdapter intentionally has no load_model method. If runtime
|
||||||
|
# provenance were checked after adapter loading, this would raise instead
|
||||||
|
# of returning the explicit unavailable capability state.
|
||||||
|
result = DetectionService.run_detection(
|
||||||
|
db=db,
|
||||||
|
project_id=project_id,
|
||||||
|
dataset_id=dataset_id,
|
||||||
|
model_id="yolo-configured",
|
||||||
|
confidence_threshold=0.5,
|
||||||
|
tile_manifest_path=str(_manifest(tmp_path)),
|
||||||
|
settings=settings,
|
||||||
|
yolo_adapter_class=AvailableAdapter,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.status == "failed"
|
||||||
|
assert result.error_code == "DETECTION_MODEL_UNAVAILABLE"
|
||||||
|
assert "sidecar" in result.message
|
||||||
|
|
||||||
|
|
||||||
def test_yolo_run_rejects_manifest_over_tile_limit(tmp_path: Path) -> None:
|
def test_yolo_run_rejects_manifest_over_tile_limit(tmp_path: Path) -> None:
|
||||||
db, project_id, dataset_id = _project_and_dataset()
|
db, project_id, dataset_id = _project_and_dataset()
|
||||||
model_path = tmp_path / "model.pt"
|
model_path = tmp_path / "model.pt"
|
||||||
model_path.write_bytes(b"local weights")
|
model_path.write_bytes(b"local weights")
|
||||||
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_max_tiles=1)
|
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_max_tiles=1)
|
||||||
|
_write_model_sidecar(model_path, settings, db=db)
|
||||||
manifest_path = _manifest(tmp_path, tile_count=2)
|
manifest_path = _manifest(tmp_path, tile_count=2)
|
||||||
|
|
||||||
result = DetectionService.run_detection(
|
result = DetectionService.run_detection(
|
||||||
@@ -333,6 +487,7 @@ def test_yolo_run_rejects_missing_tile_manifest_file(tmp_path: Path) -> None:
|
|||||||
model_path = tmp_path / "model.pt"
|
model_path = tmp_path / "model.pt"
|
||||||
model_path.write_bytes(b"local weights")
|
model_path.write_bytes(b"local weights")
|
||||||
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
||||||
|
_write_model_sidecar(model_path, settings, db=db)
|
||||||
|
|
||||||
result = DetectionService.run_detection(
|
result = DetectionService.run_detection(
|
||||||
db=db,
|
db=db,
|
||||||
@@ -354,6 +509,7 @@ def test_yolo_run_rejects_invalid_tile_manifest_json(tmp_path: Path) -> None:
|
|||||||
model_path = tmp_path / "model.pt"
|
model_path = tmp_path / "model.pt"
|
||||||
model_path.write_bytes(b"local weights")
|
model_path.write_bytes(b"local weights")
|
||||||
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
||||||
|
_write_model_sidecar(model_path, settings, db=db)
|
||||||
manifest_path = tmp_path / "manifest.json"
|
manifest_path = tmp_path / "manifest.json"
|
||||||
manifest_path.write_text("{not-json", encoding="utf-8")
|
manifest_path.write_text("{not-json", encoding="utf-8")
|
||||||
|
|
||||||
@@ -372,6 +528,32 @@ def test_yolo_run_rejects_invalid_tile_manifest_json(tmp_path: Path) -> None:
|
|||||||
assert result.error_code == "DETECTION_TILE_MANIFEST_INVALID"
|
assert result.error_code == "DETECTION_TILE_MANIFEST_INVALID"
|
||||||
|
|
||||||
|
|
||||||
|
def test_yolo_run_rejects_unbound_model_snapshot_before_adapter_load(tmp_path: Path) -> None:
|
||||||
|
db, project_id, dataset_id = _project_and_dataset()
|
||||||
|
model_path = tmp_path / "model.pt"
|
||||||
|
model_path.write_bytes(b"structurally valid but unbound model")
|
||||||
|
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
||||||
|
# A catalog/preflight sidecar alone is deliberately insufficient for a
|
||||||
|
# production call. Do not register the declared source IDs in ``db``.
|
||||||
|
_write_model_sidecar(model_path, settings)
|
||||||
|
NeverLoadUnboundModelAdapter.load_calls = 0
|
||||||
|
|
||||||
|
result = DetectionService.run_detection(
|
||||||
|
db=db,
|
||||||
|
project_id=project_id,
|
||||||
|
dataset_id=dataset_id,
|
||||||
|
model_id="yolo-configured",
|
||||||
|
confidence_threshold=0.5,
|
||||||
|
tile_manifest_path=str(_manifest(tmp_path)),
|
||||||
|
settings=settings,
|
||||||
|
yolo_adapter_class=NeverLoadUnboundModelAdapter,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.status == "failed"
|
||||||
|
assert result.error_code == "MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND"
|
||||||
|
assert NeverLoadUnboundModelAdapter.load_calls == 0
|
||||||
|
|
||||||
|
|
||||||
def test_pixel_bbox_to_epsg4326_polygon_from_gdal_transform() -> None:
|
def test_pixel_bbox_to_epsg4326_polygon_from_gdal_transform() -> None:
|
||||||
polygon = pixel_bbox_to_epsg4326_polygon(
|
polygon = pixel_bbox_to_epsg4326_polygon(
|
||||||
bbox=[10, 20, 30, 40],
|
bbox=[10, 20, 30, 40],
|
||||||
@@ -390,6 +572,7 @@ def test_yolo_run_persists_mocked_georeferenced_detections(tmp_path: Path) -> No
|
|||||||
model_path = tmp_path / "model.pt"
|
model_path = tmp_path / "model.pt"
|
||||||
model_path.write_bytes(b"local weights")
|
model_path.write_bytes(b"local weights")
|
||||||
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_model_version="local-test")
|
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_model_version="local-test")
|
||||||
|
_write_model_sidecar(model_path, settings, db=db)
|
||||||
manifest_path = _manifest(tmp_path, tile_count=1)
|
manifest_path = _manifest(tmp_path, tile_count=1)
|
||||||
|
|
||||||
result = DetectionService.run_detection(
|
result = DetectionService.run_detection(
|
||||||
@@ -416,7 +599,10 @@ def test_yolo_run_persists_mocked_georeferenced_detections(tmp_path: Path) -> No
|
|||||||
assert detections[0].confidence == 0.91
|
assert detections[0].confidence == 0.91
|
||||||
assert detections[0].source_tile_path.endswith("tile_0000.tif")
|
assert detections[0].source_tile_path.endswith("tile_0000.tif")
|
||||||
assert detections[0].bbox_json == {"x_min": 10.0, "y_min": 20.0, "x_max": 30.0, "y_max": 40.0}
|
assert detections[0].bbox_json == {"x_min": 10.0, "y_min": 20.0, "x_max": 30.0, "y_max": 40.0}
|
||||||
assert detections[0].properties_json == {"adapter": "mock", "tile_index": 0}
|
assert detections[0].properties_json["adapter"] == "mock"
|
||||||
|
assert detections[0].properties_json["tile_index"] == 0
|
||||||
|
assert detections[0].properties_json["runtime_model_provenance"]["model_sha256"] == sha256(model_path.read_bytes()).hexdigest()
|
||||||
|
assert runs[0].parameters_json["runtime_model_provenance"]["data_contract_key"] == "geointel.model.pytorch"
|
||||||
assert runs[0].status == "success"
|
assert runs[0].status == "success"
|
||||||
assert jobs[0].status == "success"
|
assert jobs[0].status == "success"
|
||||||
|
|
||||||
@@ -426,6 +612,7 @@ def test_yolo_class_filter_is_case_insensitive_and_persists_canonical_class(tmp_
|
|||||||
model_path = tmp_path / "model.pt"
|
model_path = tmp_path / "model.pt"
|
||||||
model_path.write_bytes(b"local weights")
|
model_path.write_bytes(b"local weights")
|
||||||
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
settings = _settings(tmp_path, yolo_model_path=str(model_path))
|
||||||
|
_write_model_sidecar(model_path, settings, db=db)
|
||||||
manifest_path = _manifest(tmp_path, tile_count=1)
|
manifest_path = _manifest(tmp_path, tile_count=1)
|
||||||
|
|
||||||
result = DetectionService.run_detection(
|
result = DetectionService.run_detection(
|
||||||
@@ -453,6 +640,7 @@ def test_yolo_run_suppresses_cross_tile_duplicate_detections(tmp_path: Path) ->
|
|||||||
model_path = tmp_path / "model.pt"
|
model_path = tmp_path / "model.pt"
|
||||||
model_path.write_bytes(b"local weights")
|
model_path.write_bytes(b"local weights")
|
||||||
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_duplicate_iou_threshold=0.5)
|
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_duplicate_iou_threshold=0.5)
|
||||||
|
_write_model_sidecar(model_path, settings, db=db)
|
||||||
manifest_path = _manifest(tmp_path, tile_count=2)
|
manifest_path = _manifest(tmp_path, tile_count=2)
|
||||||
|
|
||||||
result = DetectionService.run_detection(
|
result = DetectionService.run_detection(
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from shapely.geometry import Polygon, box
|
|||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
from app.main import app
|
from app.main import app
|
||||||
from app.db.session import get_db
|
from app.db.session import get_db
|
||||||
from app.models import AnalysisRun, Dataset, Detection, Metric, QualityCheck, VectorFeature
|
from app.models import AnalysisRun, Dataset, Detection, Metric, QualityCheck, SourceRegistry, SourceSnapshot, VectorFeature
|
||||||
from app.services.detection_service import DetectionService
|
from app.services.detection_service import DetectionService
|
||||||
|
|
||||||
|
|
||||||
@@ -96,11 +96,52 @@ def _source_dataset(project_id, dataset_id):
|
|||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name="source.tif",
|
name="source.tif",
|
||||||
dataset_type="raster",
|
dataset_type="raster",
|
||||||
source="manual",
|
source="test",
|
||||||
source_name="manual",
|
source_name="test",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _authoritative_reference(dataset: Dataset) -> Dataset:
|
||||||
|
"""Model the reference as a fully governed GRB fixture, never test data."""
|
||||||
|
|
||||||
|
source_id = uuid4()
|
||||||
|
snapshot_id = uuid4()
|
||||||
|
checksum = "a" * 64
|
||||||
|
source = SourceRegistry(
|
||||||
|
id=source_id,
|
||||||
|
source_key="grb",
|
||||||
|
display_name="GRB QA fixture",
|
||||||
|
classification="authoritative",
|
||||||
|
authority_name="Digitaal Vlaanderen",
|
||||||
|
authority_scope_json={"zone": "Flanders"},
|
||||||
|
usage_policy_json={"ground_truth_allowed": True, "validation_authority": {"building_validation": "primary"}},
|
||||||
|
)
|
||||||
|
snapshot = SourceSnapshot(
|
||||||
|
id=snapshot_id,
|
||||||
|
source_registry_id=source_id,
|
||||||
|
snapshot_key=f"detection-qa-{dataset.id}",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
ingest_status="ingested",
|
||||||
|
freshness_status="current",
|
||||||
|
)
|
||||||
|
dataset.source = "grb"
|
||||||
|
dataset.source_name = "grb"
|
||||||
|
dataset.dataset_role = "reference"
|
||||||
|
dataset.status = "ready"
|
||||||
|
dataset.checksum_sha256 = checksum
|
||||||
|
dataset.source_registry_id = source_id
|
||||||
|
dataset.source_snapshot_id = snapshot_id
|
||||||
|
dataset.data_contract_key = "geointel.vector.geojson"
|
||||||
|
dataset.data_contract_version = "1.0.0"
|
||||||
|
dataset.validation_status = "passed"
|
||||||
|
dataset.provenance_status = "complete"
|
||||||
|
dataset.lineage_status = "complete"
|
||||||
|
dataset.quarantine_status = "not_quarantined"
|
||||||
|
dataset.source_registry = source
|
||||||
|
dataset.source_snapshot = snapshot
|
||||||
|
return dataset
|
||||||
|
|
||||||
|
|
||||||
def test_detection_geojson_feature_collection_shape() -> None:
|
def test_detection_geojson_feature_collection_shape() -> None:
|
||||||
project_id = uuid4()
|
project_id = uuid4()
|
||||||
dataset_id = uuid4()
|
dataset_id = uuid4()
|
||||||
@@ -201,14 +242,14 @@ def test_detection_qa_persists_quality_check_and_metrics() -> None:
|
|||||||
reference_dataset_id = uuid4()
|
reference_dataset_id = uuid4()
|
||||||
analysis_run_id = uuid4()
|
analysis_run_id = uuid4()
|
||||||
detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1))
|
detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1))
|
||||||
reference_dataset = Dataset(
|
reference_dataset = _authoritative_reference(Dataset(
|
||||||
id=reference_dataset_id,
|
id=reference_dataset_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name="reference.geojson",
|
name="reference.geojson",
|
||||||
dataset_type="vector",
|
dataset_type="vector",
|
||||||
source="manual",
|
source="test",
|
||||||
dataset_role="reference",
|
dataset_role="reference",
|
||||||
)
|
))
|
||||||
reference_feature = VectorFeature(
|
reference_feature = VectorFeature(
|
||||||
id=uuid4(),
|
id=uuid4(),
|
||||||
dataset_id=reference_dataset_id,
|
dataset_id=reference_dataset_id,
|
||||||
@@ -262,7 +303,7 @@ def test_detection_qa_rejects_non_overlapping_historical_reference_editions() ->
|
|||||||
source_dataset.source_metadata = {"product_key": "2020", "supports_detection": False}
|
source_dataset.source_metadata = {"product_key": "2020", "supports_detection": False}
|
||||||
source_dataset.valid_from = datetime(2020, 1, 1, tzinfo=UTC)
|
source_dataset.valid_from = datetime(2020, 1, 1, tzinfo=UTC)
|
||||||
source_dataset.valid_to = datetime(2020, 12, 31, 23, 59, 59, tzinfo=UTC)
|
source_dataset.valid_to = datetime(2020, 12, 31, 23, 59, 59, tzinfo=UTC)
|
||||||
reference_dataset = Dataset(
|
reference_dataset = _authoritative_reference(Dataset(
|
||||||
id=reference_dataset_id,
|
id=reference_dataset_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name="current-grb.geojson",
|
name="current-grb.geojson",
|
||||||
@@ -272,7 +313,7 @@ def test_detection_qa_rejects_non_overlapping_historical_reference_editions() ->
|
|||||||
dataset_role="reference",
|
dataset_role="reference",
|
||||||
valid_from=datetime(2026, 7, 1, tzinfo=UTC),
|
valid_from=datetime(2026, 7, 1, tzinfo=UTC),
|
||||||
valid_to=datetime(2026, 7, 31, 23, 59, 59, tzinfo=UTC),
|
valid_to=datetime(2026, 7, 31, 23, 59, 59, tzinfo=UTC),
|
||||||
)
|
))
|
||||||
db = FakeSession(
|
db = FakeSession(
|
||||||
objects={
|
objects={
|
||||||
(AnalysisRun, analysis_run_id): AnalysisRun(
|
(AnalysisRun, analysis_run_id): AnalysisRun(
|
||||||
@@ -305,14 +346,14 @@ def test_detection_qa_no_match_case_persists_zero_scores() -> None:
|
|||||||
reference_dataset_id = uuid4()
|
reference_dataset_id = uuid4()
|
||||||
analysis_run_id = uuid4()
|
analysis_run_id = uuid4()
|
||||||
detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1))
|
detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1))
|
||||||
reference_dataset = Dataset(
|
reference_dataset = _authoritative_reference(Dataset(
|
||||||
id=reference_dataset_id,
|
id=reference_dataset_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name="reference.geojson",
|
name="reference.geojson",
|
||||||
dataset_type="vector",
|
dataset_type="vector",
|
||||||
source="manual",
|
source="test",
|
||||||
dataset_role="reference",
|
dataset_role="reference",
|
||||||
)
|
))
|
||||||
reference_feature = VectorFeature(
|
reference_feature = VectorFeature(
|
||||||
id=uuid4(),
|
id=uuid4(),
|
||||||
dataset_id=reference_dataset_id,
|
dataset_id=reference_dataset_id,
|
||||||
@@ -349,14 +390,14 @@ def test_configured_yolo_qa_requires_persisted_tile_manifest_provenance() -> Non
|
|||||||
reference_dataset_id = uuid4()
|
reference_dataset_id = uuid4()
|
||||||
analysis_run_id = uuid4()
|
analysis_run_id = uuid4()
|
||||||
detection = _detection(project_id, dataset_id, analysis_run_id)
|
detection = _detection(project_id, dataset_id, analysis_run_id)
|
||||||
reference_dataset = Dataset(
|
reference_dataset = _authoritative_reference(Dataset(
|
||||||
id=reference_dataset_id,
|
id=reference_dataset_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name="reference.geojson",
|
name="reference.geojson",
|
||||||
dataset_type="vector",
|
dataset_type="vector",
|
||||||
source="manual",
|
source="test",
|
||||||
dataset_role="reference",
|
dataset_role="reference",
|
||||||
)
|
))
|
||||||
reference_feature = VectorFeature(
|
reference_feature = VectorFeature(
|
||||||
id=uuid4(),
|
id=uuid4(),
|
||||||
dataset_id=reference_dataset_id,
|
dataset_id=reference_dataset_id,
|
||||||
@@ -421,14 +462,14 @@ def test_detection_qa_excludes_references_outside_persisted_tile_coverage(tmp_pa
|
|||||||
analysis_run_id = uuid4()
|
analysis_run_id = uuid4()
|
||||||
manifest_path = _coverage_manifest(tmp_path, dataset_id, bounds=(0.0, 0.0, 1.0, 1.0))
|
manifest_path = _coverage_manifest(tmp_path, dataset_id, bounds=(0.0, 0.0, 1.0, 1.0))
|
||||||
detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0.1, 0.1, 0.9, 0.9))
|
detection = _detection(project_id, dataset_id, analysis_run_id, geom=box(0.1, 0.1, 0.9, 0.9))
|
||||||
reference_dataset = Dataset(
|
reference_dataset = _authoritative_reference(Dataset(
|
||||||
id=reference_dataset_id,
|
id=reference_dataset_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name="reference.geojson",
|
name="reference.geojson",
|
||||||
dataset_type="vector",
|
dataset_type="vector",
|
||||||
source="manual",
|
source="test",
|
||||||
dataset_role="reference",
|
dataset_role="reference",
|
||||||
)
|
))
|
||||||
inside_reference = VectorFeature(
|
inside_reference = VectorFeature(
|
||||||
id=uuid4(),
|
id=uuid4(),
|
||||||
dataset_id=reference_dataset_id,
|
dataset_id=reference_dataset_id,
|
||||||
@@ -489,14 +530,14 @@ def test_detection_qa_reports_box_to_footprint_diagnostic_without_changing_stric
|
|||||||
l_shaped_footprint = Polygon(
|
l_shaped_footprint = Polygon(
|
||||||
[(0.0, 0.0), (2.0, 0.0), (2.0, 0.4), (0.4, 0.4), (0.4, 2.0), (0.0, 2.0), (0.0, 0.0)]
|
[(0.0, 0.0), (2.0, 0.0), (2.0, 0.4), (0.4, 0.4), (0.4, 2.0), (0.0, 2.0), (0.0, 0.0)]
|
||||||
)
|
)
|
||||||
reference_dataset = Dataset(
|
reference_dataset = _authoritative_reference(Dataset(
|
||||||
id=reference_dataset_id,
|
id=reference_dataset_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name="reference.geojson",
|
name="reference.geojson",
|
||||||
dataset_type="vector",
|
dataset_type="vector",
|
||||||
source="manual",
|
source="test",
|
||||||
dataset_role="reference",
|
dataset_role="reference",
|
||||||
)
|
))
|
||||||
reference_feature = VectorFeature(
|
reference_feature = VectorFeature(
|
||||||
id=uuid4(),
|
id=uuid4(),
|
||||||
dataset_id=reference_dataset_id,
|
dataset_id=reference_dataset_id,
|
||||||
|
|||||||
@@ -10,7 +10,18 @@ from shapely.geometry import MultiPolygon, box, mapping
|
|||||||
|
|
||||||
from app.db.session import get_db
|
from app.db.session import get_db
|
||||||
from app.main import app
|
from app.main import app
|
||||||
from app.models import AnalysisRun, Dataset, Job, Metric, Project, QualityCheck, Segmentation, VectorFeature
|
from app.models import (
|
||||||
|
AnalysisRun,
|
||||||
|
Dataset,
|
||||||
|
Job,
|
||||||
|
Metric,
|
||||||
|
Project,
|
||||||
|
QualityCheck,
|
||||||
|
Segmentation,
|
||||||
|
SourceRegistry,
|
||||||
|
SourceSnapshot,
|
||||||
|
VectorFeature,
|
||||||
|
)
|
||||||
from app.services.model_registry_service import ModelRegistryService
|
from app.services.model_registry_service import ModelRegistryService
|
||||||
from app.services.segmentation_service import SegmentationService
|
from app.services.segmentation_service import SegmentationService
|
||||||
|
|
||||||
@@ -75,7 +86,7 @@ def _project_and_dataset(dataset_type: str = "raster"):
|
|||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name="source.tif",
|
name="source.tif",
|
||||||
dataset_type=dataset_type,
|
dataset_type=dataset_type,
|
||||||
source="user_upload",
|
source="test",
|
||||||
storage_path="storage/uploads/source.tif",
|
storage_path="storage/uploads/source.tif",
|
||||||
)
|
)
|
||||||
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
|
||||||
@@ -105,6 +116,47 @@ def _segmentation(project_id, dataset_id, analysis_run_id, class_name="vegetatio
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _authoritative_reference(dataset: Dataset) -> Dataset:
|
||||||
|
"""Model segmentation QA references as governed authoritative GRB evidence."""
|
||||||
|
|
||||||
|
source_id = uuid4()
|
||||||
|
snapshot_id = uuid4()
|
||||||
|
checksum = "a" * 64
|
||||||
|
source = SourceRegistry(
|
||||||
|
id=source_id,
|
||||||
|
source_key="grb",
|
||||||
|
display_name="GRB segmentation QA fixture",
|
||||||
|
classification="authoritative",
|
||||||
|
authority_name="Digitaal Vlaanderen",
|
||||||
|
authority_scope_json={"zone": "Flanders"},
|
||||||
|
usage_policy_json={"ground_truth_allowed": True, "validation_authority": {"building_validation": "primary"}},
|
||||||
|
)
|
||||||
|
snapshot = SourceSnapshot(
|
||||||
|
id=snapshot_id,
|
||||||
|
source_registry_id=source_id,
|
||||||
|
snapshot_key=f"segmentation-qa-{dataset.id}",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
ingest_status="ingested",
|
||||||
|
freshness_status="current",
|
||||||
|
)
|
||||||
|
dataset.source = "grb"
|
||||||
|
dataset.source_name = "grb"
|
||||||
|
dataset.dataset_role = "reference"
|
||||||
|
dataset.status = "ready"
|
||||||
|
dataset.checksum_sha256 = checksum
|
||||||
|
dataset.source_registry_id = source_id
|
||||||
|
dataset.source_snapshot_id = snapshot_id
|
||||||
|
dataset.data_contract_key = "geointel.vector.geojson"
|
||||||
|
dataset.data_contract_version = "1.0.0"
|
||||||
|
dataset.validation_status = "passed"
|
||||||
|
dataset.provenance_status = "complete"
|
||||||
|
dataset.lineage_status = "complete"
|
||||||
|
dataset.quarantine_status = "not_quarantined"
|
||||||
|
dataset.source_registry = source
|
||||||
|
dataset.source_snapshot = snapshot
|
||||||
|
return dataset
|
||||||
|
|
||||||
|
|
||||||
def test_model_registry_returns_segmentation_states() -> None:
|
def test_model_registry_returns_segmentation_states() -> None:
|
||||||
models = {model.model_id: model for model in ModelRegistryService.list_model_capabilities(task_type="segmentation")}
|
models = {model.model_id: model for model in ModelRegistryService.list_model_capabilities(task_type="segmentation")}
|
||||||
|
|
||||||
@@ -265,14 +317,14 @@ def test_segmentation_qa_persists_quality_check_and_metrics() -> None:
|
|||||||
reference_dataset_id = uuid4()
|
reference_dataset_id = uuid4()
|
||||||
analysis_run_id = uuid4()
|
analysis_run_id = uuid4()
|
||||||
segmentation = _segmentation(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1))
|
segmentation = _segmentation(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1))
|
||||||
reference_dataset = Dataset(
|
reference_dataset = _authoritative_reference(Dataset(
|
||||||
id=reference_dataset_id,
|
id=reference_dataset_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name="reference.geojson",
|
name="reference.geojson",
|
||||||
dataset_type="vector",
|
dataset_type="vector",
|
||||||
source="manual",
|
source="test",
|
||||||
dataset_role="reference",
|
dataset_role="reference",
|
||||||
)
|
))
|
||||||
reference_feature = VectorFeature(
|
reference_feature = VectorFeature(
|
||||||
id=uuid4(),
|
id=uuid4(),
|
||||||
dataset_id=reference_dataset_id,
|
dataset_id=reference_dataset_id,
|
||||||
@@ -282,6 +334,7 @@ def test_segmentation_qa_persists_quality_check_and_metrics() -> None:
|
|||||||
db = FakeSession(
|
db = FakeSession(
|
||||||
objects={
|
objects={
|
||||||
(AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="segmentation", status="success", parameters_json={}),
|
(AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="segmentation", status="success", parameters_json={}),
|
||||||
|
(Dataset, dataset_id): Dataset(id=dataset_id, project_id=project_id, name="fixture.tif", dataset_type="raster", source="test"),
|
||||||
(Dataset, reference_dataset_id): reference_dataset,
|
(Dataset, reference_dataset_id): reference_dataset,
|
||||||
},
|
},
|
||||||
query_rows={Segmentation: [segmentation], VectorFeature: [reference_feature]},
|
query_rows={Segmentation: [segmentation], VectorFeature: [reference_feature]},
|
||||||
@@ -318,14 +371,14 @@ def test_segmentation_qa_no_match_case_persists_zero_scores() -> None:
|
|||||||
reference_dataset_id = uuid4()
|
reference_dataset_id = uuid4()
|
||||||
analysis_run_id = uuid4()
|
analysis_run_id = uuid4()
|
||||||
segmentation = _segmentation(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1))
|
segmentation = _segmentation(project_id, dataset_id, analysis_run_id, geom=box(0, 0, 1, 1))
|
||||||
reference_dataset = Dataset(
|
reference_dataset = _authoritative_reference(Dataset(
|
||||||
id=reference_dataset_id,
|
id=reference_dataset_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name="reference.geojson",
|
name="reference.geojson",
|
||||||
dataset_type="vector",
|
dataset_type="vector",
|
||||||
source="manual",
|
source="test",
|
||||||
dataset_role="reference",
|
dataset_role="reference",
|
||||||
)
|
))
|
||||||
reference_feature = VectorFeature(
|
reference_feature = VectorFeature(
|
||||||
id=uuid4(),
|
id=uuid4(),
|
||||||
dataset_id=reference_dataset_id,
|
dataset_id=reference_dataset_id,
|
||||||
@@ -335,6 +388,7 @@ def test_segmentation_qa_no_match_case_persists_zero_scores() -> None:
|
|||||||
db = FakeSession(
|
db = FakeSession(
|
||||||
objects={
|
objects={
|
||||||
(AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="segmentation", status="success", parameters_json={}),
|
(AnalysisRun, analysis_run_id): AnalysisRun(id=analysis_run_id, project_id=project_id, dataset_id=dataset_id, analysis_type="segmentation", status="success", parameters_json={}),
|
||||||
|
(Dataset, dataset_id): Dataset(id=dataset_id, project_id=project_id, name="fixture.tif", dataset_type="raster", source="test"),
|
||||||
(Dataset, reference_dataset_id): reference_dataset,
|
(Dataset, reference_dataset_id): reference_dataset,
|
||||||
},
|
},
|
||||||
query_rows={Segmentation: [segmentation], VectorFeature: [reference_feature]},
|
query_rows={Segmentation: [segmentation], VectorFeature: [reference_feature]},
|
||||||
|
|||||||
@@ -0,0 +1,410 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import importlib.util
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from uuid import uuid4
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
SCRIPT = ROOT / "scripts" / "training_dataset_eligibility.py"
|
||||||
|
SPEC = importlib.util.spec_from_file_location("training_dataset_eligibility", SCRIPT)
|
||||||
|
assert SPEC and SPEC.loader
|
||||||
|
MODULE = importlib.util.module_from_spec(SPEC)
|
||||||
|
SPEC.loader.exec_module(MODULE)
|
||||||
|
|
||||||
|
CHECKSUM = "a" * 64
|
||||||
|
UNSET = object()
|
||||||
|
|
||||||
|
|
||||||
|
def source_registry(
|
||||||
|
*,
|
||||||
|
classification: str = "authoritative",
|
||||||
|
training_allowed: bool = True,
|
||||||
|
ground_truth_allowed: bool = True,
|
||||||
|
allowed_tasks: list[str] | None = None,
|
||||||
|
building_validation_authority: str = "primary",
|
||||||
|
) -> SimpleNamespace:
|
||||||
|
return SimpleNamespace(
|
||||||
|
id="source-registry-1",
|
||||||
|
source_key="governed-source",
|
||||||
|
classification=classification,
|
||||||
|
freshness_status="current",
|
||||||
|
ingest_status="ingested",
|
||||||
|
usage_policy_json={
|
||||||
|
"training_allowed": training_allowed,
|
||||||
|
"ground_truth_allowed": ground_truth_allowed,
|
||||||
|
"allowed_tasks": allowed_tasks or ["building_validation", "building_labels"],
|
||||||
|
"validation_authority": {
|
||||||
|
"building_validation": building_validation_authority,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def source_snapshot(*, checksum: str = CHECKSUM) -> SimpleNamespace:
|
||||||
|
return SimpleNamespace(
|
||||||
|
id="source-snapshot-1",
|
||||||
|
snapshot_key="2026-08-01",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
freshness_status="current",
|
||||||
|
ingest_status="ingested",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def governed_dataset(
|
||||||
|
*,
|
||||||
|
role: str,
|
||||||
|
registry: SimpleNamespace | None | object = UNSET,
|
||||||
|
snapshot: SimpleNamespace | None | object = UNSET,
|
||||||
|
**overrides: object,
|
||||||
|
) -> SimpleNamespace:
|
||||||
|
dataset_type = "raster" if role == "raster" else "vector"
|
||||||
|
values: dict[str, object] = {
|
||||||
|
"id": f"dataset-{role}",
|
||||||
|
"dataset_type": dataset_type,
|
||||||
|
"dataset_role": "source" if role == "raster" else "reference",
|
||||||
|
"source": "governed_import",
|
||||||
|
"source_name": "governed-source",
|
||||||
|
"checksum_sha256": CHECKSUM,
|
||||||
|
"data_contract_key": f"{role}-contract",
|
||||||
|
"data_contract_version": "1.0.0",
|
||||||
|
"validation_status": "passed",
|
||||||
|
"provenance_status": "complete",
|
||||||
|
"lineage_status": "not_applicable",
|
||||||
|
"quarantine_status": "not_quarantined",
|
||||||
|
"status": "ready",
|
||||||
|
"metadata_json": {},
|
||||||
|
"provenance_metadata": {},
|
||||||
|
"source_registry": source_registry() if registry is UNSET else registry,
|
||||||
|
"source_snapshot": source_snapshot() if snapshot is UNSET else snapshot,
|
||||||
|
}
|
||||||
|
values.update(overrides)
|
||||||
|
return SimpleNamespace(**values)
|
||||||
|
|
||||||
|
|
||||||
|
def test_governed_authoritative_pair_is_eligible_for_operational_training() -> None:
|
||||||
|
raster = governed_dataset(
|
||||||
|
role="raster",
|
||||||
|
registry=source_registry(ground_truth_allowed=False),
|
||||||
|
)
|
||||||
|
reference = governed_dataset(role="reference")
|
||||||
|
|
||||||
|
decision = MODULE.training_pair_evidence(raster=raster, reference=reference)
|
||||||
|
|
||||||
|
assert decision["eligible"] is True
|
||||||
|
assert decision["raster"]["reasons"] == []
|
||||||
|
assert decision["reference"]["evidence"]["source_ground_truth_allowed"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_operational_training_rejects_invalid_quarantined_incomplete_and_untrusted_inputs() -> None:
|
||||||
|
dataset = governed_dataset(
|
||||||
|
role="reference",
|
||||||
|
validation_status="failed",
|
||||||
|
provenance_status="incomplete",
|
||||||
|
lineage_status="incomplete",
|
||||||
|
quarantine_status="quarantined",
|
||||||
|
source_registry=source_registry(
|
||||||
|
classification="contextual",
|
||||||
|
training_allowed=False,
|
||||||
|
ground_truth_allowed=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
||||||
|
|
||||||
|
assert decision.eligible is False
|
||||||
|
assert set(decision.reasons) >= {
|
||||||
|
"validation_failed",
|
||||||
|
"dataset_quarantined",
|
||||||
|
"provenance_not_complete",
|
||||||
|
"lineage_not_complete",
|
||||||
|
"source_not_allowed_for_training",
|
||||||
|
"reference_source_not_authoritative",
|
||||||
|
"reference_source_not_ground_truth_allowed",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_operational_training_rejects_a_due_source_snapshot() -> None:
|
||||||
|
snapshot = source_snapshot()
|
||||||
|
snapshot.freshness_status = "due"
|
||||||
|
dataset = governed_dataset(role="reference", snapshot=snapshot)
|
||||||
|
|
||||||
|
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
||||||
|
|
||||||
|
assert decision.eligible is False
|
||||||
|
assert "source_snapshot_freshness_not_approved" in decision.reasons
|
||||||
|
|
||||||
|
|
||||||
|
def test_osm_like_context_is_never_accepted_as_building_ground_truth() -> None:
|
||||||
|
dataset = governed_dataset(
|
||||||
|
role="reference",
|
||||||
|
source_name="osm",
|
||||||
|
source_registry=source_registry(
|
||||||
|
classification="contextual",
|
||||||
|
training_allowed=False,
|
||||||
|
ground_truth_allowed=False,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
||||||
|
|
||||||
|
assert decision.eligible is False
|
||||||
|
assert "source_not_allowed_for_training" in decision.reasons
|
||||||
|
assert "reference_source_not_authoritative" in decision.reasons
|
||||||
|
|
||||||
|
|
||||||
|
def test_regional_building_sources_pending_primary_authority_cannot_enter_training_labels() -> None:
|
||||||
|
for source_key in ("spw_picc", "urbis"):
|
||||||
|
dataset = governed_dataset(
|
||||||
|
role="reference",
|
||||||
|
source_name=source_key,
|
||||||
|
source_registry=source_registry(
|
||||||
|
allowed_tasks=["building_validation", "building_labels"],
|
||||||
|
building_validation_authority="regional_primary_pending_contract",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
||||||
|
|
||||||
|
assert decision.eligible is False
|
||||||
|
assert "reference_building_validation_not_primary" in decision.reasons
|
||||||
|
|
||||||
|
|
||||||
|
def test_authoritative_source_without_building_validation_task_cannot_be_used_as_a_label_reference() -> None:
|
||||||
|
dataset = governed_dataset(
|
||||||
|
role="reference",
|
||||||
|
source_registry=source_registry(
|
||||||
|
allowed_tasks=["elevation_validation"],
|
||||||
|
building_validation_authority="corroborative",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
||||||
|
|
||||||
|
assert decision.eligible is False
|
||||||
|
assert set(decision.reasons) >= {
|
||||||
|
"reference_source_not_approved_for_building_validation",
|
||||||
|
"reference_building_validation_not_primary",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_dataset_and_snapshot_registry_bindings_cannot_be_forged() -> None:
|
||||||
|
snapshot = source_snapshot()
|
||||||
|
snapshot.source_registry_id = "different-registry"
|
||||||
|
dataset = governed_dataset(
|
||||||
|
role="reference",
|
||||||
|
registry=source_registry(),
|
||||||
|
snapshot=snapshot,
|
||||||
|
source_registry_id="different-registry",
|
||||||
|
source_snapshot_id="different-snapshot",
|
||||||
|
)
|
||||||
|
|
||||||
|
decision = MODULE.evaluate_dataset_training_eligibility(dataset, role="reference")
|
||||||
|
|
||||||
|
assert decision.eligible is False
|
||||||
|
assert set(decision.reasons) >= {
|
||||||
|
"dataset_source_registry_binding_mismatch",
|
||||||
|
"dataset_source_snapshot_binding_mismatch",
|
||||||
|
"source_snapshot_registry_mismatch",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_fixture_mode_only_relaxes_legacy_provenance_for_explicit_fixtures() -> None:
|
||||||
|
fixture = governed_dataset(
|
||||||
|
role="reference",
|
||||||
|
source="fixture",
|
||||||
|
source_name="fixture",
|
||||||
|
validation_status=None,
|
||||||
|
provenance_status="incomplete",
|
||||||
|
lineage_status="incomplete",
|
||||||
|
data_contract_key=None,
|
||||||
|
data_contract_version=None,
|
||||||
|
checksum_sha256=None,
|
||||||
|
source_registry=None,
|
||||||
|
source_snapshot=None,
|
||||||
|
metadata_json={"fixture": True},
|
||||||
|
)
|
||||||
|
unmarked = governed_dataset(
|
||||||
|
role="reference",
|
||||||
|
validation_status=None,
|
||||||
|
provenance_status="incomplete",
|
||||||
|
lineage_status="incomplete",
|
||||||
|
source_registry=None,
|
||||||
|
source_snapshot=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert MODULE.evaluate_dataset_training_eligibility(
|
||||||
|
fixture,
|
||||||
|
role="reference",
|
||||||
|
fixture_mode=True,
|
||||||
|
).eligible is True
|
||||||
|
rejected = MODULE.evaluate_dataset_training_eligibility(
|
||||||
|
unmarked,
|
||||||
|
role="reference",
|
||||||
|
fixture_mode=True,
|
||||||
|
)
|
||||||
|
assert rejected.eligible is False
|
||||||
|
assert "fixture_mode_requires_explicit_fixture" in rejected.reasons
|
||||||
|
|
||||||
|
|
||||||
|
def test_fixture_mode_never_allows_failed_validation_or_quarantine() -> None:
|
||||||
|
fixture = governed_dataset(
|
||||||
|
role="raster",
|
||||||
|
source="fixture",
|
||||||
|
source_name="fixture",
|
||||||
|
validation_status="failed",
|
||||||
|
quarantine_status="quarantined",
|
||||||
|
source_registry=None,
|
||||||
|
source_snapshot=None,
|
||||||
|
)
|
||||||
|
|
||||||
|
decision = MODULE.evaluate_dataset_training_eligibility(fixture, role="raster", fixture_mode=True)
|
||||||
|
|
||||||
|
assert decision.eligible is False
|
||||||
|
assert set(decision.reasons) >= {"validation_failed", "dataset_quarantined"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_manifest_gate_rejects_missing_or_tampered_pair_decisions() -> None:
|
||||||
|
raster = governed_dataset(
|
||||||
|
role="raster",
|
||||||
|
registry=source_registry(ground_truth_allowed=False),
|
||||||
|
)
|
||||||
|
reference = governed_dataset(role="reference")
|
||||||
|
pair = MODULE.training_pair_evidence(raster=raster, reference=reference)
|
||||||
|
manifest = {
|
||||||
|
"samples": [{"sample_slug": "governed", "training_eligibility": pair}],
|
||||||
|
"training_eligibility": {
|
||||||
|
"policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||||
|
"status": "eligible",
|
||||||
|
"fixture_mode": False,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
assert MODULE.manifest_training_eligibility_failures(manifest) == []
|
||||||
|
tampered = {
|
||||||
|
**manifest,
|
||||||
|
"samples": [{"sample_slug": "governed", "training_eligibility": {**pair, "eligible": False}}],
|
||||||
|
}
|
||||||
|
failures = MODULE.manifest_training_eligibility_failures(tampered)
|
||||||
|
assert "governed:training_pair_not_eligible" in failures
|
||||||
|
assert MODULE.manifest_training_eligibility_failures({"samples": []}) == [
|
||||||
|
"manifest_training_eligibility_missing"
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def test_frozen_manifest_gate_detects_checksum_tampering(tmp_path: Path) -> None:
|
||||||
|
raster = governed_dataset(
|
||||||
|
role="raster",
|
||||||
|
registry=source_registry(ground_truth_allowed=False),
|
||||||
|
)
|
||||||
|
reference = governed_dataset(role="reference")
|
||||||
|
pair = MODULE.training_pair_evidence(raster=raster, reference=reference)
|
||||||
|
manifest_path = tmp_path / "operator_samples_manifest.json"
|
||||||
|
manifest_path.write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"immutable": True,
|
||||||
|
"training_eligibility": {
|
||||||
|
"policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||||
|
"status": "eligible",
|
||||||
|
"fixture_mode": False,
|
||||||
|
},
|
||||||
|
"samples": [{"sample_slug": "governed", "training_eligibility": pair}],
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
(tmp_path / "corpus-freeze.json").write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"schema_version": 2,
|
||||||
|
"manifest_sha256": hashlib.sha256(manifest_path.read_bytes()).hexdigest(),
|
||||||
|
"immutable": True,
|
||||||
|
"training_eligibility_policy": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||||
|
"fixture_mode": False,
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert MODULE.frozen_manifest_training_eligibility_failures(manifest_path) == []
|
||||||
|
manifest_path.write_text(manifest_path.read_text(encoding="utf-8") + "\n", encoding="utf-8")
|
||||||
|
assert "corpus_manifest_checksum_mismatch" in MODULE.frozen_manifest_training_eligibility_failures(
|
||||||
|
manifest_path
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_live_manifest_gate_revokes_a_frozen_pair_when_an_upstream_dataset_is_quarantined() -> None:
|
||||||
|
raster_id = uuid4()
|
||||||
|
reference_id = uuid4()
|
||||||
|
raster_registry = source_registry(ground_truth_allowed=False)
|
||||||
|
reference_registry = source_registry()
|
||||||
|
raster_snapshot = source_snapshot()
|
||||||
|
reference_snapshot = source_snapshot()
|
||||||
|
raster_snapshot.source_registry_id = raster_registry.id
|
||||||
|
reference_snapshot.source_registry_id = reference_registry.id
|
||||||
|
raster = governed_dataset(
|
||||||
|
role="raster",
|
||||||
|
id=raster_id,
|
||||||
|
source_registry=raster_registry,
|
||||||
|
source_snapshot=raster_snapshot,
|
||||||
|
source_registry_id=raster_registry.id,
|
||||||
|
source_snapshot_id=raster_snapshot.id,
|
||||||
|
)
|
||||||
|
reference = governed_dataset(
|
||||||
|
role="reference",
|
||||||
|
id=reference_id,
|
||||||
|
source_registry=reference_registry,
|
||||||
|
source_snapshot=reference_snapshot,
|
||||||
|
source_registry_id=reference_registry.id,
|
||||||
|
source_snapshot_id=reference_snapshot.id,
|
||||||
|
)
|
||||||
|
pair = MODULE.training_pair_evidence(raster=raster, reference=reference)
|
||||||
|
manifest = {
|
||||||
|
"training_eligibility": {
|
||||||
|
"policy_version": MODULE.TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||||
|
"status": "eligible",
|
||||||
|
"fixture_mode": False,
|
||||||
|
},
|
||||||
|
"samples": [
|
||||||
|
{
|
||||||
|
"sample_slug": "governed-aoi",
|
||||||
|
"raster_dataset_id": str(raster_id),
|
||||||
|
"reference_dataset_id": str(reference_id),
|
||||||
|
"training_eligibility": pair,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
class DatasetModel:
|
||||||
|
pass
|
||||||
|
|
||||||
|
class Session:
|
||||||
|
def __init__(self) -> None:
|
||||||
|
self.closed = False
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def get(model, item_id):
|
||||||
|
assert model is DatasetModel
|
||||||
|
return {raster_id: raster, reference_id: reference}.get(item_id)
|
||||||
|
|
||||||
|
def close(self) -> None:
|
||||||
|
self.closed = True
|
||||||
|
|
||||||
|
assert MODULE.live_manifest_training_eligibility_failures(
|
||||||
|
manifest,
|
||||||
|
session_factory=Session,
|
||||||
|
dataset_model=DatasetModel,
|
||||||
|
) == []
|
||||||
|
|
||||||
|
raster.quarantine_status = "quarantined"
|
||||||
|
failures = MODULE.live_manifest_training_eligibility_failures(
|
||||||
|
manifest,
|
||||||
|
session_factory=Session,
|
||||||
|
dataset_model=DatasetModel,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "governed-aoi:raster_live_revoked:dataset_quarantined" in failures
|
||||||
@@ -0,0 +1,414 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[2]
|
||||||
|
SCRIPTS = ROOT / "scripts"
|
||||||
|
if str(SCRIPTS) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SCRIPTS))
|
||||||
|
SCRIPT = SCRIPTS / "training_release_manifest.py"
|
||||||
|
SPEC = importlib.util.spec_from_file_location("training_release_manifest", SCRIPT)
|
||||||
|
assert SPEC and SPEC.loader
|
||||||
|
MODULE = importlib.util.module_from_spec(SPEC)
|
||||||
|
sys.modules[SPEC.name] = MODULE
|
||||||
|
SPEC.loader.exec_module(MODULE)
|
||||||
|
|
||||||
|
|
||||||
|
def sha256(path: Path) -> str:
|
||||||
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _synthetic_release_uses_a_static_live_registry_spy(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""Filesystem unit fixtures cannot resolve a real database, but must ask for it."""
|
||||||
|
|
||||||
|
original_assert = MODULE.assert_frozen_manifest_training_eligible
|
||||||
|
original_failures = MODULE.frozen_manifest_training_eligibility_failures
|
||||||
|
|
||||||
|
def static_assert(*args, **kwargs):
|
||||||
|
assert kwargs.get("verify_live") is True
|
||||||
|
kwargs["verify_live"] = False
|
||||||
|
return original_assert(*args, **kwargs)
|
||||||
|
|
||||||
|
def static_failures(*args, **kwargs):
|
||||||
|
assert kwargs.get("verify_live") is True
|
||||||
|
kwargs["verify_live"] = False
|
||||||
|
return original_failures(*args, **kwargs)
|
||||||
|
|
||||||
|
monkeypatch.setattr(MODULE, "assert_frozen_manifest_training_eligible", static_assert)
|
||||||
|
monkeypatch.setattr(MODULE, "frozen_manifest_training_eligibility_failures", static_failures)
|
||||||
|
|
||||||
|
|
||||||
|
def write_corpus_manifest(tmp_path: Path, *, fixture_mode: bool = False) -> Path:
|
||||||
|
policy = "geointel-training-source-eligibility/v1"
|
||||||
|
def pair(sample_slug: str) -> dict:
|
||||||
|
raster_id = f"dataset:raster:{sample_slug}"
|
||||||
|
reference_id = f"dataset:reference:{sample_slug}"
|
||||||
|
return {
|
||||||
|
"policy_version": policy,
|
||||||
|
"eligible": True,
|
||||||
|
"fixture_mode": fixture_mode,
|
||||||
|
"raster": {
|
||||||
|
"eligible": True,
|
||||||
|
"reasons": [],
|
||||||
|
"evidence": {
|
||||||
|
"dataset_id": raster_id,
|
||||||
|
"checksum_sha256": "a" * 64,
|
||||||
|
"source_registry_id": "registry:orthophoto",
|
||||||
|
"source_snapshot_id": "snapshot:orthophoto",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"reference": {
|
||||||
|
"eligible": True,
|
||||||
|
"reasons": [],
|
||||||
|
"evidence": {
|
||||||
|
"dataset_id": reference_id,
|
||||||
|
"checksum_sha256": "b" * 64,
|
||||||
|
"source_registry_id": "registry:grb",
|
||||||
|
"source_snapshot_id": "snapshot:grb",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
samples = []
|
||||||
|
for sample_slug, split in (("fixture-train", "train"), ("fixture-val", "val")):
|
||||||
|
samples.append(
|
||||||
|
{
|
||||||
|
"sample_slug": sample_slug,
|
||||||
|
"split": split,
|
||||||
|
"raster_dataset_id": f"dataset:raster:{sample_slug}",
|
||||||
|
"reference_dataset_id": f"dataset:reference:{sample_slug}",
|
||||||
|
"training_eligibility": pair(sample_slug),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
manifest = tmp_path / "operator_samples_manifest.json"
|
||||||
|
manifest.write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"immutable": True,
|
||||||
|
"training_eligibility": {
|
||||||
|
"policy_version": policy,
|
||||||
|
"status": "eligible",
|
||||||
|
"fixture_mode": fixture_mode,
|
||||||
|
},
|
||||||
|
"samples": samples,
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
(tmp_path / "corpus-freeze.json").write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"schema_version": 2,
|
||||||
|
"immutable": True,
|
||||||
|
"fixture_mode": fixture_mode,
|
||||||
|
"training_eligibility_policy": policy,
|
||||||
|
"manifest_sha256": sha256(manifest),
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
return manifest
|
||||||
|
|
||||||
|
|
||||||
|
def write_yolo_dataset(tmp_path: Path, *, empty_train_label: bool = False) -> Path:
|
||||||
|
dataset_root = tmp_path / "dataset"
|
||||||
|
for split, sample_slug in (("train", "fixture-train"), ("val", "fixture-val")):
|
||||||
|
image = dataset_root / "images" / split / f"{sample_slug}.png"
|
||||||
|
label = dataset_root / "labels" / split / f"{sample_slug}.txt"
|
||||||
|
image.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
label.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
image.write_bytes(f"{split}-image".encode("utf-8"))
|
||||||
|
label.write_text("" if split == "train" and empty_train_label else "0 0.5 0.5 0.2 0.2\n", encoding="utf-8")
|
||||||
|
yaml_path = dataset_root / "dataset.yaml"
|
||||||
|
yaml_path.write_text(
|
||||||
|
f"path: {dataset_root}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
return yaml_path
|
||||||
|
|
||||||
|
|
||||||
|
def write_accepted_review_audit(tmp_path: Path, corpus_manifest: Path) -> Path:
|
||||||
|
artifacts = {}
|
||||||
|
for sample_slug in ("fixture-train", "fixture-val"):
|
||||||
|
artifact = tmp_path / f"{sample_slug}-contact-sheet.png"
|
||||||
|
artifact.write_bytes(f"reviewed {sample_slug}".encode("utf-8"))
|
||||||
|
artifacts[sample_slug] = artifact
|
||||||
|
decisions = tmp_path / "review-decisions.json"
|
||||||
|
decisions.write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"decisions": [
|
||||||
|
{
|
||||||
|
"sample_slug": sample_slug,
|
||||||
|
"decision": "accepted",
|
||||||
|
"reviewer": "reviewer@example.test",
|
||||||
|
"reviewed_at": "2026-08-01T12:00:00+00:00",
|
||||||
|
"reviewed_artifact_path": str(artifact.resolve()),
|
||||||
|
"reviewed_artifact_sha256": sha256(artifact),
|
||||||
|
}
|
||||||
|
for sample_slug, artifact in artifacts.items()
|
||||||
|
]
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
evidence = {
|
||||||
|
"review_decisions_path": str(decisions.resolve()),
|
||||||
|
"review_decisions_sha256": sha256(decisions),
|
||||||
|
"required_sample_count": 2,
|
||||||
|
"accepted_sample_count": 2,
|
||||||
|
"accepted_sample_slugs": ["fixture-train", "fixture-val"],
|
||||||
|
"reviewer_ids": ["reviewer@example.test"],
|
||||||
|
"reviewed_at_by_sample": {
|
||||||
|
"fixture-train": "2026-08-01T12:00:00+00:00",
|
||||||
|
"fixture-val": "2026-08-01T12:00:00+00:00",
|
||||||
|
},
|
||||||
|
"reviewed_artifact_path_by_sample": {
|
||||||
|
sample_slug: str(artifact.resolve()) for sample_slug, artifact in artifacts.items()
|
||||||
|
},
|
||||||
|
"reviewed_artifact_sha256_by_sample": {
|
||||||
|
sample_slug: sha256(artifact) for sample_slug, artifact in artifacts.items()
|
||||||
|
},
|
||||||
|
}
|
||||||
|
audit = tmp_path / "belgium-building-corpus-audit.json"
|
||||||
|
audit.write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"status": "ok",
|
||||||
|
"manifest_immutable": True,
|
||||||
|
"spatial_leakage_status": "ok",
|
||||||
|
"review_complete": True,
|
||||||
|
"corpus_manifest_path": str(corpus_manifest.resolve()),
|
||||||
|
"corpus_manifest_sha256": sha256(corpus_manifest),
|
||||||
|
"human_review_evidence": evidence,
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
return audit
|
||||||
|
|
||||||
|
|
||||||
|
def test_valid_operational_release_binds_yaml_assets_corpus_and_human_review(tmp_path: Path) -> None:
|
||||||
|
corpus_manifest = write_corpus_manifest(tmp_path)
|
||||||
|
yaml_path = write_yolo_dataset(tmp_path)
|
||||||
|
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
|
||||||
|
|
||||||
|
paths = MODULE.create_training_release_manifest(
|
||||||
|
train_yaml=yaml_path,
|
||||||
|
corpus_manifest=corpus_manifest,
|
||||||
|
review_audit_path=review_audit,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert all(path.is_file() for path in paths.values())
|
||||||
|
assert MODULE.training_release_failures(
|
||||||
|
train_yaml=yaml_path,
|
||||||
|
corpus_manifest=corpus_manifest,
|
||||||
|
) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_tile_summary_must_be_an_exact_view_of_the_live_verified_release(tmp_path: Path) -> None:
|
||||||
|
corpus_manifest = write_corpus_manifest(tmp_path)
|
||||||
|
yaml_path = write_yolo_dataset(tmp_path)
|
||||||
|
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
|
||||||
|
paths = MODULE.create_training_release_manifest(
|
||||||
|
train_yaml=yaml_path,
|
||||||
|
corpus_manifest=corpus_manifest,
|
||||||
|
review_audit_path=review_audit,
|
||||||
|
)
|
||||||
|
assets = json.loads(paths["asset_manifest"].read_text(encoding="utf-8"))
|
||||||
|
summary = yaml_path.parent / "yolo_tile_dataset_summary.json"
|
||||||
|
summary.write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"dataset_yaml": str(yaml_path.resolve()),
|
||||||
|
"training_release_manifest": str(paths["release_manifest"].resolve()),
|
||||||
|
"training_release_manifest_sha256": sha256(paths["release_manifest"]),
|
||||||
|
"training_asset_manifest": str(paths["asset_manifest"].resolve()),
|
||||||
|
"source_manifest_sha256": sha256(corpus_manifest),
|
||||||
|
"tiles": [
|
||||||
|
{
|
||||||
|
"split": entry["split"],
|
||||||
|
"image_path": entry["image_path"],
|
||||||
|
"label_path": entry["label_path"],
|
||||||
|
}
|
||||||
|
for entry in assets["entries"]
|
||||||
|
],
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
MODULE.assert_yolo_summary_bound_to_training_release(
|
||||||
|
summary_path=summary,
|
||||||
|
train_yaml=yaml_path,
|
||||||
|
corpus_manifest=corpus_manifest,
|
||||||
|
)
|
||||||
|
payload = json.loads(summary.read_text(encoding="utf-8"))
|
||||||
|
payload["tiles"] = payload["tiles"][:1]
|
||||||
|
summary.write_text(json.dumps(payload), encoding="utf-8")
|
||||||
|
|
||||||
|
with pytest.raises(MODULE.TrainingReleaseError, match="complete immutable view"):
|
||||||
|
MODULE.assert_yolo_summary_bound_to_training_release(
|
||||||
|
summary_path=summary,
|
||||||
|
train_yaml=yaml_path,
|
||||||
|
corpus_manifest=corpus_manifest,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_unbound_or_changed_yaml_is_rejected_before_training(tmp_path: Path) -> None:
|
||||||
|
corpus_manifest = write_corpus_manifest(tmp_path)
|
||||||
|
yaml_path = write_yolo_dataset(tmp_path)
|
||||||
|
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
|
||||||
|
|
||||||
|
assert MODULE.training_release_failures(train_yaml=yaml_path) == ["training_release_manifest_missing"]
|
||||||
|
MODULE.create_training_release_manifest(
|
||||||
|
train_yaml=yaml_path,
|
||||||
|
corpus_manifest=corpus_manifest,
|
||||||
|
review_audit_path=review_audit,
|
||||||
|
)
|
||||||
|
yaml_path.write_text(yaml_path.read_text(encoding="utf-8") + "# tampered\n", encoding="utf-8")
|
||||||
|
|
||||||
|
assert "training_release_yaml_checksum_mismatch" in MODULE.training_release_failures(
|
||||||
|
train_yaml=yaml_path,
|
||||||
|
corpus_manifest=corpus_manifest,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_changed_label_asset_is_rejected_even_when_yaml_bytes_are_unchanged(tmp_path: Path) -> None:
|
||||||
|
corpus_manifest = write_corpus_manifest(tmp_path)
|
||||||
|
yaml_path = write_yolo_dataset(tmp_path)
|
||||||
|
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
|
||||||
|
MODULE.create_training_release_manifest(
|
||||||
|
train_yaml=yaml_path,
|
||||||
|
corpus_manifest=corpus_manifest,
|
||||||
|
review_audit_path=review_audit,
|
||||||
|
)
|
||||||
|
label = yaml_path.parent / "labels" / "train" / "fixture-train.txt"
|
||||||
|
label.write_text("0 0.4 0.4 0.2 0.2\n", encoding="utf-8")
|
||||||
|
|
||||||
|
failures = MODULE.training_release_failures(
|
||||||
|
train_yaml=yaml_path,
|
||||||
|
corpus_manifest=corpus_manifest,
|
||||||
|
)
|
||||||
|
assert "training_release_asset_manifest_content_mismatch" in failures
|
||||||
|
|
||||||
|
|
||||||
|
def test_operational_release_requires_complete_accepted_human_review(tmp_path: Path) -> None:
|
||||||
|
corpus_manifest = write_corpus_manifest(tmp_path)
|
||||||
|
yaml_path = write_yolo_dataset(tmp_path)
|
||||||
|
incomplete_audit = tmp_path / "audit.json"
|
||||||
|
incomplete_audit.write_text(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"status": "needs_human_review",
|
||||||
|
"manifest_immutable": True,
|
||||||
|
"spatial_leakage_status": "ok",
|
||||||
|
"review_complete": False,
|
||||||
|
}
|
||||||
|
),
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
MODULE.create_training_release_manifest(
|
||||||
|
train_yaml=yaml_path,
|
||||||
|
corpus_manifest=corpus_manifest,
|
||||||
|
review_audit_path=incomplete_audit,
|
||||||
|
)
|
||||||
|
except MODULE.TrainingReleaseError as exc:
|
||||||
|
assert "review_complete_not_true" in str(exc)
|
||||||
|
assert "accepted_human_review_evidence_missing" in str(exc)
|
||||||
|
else:
|
||||||
|
raise AssertionError("operational release accepted an incomplete human review")
|
||||||
|
|
||||||
|
|
||||||
|
def test_operational_release_rejects_tampered_accepted_review_artifact(tmp_path: Path) -> None:
|
||||||
|
corpus_manifest = write_corpus_manifest(tmp_path)
|
||||||
|
yaml_path = write_yolo_dataset(tmp_path)
|
||||||
|
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
|
||||||
|
artifact = tmp_path / "fixture-train-contact-sheet.png"
|
||||||
|
artifact.write_bytes(b"changed after review")
|
||||||
|
|
||||||
|
try:
|
||||||
|
MODULE.create_training_release_manifest(
|
||||||
|
train_yaml=yaml_path,
|
||||||
|
corpus_manifest=corpus_manifest,
|
||||||
|
review_audit_path=review_audit,
|
||||||
|
)
|
||||||
|
except MODULE.TrainingReleaseError as exc:
|
||||||
|
assert "reviewed_artifact_checksum_mismatch" in str(exc)
|
||||||
|
else:
|
||||||
|
raise AssertionError("tampered human-review artifact was accepted")
|
||||||
|
|
||||||
|
|
||||||
|
def test_fixture_relaxation_requires_explicit_fixture_corpus_and_is_not_operational(tmp_path: Path) -> None:
|
||||||
|
fixture_manifest = write_corpus_manifest(tmp_path, fixture_mode=True)
|
||||||
|
yaml_path = write_yolo_dataset(tmp_path)
|
||||||
|
MODULE.create_training_release_manifest(
|
||||||
|
train_yaml=yaml_path,
|
||||||
|
corpus_manifest=fixture_manifest,
|
||||||
|
fixture_mode=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert MODULE.training_release_failures(
|
||||||
|
train_yaml=yaml_path,
|
||||||
|
corpus_manifest=fixture_manifest,
|
||||||
|
fixture_mode=True,
|
||||||
|
) == []
|
||||||
|
assert "training_release_fixture_mode_mismatch" in MODULE.training_release_failures(
|
||||||
|
train_yaml=yaml_path,
|
||||||
|
corpus_manifest=fixture_manifest,
|
||||||
|
fixture_mode=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_release_contracts_a_reviewed_empty_label_as_explicit_pure_background(tmp_path: Path) -> None:
|
||||||
|
corpus_manifest = write_corpus_manifest(tmp_path)
|
||||||
|
yaml_path = write_yolo_dataset(tmp_path, empty_train_label=True)
|
||||||
|
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
|
||||||
|
|
||||||
|
paths = MODULE.create_training_release_manifest(
|
||||||
|
train_yaml=yaml_path,
|
||||||
|
corpus_manifest=corpus_manifest,
|
||||||
|
review_audit_path=review_audit,
|
||||||
|
)
|
||||||
|
|
||||||
|
labels = json.loads(paths["label_contract_manifest"].read_text(encoding="utf-8"))
|
||||||
|
assert labels["counts"]["pure_background"] == 1
|
||||||
|
assert MODULE.training_release_failures(train_yaml=yaml_path, corpus_manifest=corpus_manifest) == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_empty_label_without_accepted_sample_review_cannot_become_a_background_negative(tmp_path: Path) -> None:
|
||||||
|
corpus_manifest = write_corpus_manifest(tmp_path)
|
||||||
|
yaml_path = write_yolo_dataset(tmp_path, empty_train_label=True)
|
||||||
|
review_audit = write_accepted_review_audit(tmp_path, corpus_manifest)
|
||||||
|
audit_payload = json.loads(review_audit.read_text(encoding="utf-8"))
|
||||||
|
evidence = audit_payload["human_review_evidence"]
|
||||||
|
evidence["accepted_sample_slugs"] = ["fixture-val"]
|
||||||
|
review_audit.write_text(json.dumps(audit_payload), encoding="utf-8")
|
||||||
|
review = {
|
||||||
|
"status": "accepted",
|
||||||
|
"fixture_only": False,
|
||||||
|
"review_complete": True,
|
||||||
|
"evidence": evidence,
|
||||||
|
}
|
||||||
|
|
||||||
|
try:
|
||||||
|
MODULE.build_training_label_contract_manifest(
|
||||||
|
corpus_manifest_path=corpus_manifest,
|
||||||
|
asset_manifest=MODULE.build_yolo_asset_manifest(yaml_path),
|
||||||
|
review=review,
|
||||||
|
fixture_mode=False,
|
||||||
|
)
|
||||||
|
except MODULE.TrainingReleaseError as exc:
|
||||||
|
assert "Pure-background label sample was not accepted by review" in str(exc)
|
||||||
|
else:
|
||||||
|
raise AssertionError("unreviewed empty label was accepted as a background negative")
|
||||||
@@ -1,7 +1,6 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from types import SimpleNamespace
|
|
||||||
from uuid import UUID, uuid4
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
from app.core.errors import AppError
|
from app.core.errors import AppError
|
||||||
|
|||||||
@@ -128,6 +128,8 @@ COPY scripts/evaluate_belgium_building_candidate.py /app/scripts/evaluate_belgiu
|
|||||||
COPY scripts/assess_belgium_building_training_iteration.py /app/scripts/assess_belgium_building_training_iteration.py
|
COPY scripts/assess_belgium_building_training_iteration.py /app/scripts/assess_belgium_building_training_iteration.py
|
||||||
COPY scripts/run_belgium_building_training_loop.py /app/scripts/run_belgium_building_training_loop.py
|
COPY scripts/run_belgium_building_training_loop.py /app/scripts/run_belgium_building_training_loop.py
|
||||||
COPY scripts/build_failure_driven_yolo_sampling.py /app/scripts/build_failure_driven_yolo_sampling.py
|
COPY scripts/build_failure_driven_yolo_sampling.py /app/scripts/build_failure_driven_yolo_sampling.py
|
||||||
|
COPY scripts/training_dataset_eligibility.py /app/scripts/training_dataset_eligibility.py
|
||||||
|
COPY scripts/training_release_manifest.py /app/scripts/training_release_manifest.py
|
||||||
COPY scripts/build_grayscale_yolo_dataset.py /app/scripts/build_grayscale_yolo_dataset.py
|
COPY scripts/build_grayscale_yolo_dataset.py /app/scripts/build_grayscale_yolo_dataset.py
|
||||||
COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py
|
COPY scripts/audit_operator_yolo_dataset_quality.py /app/scripts/audit_operator_yolo_dataset_quality.py
|
||||||
COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py
|
COPY scripts/render_operator_yolo_label_qa_contact_sheets.py /app/scripts/render_operator_yolo_label_qa_contact_sheets.py
|
||||||
|
|||||||
+37
-4
@@ -393,15 +393,48 @@ Fields:
|
|||||||
|
|
||||||
- `file`: dataset file.
|
- `file`: dataset file.
|
||||||
- `dataset_type`: `vector`, `geojson` (legacy), `raster`.
|
- `dataset_type`: `vector`, `geojson` (legacy), `raster`.
|
||||||
- `source`: free text, e.g. `user_upload`, `grb`, `osm`.
|
- `source`: descriptive caller text, e.g. `user_upload`. It is retained as a
|
||||||
|
claim only and never establishes authority.
|
||||||
- `dataset_role`: `source`, `derived`, or `reference` (default `source`).
|
- `dataset_role`: `source`, `derived`, or `reference` (default `source`).
|
||||||
- `source_name`: optional source identity, e.g. `manual`, `grb`, `osm`; reference uploads default to `manual` when omitted.
|
- `source_name`: optional descriptive claim. Public uploads are always bound
|
||||||
|
to the server-owned `manual` registry entry, including when this field says
|
||||||
|
`grb`, `osm` or another official name. Reference uploads also default to
|
||||||
|
`manual`.
|
||||||
- `reference_layer_name`: optional reference layer label, e.g. `buildings`; only retained for reference datasets.
|
- `reference_layer_name`: optional reference layer label, e.g. `buildings`; only retained for reference datasets.
|
||||||
- `area_id`: optional.
|
- `area_id`: optional.
|
||||||
|
|
||||||
Response: `DatasetRead` with extracted metadata if supported.
|
Response: `DatasetRead` with extracted metadata if supported, plus
|
||||||
|
`source_registry_id`, `source_snapshot_id`, exact data-contract key/version,
|
||||||
|
validation report/status, provenance/lineage status, quarantine status and an
|
||||||
|
idempotent ingest key. A malformed or doubtful artifact is retained as
|
||||||
|
`status=quarantined`; it is not silently discarded or made ready.
|
||||||
|
|
||||||
Vector uploads remain stored as original files and are also persisted into `vector_features` as queryable PostGIS state.
|
Vector uploads remain stored as original files and are also persisted into
|
||||||
|
`vector_features` as queryable PostGIS state only after their contract passes.
|
||||||
|
Non-EPSG:4326 vector coordinates are explicitly transformed before canonical
|
||||||
|
feature persistence; relabelling Lambert coordinates as EPSG:4326 is rejected.
|
||||||
|
|
||||||
|
### GET `/api/v1/source-registry`
|
||||||
|
|
||||||
|
Lists server-owned source definitions. Optional query parameter
|
||||||
|
`classification` is one of `authoritative`, `corroborative`, `contextual`,
|
||||||
|
`derived`, or `experimental`. This endpoint reports policy and snapshot counts;
|
||||||
|
it does not assert that historical datasets carrying a matching text field are
|
||||||
|
trusted. It is operator-only because source snapshots can contain
|
||||||
|
operator-acquisition provenance. A demo session uses its project-scoped
|
||||||
|
dataset provenance endpoint instead.
|
||||||
|
|
||||||
|
### GET `/api/v1/source-registry/{source_key}`
|
||||||
|
|
||||||
|
Returns one source definition and its immutable snapshots, including authority
|
||||||
|
scope, licence/restrictions, expected geometry/attributes, freshness and known
|
||||||
|
limitations. Unknown source keys return `SOURCE_REGISTRY_ENTRY_NOT_FOUND`.
|
||||||
|
|
||||||
|
### GET `/api/v1/projects/{project_id}/datasets/{dataset_id}/provenance`
|
||||||
|
|
||||||
|
Returns the dataset's bound source/snapshot, contract report, lineage edges and
|
||||||
|
quarantine records. A missing binding is visible as incomplete provenance; it
|
||||||
|
is never backfilled from a display name or caller metadata.
|
||||||
|
|
||||||
### GET `/api/v1/projects/{project_id}/datasets/orthophoto/products`
|
### GET `/api/v1/projects/{project_id}/datasets/orthophoto/products`
|
||||||
|
|
||||||
|
|||||||
@@ -69,6 +69,41 @@ Spatial index required on `geometry`.
|
|||||||
- `status text default 'created'`
|
- `status text default 'created'`
|
||||||
- `created_at timestamptz`
|
- `created_at timestamptz`
|
||||||
|
|
||||||
|
### Source registry, snapshots and quarantine (Accuracy Phase 2)
|
||||||
|
|
||||||
|
Migration `202608010001_source_registry_provenance` adds a server-owned
|
||||||
|
provenance boundary without deleting or inventing facts for existing rows.
|
||||||
|
|
||||||
|
`source_registry` stores one governed source definition per `source_key`:
|
||||||
|
|
||||||
|
- classification (`authoritative`, `corroborative`, `contextual`, `derived`,
|
||||||
|
`experimental`), authority/scope and provider adapter key;
|
||||||
|
- licence, usage restrictions, expected CRS/units/resolution, temporal and
|
||||||
|
geographic coverage, expected geometry/attributes and known limitations;
|
||||||
|
- usage policy, freshness and ingest status.
|
||||||
|
|
||||||
|
`source_snapshots` stores immutable acquired editions keyed by
|
||||||
|
`(source_registry_id, snapshot_key)`, including source version/snapshot/fetch
|
||||||
|
time, lowercase SHA-256 checksum, CRS/units/resolution, coverage, observed
|
||||||
|
schema, freshness, ingest status and raw snapshot metadata.
|
||||||
|
|
||||||
|
`datasets` and `dataset_versions` receive `source_registry_id`,
|
||||||
|
`source_snapshot_id`, data-contract key/version, validation report/status,
|
||||||
|
provenance/lineage status and idempotent `ingest_key`. New governed records
|
||||||
|
must have a passing report before they become `ready`. Legacy rows are retained
|
||||||
|
with explicit incomplete/not-validated state; a migration never promotes them
|
||||||
|
based on a historical `source_name` string.
|
||||||
|
|
||||||
|
`dataset_lineage_edges` records parent/child Dataset(+Version), transform name
|
||||||
|
and version, parameters and input/output hashes. `dataset_quarantines` retains
|
||||||
|
the rejected dataset/version/snapshot, stage, reason, validation evidence and
|
||||||
|
artifact location. Quarantine is a stateful preservation record, not a delete.
|
||||||
|
|
||||||
|
The database constraints enforce accepted status vocabularies, nonblank ingest
|
||||||
|
keys, immutable snapshot identity/checksum format, unique ingest idempotency
|
||||||
|
keys and non-self lineage edges. Service-level contract validation remains
|
||||||
|
responsible for CRS, topology, bbox, units and source-task semantics.
|
||||||
|
|
||||||
### vector_features
|
### vector_features
|
||||||
|
|
||||||
Used for imported vector datasets and derived vector outputs when feature-level storage is needed. Original files remain source artifacts; this table is the queryable PostGIS state for vector features.
|
Used for imported vector datasets and derived vector outputs when feature-level storage is needed. Original files remain source artifacts; this table is the queryable PostGIS state for vector features.
|
||||||
|
|||||||
@@ -26,6 +26,10 @@ from app.db.session import SessionLocal # noqa: E402
|
|||||||
from app.models import Dataset # noqa: E402
|
from app.models import Dataset # noqa: E402
|
||||||
|
|
||||||
from normalize_belgium_building_labels import normalize # noqa: E402
|
from normalize_belgium_building_labels import normalize # noqa: E402
|
||||||
|
from training_dataset_eligibility import ( # noqa: E402
|
||||||
|
TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||||
|
training_pair_evidence,
|
||||||
|
)
|
||||||
|
|
||||||
REGION_SOURCES = {
|
REGION_SOURCES = {
|
||||||
"flanders": ({"digitaal_vlaanderen_orthophoto"}, "grb"),
|
"flanders": ({"digitaal_vlaanderen_orthophoto"}, "grb"),
|
||||||
@@ -52,7 +56,13 @@ def _dataset_path(dataset: Dataset) -> Path:
|
|||||||
return path
|
return path
|
||||||
|
|
||||||
|
|
||||||
def _validate_pair(sample: dict[str, Any], raster: Dataset, reference: Dataset) -> tuple[str, str]:
|
def _validate_pair(
|
||||||
|
sample: dict[str, Any],
|
||||||
|
raster: Dataset,
|
||||||
|
reference: Dataset,
|
||||||
|
*,
|
||||||
|
fixture_mode: bool,
|
||||||
|
) -> tuple[str, str, dict[str, Any]]:
|
||||||
region = str(sample.get("region") or "").lower()
|
region = str(sample.get("region") or "").lower()
|
||||||
if region not in REGION_SOURCES:
|
if region not in REGION_SOURCES:
|
||||||
raise SystemExit(f"Unsupported region for {sample.get('sample_slug')}: {region}")
|
raise SystemExit(f"Unsupported region for {sample.get('sample_slug')}: {region}")
|
||||||
@@ -66,7 +76,23 @@ def _validate_pair(sample: dict[str, Any], raster: Dataset, reference: Dataset)
|
|||||||
raise SystemExit(f"Unsupported split for {sample['sample_slug']}: {split}")
|
raise SystemExit(f"Unsupported split for {sample['sample_slug']}: {split}")
|
||||||
if raster.status != "ready" or reference.status != "ready":
|
if raster.status != "ready" or reference.status != "ready":
|
||||||
raise SystemExit(f"Dataset pair is not ready for {sample['sample_slug']}")
|
raise SystemExit(f"Dataset pair is not ready for {sample['sample_slug']}")
|
||||||
return region, expected_reference
|
eligibility = training_pair_evidence(
|
||||||
|
raster=raster,
|
||||||
|
reference=reference,
|
||||||
|
fixture_mode=fixture_mode,
|
||||||
|
)
|
||||||
|
if not eligibility["eligible"]:
|
||||||
|
reasons = sorted(
|
||||||
|
{
|
||||||
|
reason
|
||||||
|
for role in ("raster", "reference")
|
||||||
|
for reason in eligibility[role]["reasons"]
|
||||||
|
}
|
||||||
|
)
|
||||||
|
raise SystemExit(
|
||||||
|
f"Dataset pair is not eligible for training for {sample['sample_slug']}: {', '.join(reasons)}"
|
||||||
|
)
|
||||||
|
return region, expected_reference, eligibility
|
||||||
|
|
||||||
|
|
||||||
def audit_spatial_leakage(samples: list[dict[str, Any]], buffer_m: float = 64.0) -> dict[str, Any]:
|
def audit_spatial_leakage(samples: list[dict[str, Any]], buffer_m: float = 64.0) -> dict[str, Any]:
|
||||||
@@ -104,6 +130,14 @@ def main() -> int:
|
|||||||
parser.add_argument("--min-label-px", type=float, default=3.0)
|
parser.add_argument("--min-label-px", type=float, default=3.0)
|
||||||
parser.add_argument("--merge-touching-roofs", action="store_true")
|
parser.add_argument("--merge-touching-roofs", action="store_true")
|
||||||
parser.add_argument("--freeze", action="store_true")
|
parser.add_argument("--freeze", action="store_true")
|
||||||
|
parser.add_argument(
|
||||||
|
"--fixture-mode",
|
||||||
|
action="store_true",
|
||||||
|
help=(
|
||||||
|
"Allow only explicitly marked fixture datasets with legacy provenance. "
|
||||||
|
"Never use this mode for an operational corpus."
|
||||||
|
),
|
||||||
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
spec = json.loads(args.spec.read_text(encoding="utf-8-sig"))
|
spec = json.loads(args.spec.read_text(encoding="utf-8-sig"))
|
||||||
@@ -129,7 +163,12 @@ def main() -> int:
|
|||||||
reference = db.get(Dataset, UUID(str(sample["reference_dataset_id"])))
|
reference = db.get(Dataset, UUID(str(sample["reference_dataset_id"])))
|
||||||
if raster is None or reference is None:
|
if raster is None or reference is None:
|
||||||
raise SystemExit(f"Persisted Dataset pair not found for {slug}")
|
raise SystemExit(f"Persisted Dataset pair not found for {slug}")
|
||||||
region, reference_source = _validate_pair(sample, raster, reference)
|
region, reference_source, eligibility = _validate_pair(
|
||||||
|
sample,
|
||||||
|
raster,
|
||||||
|
reference,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
raster_source = _dataset_path(raster)
|
raster_source = _dataset_path(raster)
|
||||||
reference_source_path = _dataset_path(reference)
|
reference_source_path = _dataset_path(reference)
|
||||||
sample_dir = pairs_dir / slug
|
sample_dir = pairs_dir / slug
|
||||||
@@ -173,14 +212,20 @@ def main() -> int:
|
|||||||
"raster_sha256": sha256(raster_target),
|
"raster_sha256": sha256(raster_target),
|
||||||
"reference_sha256": sha256(normalized_target),
|
"reference_sha256": sha256(normalized_target),
|
||||||
"label_audit_sha256": sha256(audit_target),
|
"label_audit_sha256": sha256(audit_target),
|
||||||
|
"training_eligibility": eligibility,
|
||||||
"bbox_epsg4326": (raster.source_metadata or {}).get("bbox_epsg4326")
|
"bbox_epsg4326": (raster.source_metadata or {}).get("bbox_epsg4326")
|
||||||
or sample.get("bbox_epsg4326"),
|
or sample.get("bbox_epsg4326"),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
manifest = {
|
manifest = {
|
||||||
"schema_version": 1,
|
"schema_version": 2,
|
||||||
"dataset_version": args.version,
|
"dataset_version": args.version,
|
||||||
"immutable": bool(args.freeze),
|
"immutable": bool(args.freeze),
|
||||||
|
"training_eligibility": {
|
||||||
|
"policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||||
|
"status": "eligible",
|
||||||
|
"fixture_mode": bool(args.fixture_mode),
|
||||||
|
},
|
||||||
"samples": manifest_samples,
|
"samples": manifest_samples,
|
||||||
}
|
}
|
||||||
manifest_path = output_dir / "operator_samples_manifest.json"
|
manifest_path = output_dir / "operator_samples_manifest.json"
|
||||||
@@ -192,10 +237,13 @@ def main() -> int:
|
|||||||
if leakage_audit["status"] != "ok":
|
if leakage_audit["status"] != "ok":
|
||||||
raise SystemExit("Spatial split leakage audit failed")
|
raise SystemExit("Spatial split leakage audit failed")
|
||||||
freeze = {
|
freeze = {
|
||||||
|
"schema_version": 2,
|
||||||
"dataset_version": args.version,
|
"dataset_version": args.version,
|
||||||
"manifest_sha256": sha256(manifest_path),
|
"manifest_sha256": sha256(manifest_path),
|
||||||
"sample_count": len(manifest_samples),
|
"sample_count": len(manifest_samples),
|
||||||
"immutable": bool(args.freeze),
|
"immutable": bool(args.freeze),
|
||||||
|
"training_eligibility_policy": TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||||
|
"fixture_mode": bool(args.fixture_mode),
|
||||||
}
|
}
|
||||||
(output_dir / "corpus-freeze.json").write_text(json.dumps(freeze, indent=2), encoding="utf-8")
|
(output_dir / "corpus-freeze.json").write_text(json.dumps(freeze, indent=2), encoding="utf-8")
|
||||||
print(json.dumps(freeze))
|
print(json.dumps(freeze))
|
||||||
|
|||||||
@@ -4,13 +4,49 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import hashlib
|
||||||
import json
|
import json
|
||||||
|
import sys
|
||||||
from collections import Counter
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||||
|
if str(SCRIPT_DIR) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SCRIPT_DIR))
|
||||||
|
|
||||||
|
from training_dataset_eligibility import frozen_manifest_training_eligibility_failures # noqa: E402
|
||||||
|
|
||||||
REQUIRED_SPLITS = ("train", "val", "calibration", "test", "background-test")
|
REQUIRED_SPLITS = ("train", "val", "calibration", "test", "background-test")
|
||||||
REGIONS = ("flanders", "wallonia", "brussels")
|
REGIONS = ("flanders", "wallonia", "brussels")
|
||||||
|
SHA256_LENGTH = 64
|
||||||
|
|
||||||
|
|
||||||
|
def sha256(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def valid_review_timestamp(value: Any) -> bool:
|
||||||
|
if not isinstance(value, str) or not value.strip():
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
parsed = datetime.fromisoformat(value.strip().replace("Z", "+00:00"))
|
||||||
|
except ValueError:
|
||||||
|
return False
|
||||||
|
return parsed.tzinfo is not None
|
||||||
|
|
||||||
|
|
||||||
|
def valid_sha256(value: Any) -> bool:
|
||||||
|
return (
|
||||||
|
isinstance(value, str)
|
||||||
|
and len(value) == SHA256_LENGTH
|
||||||
|
and all(character in "0123456789abcdefABCDEF" for character in value)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
@@ -19,8 +55,15 @@ def main() -> int:
|
|||||||
parser.add_argument("--output-dir", type=Path, required=True)
|
parser.add_argument("--output-dir", type=Path, required=True)
|
||||||
parser.add_argument("--review-decisions", type=Path)
|
parser.add_argument("--review-decisions", type=Path)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
manifest = json.loads((args.corpus_dir / "operator_samples_manifest.json").read_text(encoding="utf-8"))
|
manifest_path = args.corpus_dir / "operator_samples_manifest.json"
|
||||||
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
leakage = json.loads((args.corpus_dir / "spatial-leakage-audit.json").read_text(encoding="utf-8"))
|
leakage = json.loads((args.corpus_dir / "spatial-leakage-audit.json").read_text(encoding="utf-8"))
|
||||||
|
manifest_policy = manifest.get("training_eligibility")
|
||||||
|
fixture_mode = bool(manifest_policy.get("fixture_mode")) if isinstance(manifest_policy, dict) else False
|
||||||
|
eligibility_failures = frozen_manifest_training_eligibility_failures(
|
||||||
|
manifest_path,
|
||||||
|
fixture_mode=fixture_mode,
|
||||||
|
)
|
||||||
samples = manifest["samples"]
|
samples = manifest["samples"]
|
||||||
split_counts = Counter((sample["region"], sample["split"]) for sample in samples)
|
split_counts = Counter((sample["region"], sample["split"]) for sample in samples)
|
||||||
decision_counts: Counter[str] = Counter()
|
decision_counts: Counter[str] = Counter()
|
||||||
@@ -28,7 +71,7 @@ def main() -> int:
|
|||||||
total_input = 0
|
total_input = 0
|
||||||
total_accepted = 0
|
total_accepted = 0
|
||||||
temporal_unknown = 0
|
temporal_unknown = 0
|
||||||
failures: list[str] = []
|
failures: list[str] = list(eligibility_failures)
|
||||||
for region in REGIONS:
|
for region in REGIONS:
|
||||||
for split in REQUIRED_SPLITS:
|
for split in REQUIRED_SPLITS:
|
||||||
minimum = 4 if split == "train" else 2
|
minimum = 4 if split == "train" else 2
|
||||||
@@ -62,8 +105,12 @@ def main() -> int:
|
|||||||
failures.append("spatial leakage audit failed")
|
failures.append("spatial leakage audit failed")
|
||||||
reviewed = 0
|
reviewed = 0
|
||||||
review_complete = False
|
review_complete = False
|
||||||
|
review_evidence_failures: list[str] = []
|
||||||
|
accepted_review_evidence: dict[str, dict[str, Any]] = {}
|
||||||
|
review_decisions_path: Path | None = None
|
||||||
if args.review_decisions and args.review_decisions.is_file():
|
if args.review_decisions and args.review_decisions.is_file():
|
||||||
decisions = json.loads(args.review_decisions.read_text(encoding="utf-8"))
|
review_decisions_path = args.review_decisions.resolve(strict=False)
|
||||||
|
decisions = json.loads(review_decisions_path.read_text(encoding="utf-8"))
|
||||||
by_slug = {item["sample_slug"]: item for item in decisions.get("decisions", [])}
|
by_slug = {item["sample_slug"]: item for item in decisions.get("decisions", [])}
|
||||||
for item in review_queue:
|
for item in review_queue:
|
||||||
decision = by_slug.get(item["sample_slug"])
|
decision = by_slug.get(item["sample_slug"])
|
||||||
@@ -71,13 +118,70 @@ def main() -> int:
|
|||||||
item["decision"] = decision.get("decision")
|
item["decision"] = decision.get("decision")
|
||||||
item["reviewer"] = decision.get("reviewer")
|
item["reviewer"] = decision.get("reviewer")
|
||||||
item["notes"] = decision.get("notes")
|
item["notes"] = decision.get("notes")
|
||||||
if item["decision"] in {"accepted", "rejected"} and item.get("reviewer"):
|
item["reviewed_at"] = decision.get("reviewed_at")
|
||||||
|
item["reviewed_artifact_path"] = decision.get("reviewed_artifact_path")
|
||||||
|
item["reviewed_artifact_sha256"] = decision.get("reviewed_artifact_sha256")
|
||||||
|
reviewed_artifact = (
|
||||||
|
Path(item["reviewed_artifact_path"]).expanduser().resolve(strict=False)
|
||||||
|
if isinstance(item.get("reviewed_artifact_path"), str)
|
||||||
|
and item["reviewed_artifact_path"].strip()
|
||||||
|
else None
|
||||||
|
)
|
||||||
|
has_evidence = (
|
||||||
|
item["decision"] == "accepted"
|
||||||
|
and isinstance(item.get("reviewer"), str)
|
||||||
|
and bool(item["reviewer"].strip())
|
||||||
|
and valid_review_timestamp(item.get("reviewed_at"))
|
||||||
|
and isinstance(item.get("reviewed_artifact_path"), str)
|
||||||
|
and bool(item["reviewed_artifact_path"].strip())
|
||||||
|
and valid_sha256(item.get("reviewed_artifact_sha256"))
|
||||||
|
and reviewed_artifact is not None
|
||||||
|
and reviewed_artifact.is_file()
|
||||||
|
and item["reviewed_artifact_sha256"] == sha256(reviewed_artifact)
|
||||||
|
)
|
||||||
|
if has_evidence:
|
||||||
reviewed += 1
|
reviewed += 1
|
||||||
|
accepted_review_evidence[str(item["sample_slug"])] = {
|
||||||
|
"reviewer": item["reviewer"].strip(),
|
||||||
|
"reviewed_at": item["reviewed_at"],
|
||||||
|
"reviewed_artifact_path": item["reviewed_artifact_path"],
|
||||||
|
"reviewed_artifact_sha256": item["reviewed_artifact_sha256"],
|
||||||
|
}
|
||||||
|
elif item["decision"] in {"accepted", "rejected"}:
|
||||||
|
review_evidence_failures.append(
|
||||||
|
f"{item['sample_slug']}:accepted review lacks reviewer/timestamp/artifact evidence"
|
||||||
|
)
|
||||||
review_complete = reviewed == len(review_queue) and all(item["decision"] == "accepted" for item in review_queue)
|
review_complete = reviewed == len(review_queue) and all(item["decision"] == "accepted" for item in review_queue)
|
||||||
|
human_review_evidence: dict[str, Any] | None = None
|
||||||
|
if review_decisions_path is not None:
|
||||||
|
human_review_evidence = {
|
||||||
|
"review_decisions_path": str(review_decisions_path),
|
||||||
|
"review_decisions_sha256": sha256(review_decisions_path),
|
||||||
|
"required_sample_count": len(review_queue),
|
||||||
|
"accepted_sample_count": reviewed,
|
||||||
|
"accepted_sample_slugs": sorted(accepted_review_evidence),
|
||||||
|
"reviewer_ids": sorted(
|
||||||
|
{item["reviewer"] for item in accepted_review_evidence.values()}
|
||||||
|
),
|
||||||
|
"reviewed_at_by_sample": {
|
||||||
|
slug: accepted_review_evidence[slug]["reviewed_at"]
|
||||||
|
for slug in sorted(accepted_review_evidence)
|
||||||
|
},
|
||||||
|
"reviewed_artifact_path_by_sample": {
|
||||||
|
slug: accepted_review_evidence[slug]["reviewed_artifact_path"]
|
||||||
|
for slug in sorted(accepted_review_evidence)
|
||||||
|
},
|
||||||
|
"reviewed_artifact_sha256_by_sample": {
|
||||||
|
slug: accepted_review_evidence[slug]["reviewed_artifact_sha256"]
|
||||||
|
for slug in sorted(accepted_review_evidence)
|
||||||
|
},
|
||||||
|
}
|
||||||
status = "failed" if failures else ("ok" if review_complete else "needs_human_review")
|
status = "failed" if failures else ("ok" if review_complete else "needs_human_review")
|
||||||
report = {
|
report = {
|
||||||
"status": status,
|
"status": status,
|
||||||
"dataset_version": manifest["dataset_version"],
|
"dataset_version": manifest["dataset_version"],
|
||||||
|
"corpus_manifest_path": str(manifest_path.resolve(strict=False)),
|
||||||
|
"corpus_manifest_sha256": sha256(manifest_path),
|
||||||
"manifest_immutable": manifest["immutable"],
|
"manifest_immutable": manifest["immutable"],
|
||||||
"sample_count": len(samples),
|
"sample_count": len(samples),
|
||||||
"split_counts": {f"{region}/{split}": split_counts[(region, split)] for region in REGIONS for split in REQUIRED_SPLITS},
|
"split_counts": {f"{region}/{split}": split_counts[(region, split)] for region in REGIONS for split in REQUIRED_SPLITS},
|
||||||
@@ -86,8 +190,13 @@ def main() -> int:
|
|||||||
"decision_counts": dict(sorted(decision_counts.items())),
|
"decision_counts": dict(sorted(decision_counts.items())),
|
||||||
"temporal_unknown_sample_count": temporal_unknown,
|
"temporal_unknown_sample_count": temporal_unknown,
|
||||||
"spatial_leakage_status": leakage.get("status"),
|
"spatial_leakage_status": leakage.get("status"),
|
||||||
|
"training_eligibility_status": manifest_policy.get("status") if isinstance(manifest_policy, dict) else None,
|
||||||
|
"training_eligibility_fixture_mode": fixture_mode,
|
||||||
|
"training_eligibility_failures": eligibility_failures,
|
||||||
"reviewed_sample_count": reviewed,
|
"reviewed_sample_count": reviewed,
|
||||||
"review_complete": review_complete,
|
"review_complete": review_complete,
|
||||||
|
"human_review_evidence": human_review_evidence,
|
||||||
|
"review_evidence_failures": sorted(set(review_evidence_failures)),
|
||||||
"failures": failures,
|
"failures": failures,
|
||||||
"review_queue": review_queue,
|
"review_queue": review_queue,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import json
|
|||||||
import sys
|
import sys
|
||||||
from collections import Counter
|
from collections import Counter
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any, Mapping
|
||||||
|
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
|
|
||||||
@@ -17,10 +17,21 @@ SCRIPT_DIR = Path(__file__).resolve().parent
|
|||||||
if str(SCRIPT_DIR) not in sys.path:
|
if str(SCRIPT_DIR) not in sys.path:
|
||||||
sys.path.insert(0, str(SCRIPT_DIR))
|
sys.path.insert(0, str(SCRIPT_DIR))
|
||||||
|
|
||||||
from evaluate_belgium_building_candidate import iou, read_references
|
from evaluate_belgium_building_candidate import iou, read_references # noqa: E402
|
||||||
|
from training_dataset_eligibility import ( # noqa: E402
|
||||||
|
TrainingEligibilityError,
|
||||||
|
assert_frozen_manifest_training_eligible,
|
||||||
|
)
|
||||||
|
from training_release_manifest import ( # noqa: E402
|
||||||
|
TrainingReleaseError,
|
||||||
|
assert_yolo_summary_bound_to_embedded_training_release,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
PROTECTED_SPLITS = {"calibration", "test", "background-test"}
|
PROTECTED_SPLITS = {"calibration", "test", "background-test"}
|
||||||
|
PROPOSAL_DATASET_PROVENANCE_NAME = "proposal-dataset-provenance.json"
|
||||||
|
PROPOSAL_DATASET_EVIDENCE_NAME = "proposal-dataset-evidence.json"
|
||||||
|
PROPOSAL_DATASET_SCHEMA_VERSION = 1
|
||||||
|
|
||||||
|
|
||||||
def sha256(path: Path) -> str:
|
def sha256(path: Path) -> str:
|
||||||
@@ -31,6 +42,125 @@ def sha256(path: Path) -> str:
|
|||||||
return digest.hexdigest()
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical_json_bytes(payload: Mapping[str, Any]) -> bytes:
|
||||||
|
return json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _immutable_payload_sha256(payload: Mapping[str, Any], *, field: str = "manifest_sha256") -> str:
|
||||||
|
normalized = dict(payload)
|
||||||
|
normalized.pop(field, None)
|
||||||
|
return hashlib.sha256(_canonical_json_bytes(normalized)).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _write_immutable_json(path: Path, payload: Mapping[str, Any]) -> None:
|
||||||
|
"""Persist one immutable sidecar without silently replacing prior evidence."""
|
||||||
|
|
||||||
|
encoded = json.dumps(dict(payload), ensure_ascii=False, indent=2, sort_keys=True) + "\n"
|
||||||
|
if path.exists():
|
||||||
|
if path.read_text(encoding="utf-8") != encoded:
|
||||||
|
raise RuntimeError(f"immutable provenance artifact already exists with different content: {path}")
|
||||||
|
return
|
||||||
|
path.write_text(encoded, encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def assert_summary_source_manifest_binding(summary: Mapping[str, Any], corpus_manifest_path: Path) -> str:
|
||||||
|
"""Require the proposal summary to name the exact frozen corpus bytes."""
|
||||||
|
|
||||||
|
expected = sha256(corpus_manifest_path)
|
||||||
|
observed = summary.get("source_manifest_sha256")
|
||||||
|
if observed != expected:
|
||||||
|
raise ValueError(
|
||||||
|
"proposal source summary is not bound to the supplied governed corpus manifest "
|
||||||
|
f"(expected {expected}, observed {observed!r})"
|
||||||
|
)
|
||||||
|
return expected
|
||||||
|
|
||||||
|
|
||||||
|
def load_governed_corpus_manifest(corpus_manifest_path: Path, *, fixture_mode: bool) -> dict[str, Any]:
|
||||||
|
"""Validate frozen corpus evidence and re-check the live Dataset state.
|
||||||
|
|
||||||
|
This must run before importing Ultralytics/PyTorch so a revocation cannot
|
||||||
|
consume GPU work or create proposal crops.
|
||||||
|
"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
manifest = assert_frozen_manifest_training_eligible(
|
||||||
|
corpus_manifest_path,
|
||||||
|
fixture_mode=fixture_mode,
|
||||||
|
verify_live=True,
|
||||||
|
)
|
||||||
|
except TrainingEligibilityError as exc:
|
||||||
|
raise ValueError(str(exc)) from exc
|
||||||
|
if not isinstance(manifest, dict): # pragma: no cover - defensive contract boundary
|
||||||
|
raise ValueError("governed corpus manifest must be a JSON object")
|
||||||
|
return manifest
|
||||||
|
|
||||||
|
|
||||||
|
def _source_file_evidence(tile: Mapping[str, Any], *, field: str) -> tuple[Path, str]:
|
||||||
|
raw_path = tile.get(field)
|
||||||
|
if not isinstance(raw_path, str) or not raw_path.strip():
|
||||||
|
raise ValueError(f"proposal source tile has no {field}: {tile.get('sample_slug')!r}")
|
||||||
|
path = Path(raw_path).expanduser().resolve(strict=False)
|
||||||
|
if not path.is_file():
|
||||||
|
raise ValueError(f"proposal source tile {field} is unavailable: {path}")
|
||||||
|
return path, sha256(path)
|
||||||
|
|
||||||
|
|
||||||
|
def _source_tile_key(tile: Mapping[str, Any]) -> tuple[str, str, str]:
|
||||||
|
"""Return the canonical source identity used for the pre-inference cache."""
|
||||||
|
|
||||||
|
return (
|
||||||
|
str(tile.get("sample_slug") or ""),
|
||||||
|
str(Path(str(tile.get("image_path") or "")).expanduser().resolve(strict=False)),
|
||||||
|
str(Path(str(tile.get("label_path") or "")).expanduser().resolve(strict=False)),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_file_unchanged(path: Path, expected_sha256: str, *, role: str) -> None:
|
||||||
|
"""Reject a mutable source changing between evidence capture and use."""
|
||||||
|
|
||||||
|
if sha256(path) != expected_sha256:
|
||||||
|
raise RuntimeError(f"proposal {role} changed after evidence capture: {path}")
|
||||||
|
|
||||||
|
|
||||||
|
def _crop_entry(
|
||||||
|
*,
|
||||||
|
output_dir: Path,
|
||||||
|
crop_path: Path,
|
||||||
|
tile: Mapping[str, Any],
|
||||||
|
label: str,
|
||||||
|
proposal_index: int,
|
||||||
|
proposal_score: float,
|
||||||
|
source_box_xyxy: tuple[float, float, float, float],
|
||||||
|
source_image_path: Path,
|
||||||
|
source_image_sha256: str,
|
||||||
|
source_label_path: Path,
|
||||||
|
source_label_sha256: str,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
relative_path = crop_path.resolve(strict=False).relative_to(output_dir.resolve(strict=False)).as_posix()
|
||||||
|
sample_slug = tile.get("sample_slug")
|
||||||
|
split = tile.get("split")
|
||||||
|
if not isinstance(sample_slug, str) or not sample_slug.strip():
|
||||||
|
raise ValueError("proposal source tile has no sample_slug")
|
||||||
|
if split not in {"train", "val"}:
|
||||||
|
raise ValueError(f"proposal crop has unsupported split: {split!r}")
|
||||||
|
return {
|
||||||
|
"relative_path": relative_path,
|
||||||
|
"sha256": sha256(crop_path),
|
||||||
|
"size_bytes": crop_path.stat().st_size,
|
||||||
|
"split": split,
|
||||||
|
"label": label,
|
||||||
|
"sample_slug": sample_slug,
|
||||||
|
"proposal_index": proposal_index,
|
||||||
|
"proposal_score": proposal_score,
|
||||||
|
"source_box_xyxy": [float(value) for value in source_box_xyxy],
|
||||||
|
"source_image_path": str(source_image_path),
|
||||||
|
"source_image_sha256": source_image_sha256,
|
||||||
|
"source_label_path": str(source_label_path),
|
||||||
|
"source_label_sha256": source_label_sha256,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def classify_proposals(
|
def classify_proposals(
|
||||||
predictions: list[tuple[tuple[float, float, float, float], float]],
|
predictions: list[tuple[tuple[float, float, float, float], float]],
|
||||||
references: list[tuple[float, float, float, float]],
|
references: list[tuple[float, float, float, float]],
|
||||||
@@ -99,29 +229,74 @@ def main() -> int:
|
|||||||
parser.add_argument("--max-negative-per-tile", type=int, default=24)
|
parser.add_argument("--max-negative-per-tile", type=int, default=24)
|
||||||
parser.add_argument("--device", default="cuda:0")
|
parser.add_argument("--device", default="cuda:0")
|
||||||
parser.add_argument("--imgsz", type=int, default=640)
|
parser.add_argument("--imgsz", type=int, default=640)
|
||||||
|
parser.add_argument(
|
||||||
|
"--fixture-mode",
|
||||||
|
action="store_true",
|
||||||
|
help="Accept only an explicitly fixture-only corpus manifest; never use for operational training data.",
|
||||||
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
if args.output_dir.exists():
|
if args.output_dir.exists():
|
||||||
parser.error(f"output already exists: {args.output_dir}")
|
parser.error(f"output already exists: {args.output_dir}")
|
||||||
|
|
||||||
from ultralytics import YOLO
|
try:
|
||||||
|
summary = json.loads(args.summary.read_text(encoding="utf-8"))
|
||||||
summary = json.loads(args.summary.read_text(encoding="utf-8"))
|
except (OSError, json.JSONDecodeError) as exc:
|
||||||
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
|
raise SystemExit(f"proposal source summary is unreadable: {args.summary}") from exc
|
||||||
|
if not isinstance(summary, dict):
|
||||||
|
raise SystemExit("proposal source summary must be a JSON object")
|
||||||
|
try:
|
||||||
|
source_release = assert_yolo_summary_bound_to_embedded_training_release(
|
||||||
|
summary_path=args.summary,
|
||||||
|
corpus_manifest=args.corpus_manifest,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
|
manifest = load_governed_corpus_manifest(args.corpus_manifest, fixture_mode=args.fixture_mode)
|
||||||
|
corpus_manifest_sha256 = assert_summary_source_manifest_binding(summary, args.corpus_manifest)
|
||||||
|
except (ValueError, TrainingReleaseError) as exc:
|
||||||
|
raise SystemExit(str(exc)) from exc
|
||||||
tiles = eligible_tiles(summary, manifest, args.region)
|
tiles = eligible_tiles(summary, manifest, args.region)
|
||||||
|
model_path = args.model.expanduser().resolve(strict=False)
|
||||||
|
if not model_path.is_file():
|
||||||
|
raise SystemExit(f"proposal model is unavailable: {model_path}")
|
||||||
|
model_sha256 = sha256(model_path)
|
||||||
|
|
||||||
|
# Hash every input image/label before GPU inference. A crop sidecar then
|
||||||
|
# binds each generated JPEG to the exact source tile rather than its
|
||||||
|
# filename or an unverified aggregate summary.
|
||||||
|
source_evidence: dict[tuple[str, str, str], tuple[Path, str, Path, str]] = {}
|
||||||
|
for tile in tiles:
|
||||||
|
image_path, image_sha256 = _source_file_evidence(tile, field="image_path")
|
||||||
|
label_path, label_sha256 = _source_file_evidence(tile, field="label_path")
|
||||||
|
key = _source_tile_key(tile)
|
||||||
|
source_evidence[key] = (image_path, image_sha256, label_path, label_sha256)
|
||||||
|
|
||||||
args.output_dir.mkdir(parents=True)
|
args.output_dir.mkdir(parents=True)
|
||||||
counts: Counter[str] = Counter()
|
counts: Counter[str] = Counter()
|
||||||
sample_counts: Counter[str] = Counter()
|
sample_counts: Counter[str] = Counter()
|
||||||
model = YOLO(str(args.model))
|
crop_entries: list[dict[str, Any]] = []
|
||||||
|
|
||||||
|
# Importing the model is deliberately after the source-manifest and live
|
||||||
|
# Dataset checks above. A revoked corpus must not start GPU work.
|
||||||
|
from ultralytics import YOLO
|
||||||
|
|
||||||
|
model = YOLO(str(model_path))
|
||||||
for start in range(0, len(tiles), 16):
|
for start in range(0, len(tiles), 16):
|
||||||
batch_tiles = tiles[start : start + 16]
|
batch_tiles = tiles[start : start + 16]
|
||||||
|
batch_sources = [source_evidence[_source_tile_key(tile)] for tile in batch_tiles]
|
||||||
|
for image_path, image_sha256, label_path, label_sha256 in batch_sources:
|
||||||
|
_assert_file_unchanged(image_path, image_sha256, role="source image")
|
||||||
|
_assert_file_unchanged(label_path, label_sha256, role="source label")
|
||||||
results = model.predict(
|
results = model.predict(
|
||||||
[tile["image_path"] for tile in batch_tiles], conf=args.confidence,
|
[str(source[0]) for source in batch_sources], conf=args.confidence,
|
||||||
device=args.device, imgsz=args.imgsz, max_det=1000, iou=0.7, verbose=False,
|
device=args.device, imgsz=args.imgsz, max_det=1000, iou=0.7, verbose=False,
|
||||||
)
|
)
|
||||||
for tile, result in zip(batch_tiles, results, strict=True):
|
for tile, result in zip(batch_tiles, results, strict=True):
|
||||||
with Image.open(tile["image_path"]) as opened:
|
image_path, image_sha256, label_path, label_sha256 = source_evidence[_source_tile_key(tile)]
|
||||||
|
_assert_file_unchanged(image_path, image_sha256, role="source image")
|
||||||
|
_assert_file_unchanged(label_path, label_sha256, role="source label")
|
||||||
|
with Image.open(image_path) as opened:
|
||||||
source = opened.convert("RGB")
|
source = opened.convert("RGB")
|
||||||
references = read_references(Path(tile["label_path"]), source.width, source.height)
|
references = read_references(label_path, source.width, source.height)
|
||||||
proposals = [
|
proposals = [
|
||||||
(tuple(map(float, box)), float(score))
|
(tuple(map(float, box)), float(score))
|
||||||
for box, score in zip(
|
for box, score in zip(
|
||||||
@@ -139,25 +314,113 @@ def main() -> int:
|
|||||||
target_dir = args.output_dir / split / label
|
target_dir = args.output_dir / split / label
|
||||||
target_dir.mkdir(parents=True, exist_ok=True)
|
target_dir.mkdir(parents=True, exist_ok=True)
|
||||||
name = f"{tile['sample_slug']}__{Path(tile['image_path']).stem}__{proposal_index:04d}.jpg"
|
name = f"{tile['sample_slug']}__{Path(tile['image_path']).stem}__{proposal_index:04d}.jpg"
|
||||||
crop_square(source, box, args.crop_scale).save(target_dir / name, quality=92)
|
crop_path = target_dir / name
|
||||||
|
if crop_path.exists():
|
||||||
|
raise RuntimeError(f"proposal crop identity collision: {crop_path}")
|
||||||
|
crop_square(source, box, args.crop_scale).save(crop_path, quality=92)
|
||||||
|
crop_entries.append(
|
||||||
|
_crop_entry(
|
||||||
|
output_dir=args.output_dir,
|
||||||
|
crop_path=crop_path,
|
||||||
|
tile=tile,
|
||||||
|
label=label,
|
||||||
|
proposal_index=proposal_index,
|
||||||
|
proposal_score=score,
|
||||||
|
source_box_xyxy=box,
|
||||||
|
source_image_path=image_path,
|
||||||
|
source_image_sha256=image_sha256,
|
||||||
|
source_label_path=label_path,
|
||||||
|
source_label_sha256=label_sha256,
|
||||||
|
)
|
||||||
|
)
|
||||||
counts[f"{split}/{label}"] += 1
|
counts[f"{split}/{label}"] += 1
|
||||||
sample_counts[tile["sample_slug"]] += 1
|
sample_counts[tile["sample_slug"]] += 1
|
||||||
for split in ("train", "val"):
|
for split in ("train", "val"):
|
||||||
for label in ("positive", "negative"):
|
for label in ("positive", "negative"):
|
||||||
if counts[f"{split}/{label}"] == 0:
|
if counts[f"{split}/{label}"] == 0:
|
||||||
raise RuntimeError(f"empty proposal class: {split}/{label}")
|
raise RuntimeError(f"empty proposal class: {split}/{label}")
|
||||||
|
|
||||||
|
crop_entries.sort(key=lambda item: str(item["relative_path"]))
|
||||||
|
if len({str(item["relative_path"]) for item in crop_entries}) != len(crop_entries):
|
||||||
|
raise RuntimeError("proposal crop manifest contains duplicate relative paths")
|
||||||
|
source_summary_sha256 = sha256(args.summary)
|
||||||
|
corpus_freeze_path = args.corpus_manifest.parent / "corpus-freeze.json"
|
||||||
|
if not corpus_freeze_path.is_file(): # guarded above; retain a local invariant for provenance output
|
||||||
|
raise RuntimeError(f"governed corpus freeze is unavailable: {corpus_freeze_path}")
|
||||||
|
provenance: dict[str, Any] = {
|
||||||
|
"schema_version": PROPOSAL_DATASET_SCHEMA_VERSION,
|
||||||
|
"status": "ok",
|
||||||
|
"immutable": True,
|
||||||
|
"dataset_kind": "building_proposal_classifier_crops",
|
||||||
|
"fixture_mode": bool(args.fixture_mode),
|
||||||
|
"governed_corpus_live_recheck": True,
|
||||||
|
"source": {
|
||||||
|
"corpus_manifest": {
|
||||||
|
"path": str(args.corpus_manifest.expanduser().resolve(strict=False)),
|
||||||
|
"sha256": corpus_manifest_sha256,
|
||||||
|
},
|
||||||
|
"corpus_freeze": {
|
||||||
|
"path": str(corpus_freeze_path.resolve(strict=False)),
|
||||||
|
"sha256": sha256(corpus_freeze_path),
|
||||||
|
},
|
||||||
|
"summary": {
|
||||||
|
"path": str(args.summary.expanduser().resolve(strict=False)),
|
||||||
|
"sha256": source_summary_sha256,
|
||||||
|
"source_manifest_sha256": summary["source_manifest_sha256"],
|
||||||
|
"training_release_manifest": summary["training_release_manifest"],
|
||||||
|
"training_release_manifest_sha256": summary["training_release_manifest_sha256"],
|
||||||
|
"training_asset_manifest": summary["training_asset_manifest"],
|
||||||
|
},
|
||||||
|
"training_release": {
|
||||||
|
"dataset_yaml_path": source_release["dataset_yaml"]["path"],
|
||||||
|
"dataset_yaml_sha256": source_release["dataset_yaml"]["sha256"],
|
||||||
|
"corpus_manifest_sha256": source_release["corpus"]["manifest_sha256"],
|
||||||
|
},
|
||||||
|
"proposal_model": {
|
||||||
|
"path": str(model_path),
|
||||||
|
"sha256": model_sha256,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"parameters": {
|
||||||
|
"region": args.region,
|
||||||
|
"confidence": args.confidence,
|
||||||
|
"match_iou": args.match_iou,
|
||||||
|
"crop_scale": args.crop_scale,
|
||||||
|
"max_positive_per_tile": args.max_positive_per_tile,
|
||||||
|
"max_negative_per_tile": args.max_negative_per_tile,
|
||||||
|
"device": args.device,
|
||||||
|
"imgsz": args.imgsz,
|
||||||
|
},
|
||||||
|
"counts": dict(sorted(counts.items())),
|
||||||
|
"sample_counts": dict(sorted(sample_counts.items())),
|
||||||
|
"tile_count": len(tiles),
|
||||||
|
"crop_count": len(crop_entries),
|
||||||
|
"crops_sha256": hashlib.sha256(_canonical_json_bytes({"crops": crop_entries})).hexdigest(),
|
||||||
|
"crops": crop_entries,
|
||||||
|
}
|
||||||
|
provenance["manifest_sha256"] = _immutable_payload_sha256(provenance)
|
||||||
|
provenance_path = args.output_dir / PROPOSAL_DATASET_PROVENANCE_NAME
|
||||||
|
_write_immutable_json(provenance_path, provenance)
|
||||||
|
|
||||||
evidence = {
|
evidence = {
|
||||||
"schema_version": 1, "status": "ok", "model": str(args.model),
|
"schema_version": PROPOSAL_DATASET_SCHEMA_VERSION,
|
||||||
"model_sha256": sha256(args.model), "summary": str(args.summary),
|
"status": "ok",
|
||||||
"summary_sha256": sha256(args.summary), "corpus_manifest": str(args.corpus_manifest),
|
"model": str(model_path),
|
||||||
"corpus_manifest_sha256": sha256(args.corpus_manifest), "region": args.region,
|
"model_sha256": model_sha256,
|
||||||
|
"summary": str(args.summary),
|
||||||
|
"summary_sha256": source_summary_sha256,
|
||||||
|
"corpus_manifest": str(args.corpus_manifest),
|
||||||
|
"corpus_manifest_sha256": corpus_manifest_sha256,
|
||||||
|
"region": args.region,
|
||||||
"confidence": args.confidence, "match_iou": args.match_iou, "crop_scale": args.crop_scale,
|
"confidence": args.confidence, "match_iou": args.match_iou, "crop_scale": args.crop_scale,
|
||||||
"counts": dict(sorted(counts.items())), "sample_counts": dict(sorted(sample_counts.items())),
|
"counts": dict(sorted(counts.items())), "sample_counts": dict(sorted(sample_counts.items())),
|
||||||
"protected_samples_in_training": [], "tile_count": len(tiles),
|
"protected_samples_in_training": [], "tile_count": len(tiles),
|
||||||
|
"fixture_mode": bool(args.fixture_mode),
|
||||||
|
"proposal_dataset_provenance": str(provenance_path),
|
||||||
|
"proposal_dataset_provenance_sha256": sha256(provenance_path),
|
||||||
|
"proposal_dataset_manifest_sha256": provenance["manifest_sha256"],
|
||||||
}
|
}
|
||||||
(args.output_dir / "proposal-dataset-evidence.json").write_text(
|
_write_immutable_json(args.output_dir / PROPOSAL_DATASET_EVIDENCE_NAME, evidence)
|
||||||
json.dumps(evidence, indent=2), encoding="utf-8"
|
|
||||||
)
|
|
||||||
print(json.dumps(evidence, indent=2))
|
print(json.dumps(evidence, indent=2))
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|||||||
@@ -8,10 +8,25 @@ import hashlib
|
|||||||
import json
|
import json
|
||||||
import math
|
import math
|
||||||
import re
|
import re
|
||||||
|
import sys
|
||||||
from collections import Counter
|
from collections import Counter
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||||
|
if str(SCRIPT_DIR) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SCRIPT_DIR))
|
||||||
|
|
||||||
|
from training_dataset_eligibility import ( # noqa: E402
|
||||||
|
TrainingEligibilityError,
|
||||||
|
assert_frozen_manifest_training_eligible,
|
||||||
|
)
|
||||||
|
from training_release_manifest import ( # noqa: E402
|
||||||
|
TrainingReleaseError,
|
||||||
|
assert_yolo_summary_bound_to_embedded_training_release,
|
||||||
|
create_training_release_manifest,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
PRECISION_NEGATIVE_CONTEXTS = {
|
PRECISION_NEGATIVE_CONTEXTS = {
|
||||||
"coastal-urban": {"port-hard-negative", "dunes-negative"},
|
"coastal-urban": {"port-hard-negative", "dunes-negative"},
|
||||||
@@ -80,6 +95,17 @@ def build_sampling(
|
|||||||
) -> tuple[list[str], dict[str, Any]]:
|
) -> tuple[list[str], dict[str, Any]]:
|
||||||
if assessment.get("status") != "continue_training_loop":
|
if assessment.get("status") != "continue_training_loop":
|
||||||
raise ValueError("Failure-driven sampling requires a failed assessment")
|
raise ValueError("Failure-driven sampling requires a failed assessment")
|
||||||
|
protected_feedback = [
|
||||||
|
role
|
||||||
|
for role in ("test", "background")
|
||||||
|
if assessment.get(role) is not None
|
||||||
|
]
|
||||||
|
if protected_feedback:
|
||||||
|
raise ValueError(
|
||||||
|
"Failure-driven sampling is prohibited after protected "
|
||||||
|
+ "/".join(protected_feedback)
|
||||||
|
+ " evidence was opened"
|
||||||
|
)
|
||||||
if min(
|
if min(
|
||||||
positive_repeat,
|
positive_repeat,
|
||||||
negative_repeat,
|
negative_repeat,
|
||||||
@@ -97,9 +123,9 @@ def build_sampling(
|
|||||||
|
|
||||||
samples = {item["sample_slug"]: item for item in manifest["samples"]}
|
samples = {item["sample_slug"]: item for item in manifest["samples"]}
|
||||||
gates = assessment["gates"]
|
gates = assessment["gates"]
|
||||||
evaluation = assessment.get("test") or assessment.get("calibration")
|
evaluation = assessment.get("calibration")
|
||||||
if not evaluation or "regions" not in evaluation:
|
if not evaluation or "regions" not in evaluation:
|
||||||
raise ValueError("Assessment has no regional calibration or test evidence")
|
raise ValueError("Assessment has no regional calibration evidence")
|
||||||
regions = evaluation["regions"]
|
regions = evaluation["regions"]
|
||||||
weak_recall_regions = {
|
weak_recall_regions = {
|
||||||
region
|
region
|
||||||
@@ -246,7 +272,7 @@ def build_sampling(
|
|||||||
"schema_version": 1,
|
"schema_version": 1,
|
||||||
"status": "ok",
|
"status": "ok",
|
||||||
"strategy": "failed-region-positive-and-hard-negative-repeat",
|
"strategy": "failed-region-positive-and-hard-negative-repeat",
|
||||||
"failure_evidence_source": "test" if assessment.get("test") else "calibration",
|
"failure_evidence_source": "calibration",
|
||||||
"weak_recall_regions": sorted(weak_recall_regions),
|
"weak_recall_regions": sorted(weak_recall_regions),
|
||||||
"weak_precision_regions": sorted(weak_precision_regions),
|
"weak_precision_regions": sorted(weak_precision_regions),
|
||||||
"recall_dominant_regions": sorted(recall_dominant_regions),
|
"recall_dominant_regions": sorted(recall_dominant_regions),
|
||||||
@@ -290,6 +316,11 @@ def main() -> int:
|
|||||||
parser.add_argument("--corpus-manifest", type=Path, required=True)
|
parser.add_argument("--corpus-manifest", type=Path, required=True)
|
||||||
parser.add_argument("--assessment", type=Path, required=True)
|
parser.add_argument("--assessment", type=Path, required=True)
|
||||||
parser.add_argument("--output-dir", type=Path, required=True)
|
parser.add_argument("--output-dir", type=Path, required=True)
|
||||||
|
parser.add_argument(
|
||||||
|
"--review-audit",
|
||||||
|
type=Path,
|
||||||
|
help="Passed corpus audit containing accepted human-review evidence for the frozen corpus.",
|
||||||
|
)
|
||||||
parser.add_argument("--positive-repeat", type=int, default=3)
|
parser.add_argument("--positive-repeat", type=int, default=3)
|
||||||
parser.add_argument("--negative-repeat", type=int, default=4)
|
parser.add_argument("--negative-repeat", type=int, default=4)
|
||||||
parser.add_argument("--precision-positive-repeat", type=int, default=1)
|
parser.add_argument("--precision-positive-repeat", type=int, default=1)
|
||||||
@@ -299,8 +330,26 @@ def main() -> int:
|
|||||||
parser.add_argument("--sampling-round", type=int)
|
parser.add_argument("--sampling-round", type=int)
|
||||||
parser.add_argument("--precision-guard-band", type=float, default=0.03)
|
parser.add_argument("--precision-guard-band", type=float, default=0.03)
|
||||||
parser.add_argument("--recall-guard-band", type=float, default=0.03)
|
parser.add_argument("--recall-guard-band", type=float, default=0.03)
|
||||||
|
parser.add_argument(
|
||||||
|
"--fixture-mode",
|
||||||
|
action="store_true",
|
||||||
|
help="Accept only an explicitly fixture-only corpus manifest; never use for operational sampling.",
|
||||||
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
assert_frozen_manifest_training_eligible(
|
||||||
|
args.corpus_manifest,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
verify_live=True,
|
||||||
|
)
|
||||||
|
source_release = assert_yolo_summary_bound_to_embedded_training_release(
|
||||||
|
summary_path=args.summary,
|
||||||
|
corpus_manifest=args.corpus_manifest,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
|
except (TrainingEligibilityError, TrainingReleaseError) as exc:
|
||||||
|
raise SystemExit(str(exc)) from exc
|
||||||
summary = json.loads(args.summary.read_text(encoding="utf-8"))
|
summary = json.loads(args.summary.read_text(encoding="utf-8"))
|
||||||
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
|
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
|
||||||
assessment = json.loads(args.assessment.read_text(encoding="utf-8"))
|
assessment = json.loads(args.assessment.read_text(encoding="utf-8"))
|
||||||
@@ -325,7 +374,7 @@ def main() -> int:
|
|||||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
train_list = args.output_dir / "train-failure-driven.txt"
|
train_list = args.output_dir / "train-failure-driven.txt"
|
||||||
train_list.write_text("\n".join(paths) + "\n", encoding="utf-8")
|
train_list.write_text("\n".join(paths) + "\n", encoding="utf-8")
|
||||||
source_yaml = args.summary.parent / "dataset.yaml"
|
source_yaml = Path(str(source_release["dataset_yaml"]["path"]))
|
||||||
val_source = dataset_validation_source(source_yaml)
|
val_source = dataset_validation_source(source_yaml)
|
||||||
dataset_yaml = args.output_dir / "dataset.yaml"
|
dataset_yaml = args.output_dir / "dataset.yaml"
|
||||||
dataset_yaml.write_text(
|
dataset_yaml.write_text(
|
||||||
@@ -335,6 +384,15 @@ def main() -> int:
|
|||||||
"names:\n 0: building\n",
|
"names:\n 0: building\n",
|
||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
)
|
)
|
||||||
|
try:
|
||||||
|
release_paths = create_training_release_manifest(
|
||||||
|
train_yaml=dataset_yaml,
|
||||||
|
corpus_manifest=args.corpus_manifest,
|
||||||
|
review_audit_path=args.review_audit,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
|
except TrainingReleaseError as exc:
|
||||||
|
raise SystemExit(str(exc)) from exc
|
||||||
metadata.update(
|
metadata.update(
|
||||||
{
|
{
|
||||||
"summary": str(args.summary),
|
"summary": str(args.summary),
|
||||||
@@ -346,6 +404,11 @@ def main() -> int:
|
|||||||
"source_dataset_yaml": str(source_yaml),
|
"source_dataset_yaml": str(source_yaml),
|
||||||
"train_list": str(train_list),
|
"train_list": str(train_list),
|
||||||
"dataset_yaml": str(dataset_yaml),
|
"dataset_yaml": str(dataset_yaml),
|
||||||
|
"training_release_manifest": str(release_paths["release_manifest"]),
|
||||||
|
"training_release_manifest_sha256": file_sha256(release_paths["release_manifest"]),
|
||||||
|
"training_release_freeze": str(release_paths["release_freeze"]),
|
||||||
|
"training_asset_manifest": str(release_paths["asset_manifest"]),
|
||||||
|
"fixture_mode": bool(args.fixture_mode),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
output = args.output_dir / "failure-driven-sampling.json"
|
output = args.output_dir / "failure-driven-sampling.json"
|
||||||
|
|||||||
@@ -7,11 +7,24 @@ import argparse
|
|||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import shutil
|
import shutil
|
||||||
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
|
|
||||||
|
|
||||||
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||||
|
if str(SCRIPT_DIR) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SCRIPT_DIR))
|
||||||
|
|
||||||
|
from training_release_manifest import ( # noqa: E402
|
||||||
|
TrainingReleaseError,
|
||||||
|
assert_yolo_summary_bound_to_training_release,
|
||||||
|
file_sha256,
|
||||||
|
training_release_paths,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def sha256(path: Path) -> str:
|
def sha256(path: Path) -> str:
|
||||||
digest = hashlib.sha256()
|
digest = hashlib.sha256()
|
||||||
with path.open("rb") as stream:
|
with path.open("rb") as stream:
|
||||||
@@ -23,9 +36,25 @@ def sha256(path: Path) -> str:
|
|||||||
def main() -> int:
|
def main() -> int:
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
parser.add_argument("--summary", type=Path, required=True)
|
parser.add_argument("--summary", type=Path, required=True)
|
||||||
|
parser.add_argument("--train-yaml", type=Path, required=True)
|
||||||
|
parser.add_argument("--corpus-manifest", type=Path, required=True)
|
||||||
parser.add_argument("--output-dir", type=Path, required=True)
|
parser.add_argument("--output-dir", type=Path, required=True)
|
||||||
|
parser.add_argument(
|
||||||
|
"--fixture-mode",
|
||||||
|
action="store_true",
|
||||||
|
help="Accept only an explicitly fixture-only release; never creates a production-ready derived dataset.",
|
||||||
|
)
|
||||||
parser.add_argument("--force", action="store_true")
|
parser.add_argument("--force", action="store_true")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
try:
|
||||||
|
release = assert_yolo_summary_bound_to_training_release(
|
||||||
|
summary_path=args.summary,
|
||||||
|
train_yaml=args.train_yaml,
|
||||||
|
corpus_manifest=args.corpus_manifest,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
|
except TrainingReleaseError as exc:
|
||||||
|
raise SystemExit(str(exc)) from exc
|
||||||
if args.output_dir.exists():
|
if args.output_dir.exists():
|
||||||
if not args.force:
|
if not args.force:
|
||||||
raise SystemExit(f"Output exists: {args.output_dir}")
|
raise SystemExit(f"Output exists: {args.output_dir}")
|
||||||
@@ -54,6 +83,16 @@ def main() -> int:
|
|||||||
f"path: {args.output_dir}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
|
f"path: {args.output_dir}\ntrain: images/train\nval: images/val\nnames:\n 0: building\n",
|
||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
)
|
)
|
||||||
|
# The source release binds the original image bytes. These transformed
|
||||||
|
# bytes cannot inherit it, so remove its operational release pointers and
|
||||||
|
# retain them only as explicitly non-trainable parent evidence below.
|
||||||
|
for field_name in (
|
||||||
|
"training_release_manifest",
|
||||||
|
"training_release_manifest_sha256",
|
||||||
|
"training_release_freeze",
|
||||||
|
"training_asset_manifest",
|
||||||
|
):
|
||||||
|
summary.pop(field_name, None)
|
||||||
summary.update(
|
summary.update(
|
||||||
{
|
{
|
||||||
"output_dir": str(args.output_dir),
|
"output_dir": str(args.output_dir),
|
||||||
@@ -71,10 +110,19 @@ def main() -> int:
|
|||||||
"preprocessing": "luminance_rgb_replicated",
|
"preprocessing": "luminance_rgb_replicated",
|
||||||
"source_summary": str(args.summary),
|
"source_summary": str(args.summary),
|
||||||
"source_summary_sha256": sha256(args.summary),
|
"source_summary_sha256": sha256(args.summary),
|
||||||
|
"source_training_release": str(training_release_paths(args.train_yaml)["release_manifest"]),
|
||||||
|
"source_training_release_sha256": file_sha256(
|
||||||
|
training_release_paths(args.train_yaml)["release_manifest"]
|
||||||
|
),
|
||||||
|
"source_corpus_manifest_sha256": release["corpus"]["manifest_sha256"],
|
||||||
"converted_tile_count": converted,
|
"converted_tile_count": converted,
|
||||||
"output_summary": str(output_summary),
|
"output_summary": str(output_summary),
|
||||||
"output_summary_sha256": sha256(output_summary),
|
"output_summary_sha256": sha256(output_summary),
|
||||||
"dataset_yaml": str(dataset_yaml),
|
"dataset_yaml": str(dataset_yaml),
|
||||||
|
"training_eligible": False,
|
||||||
|
"training_eligibility_reason": (
|
||||||
|
"Derived image bytes require a new governed corpus, validation report and immutable training release."
|
||||||
|
),
|
||||||
}
|
}
|
||||||
(args.output_dir / "grayscale-dataset-evidence.json").write_text(
|
(args.output_dir / "grayscale-dataset-evidence.json").write_text(
|
||||||
json.dumps(evidence, indent=2), encoding="utf-8"
|
json.dumps(evidence, indent=2), encoding="utf-8"
|
||||||
|
|||||||
@@ -6,10 +6,25 @@ from __future__ import annotations
|
|||||||
import argparse
|
import argparse
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
|
import sys
|
||||||
from collections import Counter
|
from collections import Counter
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||||
|
if str(SCRIPT_DIR) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SCRIPT_DIR))
|
||||||
|
|
||||||
|
from training_dataset_eligibility import ( # noqa: E402
|
||||||
|
TrainingEligibilityError,
|
||||||
|
assert_frozen_manifest_training_eligible,
|
||||||
|
)
|
||||||
|
from training_release_manifest import ( # noqa: E402
|
||||||
|
TrainingReleaseError,
|
||||||
|
assert_yolo_summary_bound_to_embedded_training_release,
|
||||||
|
create_training_release_manifest,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
PROTECTED_SPLITS = {"calibration", "test", "background-test"}
|
PROTECTED_SPLITS = {"calibration", "test", "background-test"}
|
||||||
|
|
||||||
@@ -138,11 +153,34 @@ def main() -> int:
|
|||||||
parser.add_argument("--priority-context", action="append", default=[])
|
parser.add_argument("--priority-context", action="append", default=[])
|
||||||
parser.add_argument("--priority-repeat", type=int, default=2)
|
parser.add_argument("--priority-repeat", type=int, default=2)
|
||||||
parser.add_argument("--negative-repeat", type=int, default=2)
|
parser.add_argument("--negative-repeat", type=int, default=2)
|
||||||
|
parser.add_argument(
|
||||||
|
"--review-audit",
|
||||||
|
type=Path,
|
||||||
|
help="Accepted corpus-audit evidence; mandatory for an operational training release.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--fixture-mode",
|
||||||
|
action="store_true",
|
||||||
|
help="Accept only an explicitly fixture-only corpus manifest; never use for operational training data.",
|
||||||
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
if args.priority_repeat < 1 or args.negative_repeat < 1:
|
if args.priority_repeat < 1 or args.negative_repeat < 1:
|
||||||
parser.error("repeat counts must be positive")
|
parser.error("repeat counts must be positive")
|
||||||
summary = json.loads(args.summary.read_text(encoding="utf-8"))
|
summary = json.loads(args.summary.read_text(encoding="utf-8"))
|
||||||
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
|
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
|
||||||
|
try:
|
||||||
|
assert_frozen_manifest_training_eligible(
|
||||||
|
args.corpus_manifest,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
verify_live=True,
|
||||||
|
)
|
||||||
|
assert_yolo_summary_bound_to_embedded_training_release(
|
||||||
|
summary_path=args.summary,
|
||||||
|
corpus_manifest=args.corpus_manifest,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
|
except (TrainingEligibilityError, TrainingReleaseError) as exc:
|
||||||
|
raise SystemExit(str(exc)) from exc
|
||||||
train, val, evidence = build(
|
train, val, evidence = build(
|
||||||
summary=summary,
|
summary=summary,
|
||||||
manifest=manifest,
|
manifest=manifest,
|
||||||
@@ -161,6 +199,15 @@ def main() -> int:
|
|||||||
f"path: /\ntrain: {train_path}\nval: {val_path}\nnames:\n 0: building\n",
|
f"path: /\ntrain: {train_path}\nval: {val_path}\nnames:\n 0: building\n",
|
||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
)
|
)
|
||||||
|
try:
|
||||||
|
release_paths = create_training_release_manifest(
|
||||||
|
train_yaml=yaml_path,
|
||||||
|
corpus_manifest=args.corpus_manifest,
|
||||||
|
review_audit_path=args.review_audit,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
|
except TrainingReleaseError as exc:
|
||||||
|
raise SystemExit(str(exc)) from exc
|
||||||
evidence.update({
|
evidence.update({
|
||||||
"source_summary": str(args.summary),
|
"source_summary": str(args.summary),
|
||||||
"source_summary_sha256": sha256(args.summary),
|
"source_summary_sha256": sha256(args.summary),
|
||||||
@@ -169,6 +216,11 @@ def main() -> int:
|
|||||||
"train_sha256": sha256(train_path),
|
"train_sha256": sha256(train_path),
|
||||||
"validation_sha256": sha256(val_path),
|
"validation_sha256": sha256(val_path),
|
||||||
"dataset_yaml": str(yaml_path),
|
"dataset_yaml": str(yaml_path),
|
||||||
|
"training_release_manifest": str(release_paths["release_manifest"]),
|
||||||
|
"training_release_manifest_sha256": sha256(release_paths["release_manifest"]),
|
||||||
|
"training_release_freeze": str(release_paths["release_freeze"]),
|
||||||
|
"training_asset_manifest": str(release_paths["asset_manifest"]),
|
||||||
|
"fixture_mode": bool(args.fixture_mode),
|
||||||
})
|
})
|
||||||
encoded_evidence = json.dumps(evidence, indent=2)
|
encoded_evidence = json.dumps(evidence, indent=2)
|
||||||
(args.output_dir / "regional-dataset-evidence.json").write_text(encoded_evidence, encoding="utf-8")
|
(args.output_dir / "regional-dataset-evidence.json").write_text(encoded_evidence, encoding="utf-8")
|
||||||
|
|||||||
@@ -6,8 +6,23 @@ from __future__ import annotations
|
|||||||
import argparse
|
import argparse
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||||
|
if str(SCRIPT_DIR) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SCRIPT_DIR))
|
||||||
|
|
||||||
|
from training_dataset_eligibility import ( # noqa: E402
|
||||||
|
TrainingEligibilityError,
|
||||||
|
assert_frozen_manifest_training_eligible,
|
||||||
|
)
|
||||||
|
from training_release_manifest import ( # noqa: E402
|
||||||
|
TrainingReleaseError,
|
||||||
|
assert_yolo_summary_bound_to_embedded_training_release,
|
||||||
|
create_training_release_manifest,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def sha256(path: Path) -> str:
|
def sha256(path: Path) -> str:
|
||||||
digest = hashlib.sha256()
|
digest = hashlib.sha256()
|
||||||
@@ -31,12 +46,35 @@ def main() -> int:
|
|||||||
default=0,
|
default=0,
|
||||||
help="Include each train tile outside the expert region this many times.",
|
help="Include each train tile outside the expert region this many times.",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--fixture-mode",
|
||||||
|
action="store_true",
|
||||||
|
help="Accept only an explicitly fixture-only corpus manifest; never use for operational training data.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--review-audit",
|
||||||
|
type=Path,
|
||||||
|
help="Accepted corpus-audit evidence; mandatory for an operational training release.",
|
||||||
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
if args.positive_repeat < 1 or args.negative_repeat < 1 or args.other_region_repeat < 0:
|
if args.positive_repeat < 1 or args.negative_repeat < 1 or args.other_region_repeat < 0:
|
||||||
raise SystemExit("regional repeats must be positive and other-region repeat non-negative")
|
raise SystemExit("regional repeats must be positive and other-region repeat non-negative")
|
||||||
|
|
||||||
summary = json.loads(args.summary.read_text(encoding="utf-8"))
|
summary = json.loads(args.summary.read_text(encoding="utf-8"))
|
||||||
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
|
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
|
||||||
|
try:
|
||||||
|
assert_frozen_manifest_training_eligible(
|
||||||
|
args.corpus_manifest,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
verify_live=True,
|
||||||
|
)
|
||||||
|
assert_yolo_summary_bound_to_embedded_training_release(
|
||||||
|
summary_path=args.summary,
|
||||||
|
corpus_manifest=args.corpus_manifest,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
|
except (TrainingEligibilityError, TrainingReleaseError) as exc:
|
||||||
|
raise SystemExit(str(exc)) from exc
|
||||||
samples = {item["sample_slug"]: item for item in manifest["samples"]}
|
samples = {item["sample_slug"]: item for item in manifest["samples"]}
|
||||||
paths: list[str] = []
|
paths: list[str] = []
|
||||||
selected_samples: set[str] = set()
|
selected_samples: set[str] = set()
|
||||||
@@ -69,6 +107,15 @@ def main() -> int:
|
|||||||
f"val: {args.summary.parent / 'images' / 'val'}\nnames:\n 0: building\n",
|
f"val: {args.summary.parent / 'images' / 'val'}\nnames:\n 0: building\n",
|
||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
)
|
)
|
||||||
|
try:
|
||||||
|
release_paths = create_training_release_manifest(
|
||||||
|
train_yaml=dataset_yaml,
|
||||||
|
corpus_manifest=args.corpus_manifest,
|
||||||
|
review_audit_path=args.review_audit,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
|
except TrainingReleaseError as exc:
|
||||||
|
raise SystemExit(str(exc)) from exc
|
||||||
evidence = {
|
evidence = {
|
||||||
"schema_version": 1,
|
"schema_version": 1,
|
||||||
"status": "ok",
|
"status": "ok",
|
||||||
@@ -88,6 +135,11 @@ def main() -> int:
|
|||||||
"protected_samples_in_training": [],
|
"protected_samples_in_training": [],
|
||||||
"train_list": str(train_list),
|
"train_list": str(train_list),
|
||||||
"dataset_yaml": str(dataset_yaml),
|
"dataset_yaml": str(dataset_yaml),
|
||||||
|
"training_release_manifest": str(release_paths["release_manifest"]),
|
||||||
|
"training_release_manifest_sha256": sha256(release_paths["release_manifest"]),
|
||||||
|
"training_release_freeze": str(release_paths["release_freeze"]),
|
||||||
|
"training_asset_manifest": str(release_paths["asset_manifest"]),
|
||||||
|
"fixture_mode": bool(args.fixture_mode),
|
||||||
}
|
}
|
||||||
evidence_path = args.output_dir / "regional-expert-dataset.json"
|
evidence_path = args.output_dir / "regional-expert-dataset.json"
|
||||||
evidence_path.write_text(json.dumps(evidence, indent=2), encoding="utf-8")
|
evidence_path.write_text(json.dumps(evidence, indent=2), encoding="utf-8")
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ new provider data.
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
@@ -16,6 +17,19 @@ import sys
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Iterable
|
from typing import Any, Iterable
|
||||||
|
|
||||||
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||||
|
if str(SCRIPT_DIR) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SCRIPT_DIR))
|
||||||
|
|
||||||
|
from training_dataset_eligibility import ( # noqa: E402
|
||||||
|
TrainingEligibilityError,
|
||||||
|
assert_frozen_manifest_training_eligible,
|
||||||
|
)
|
||||||
|
from training_release_manifest import ( # noqa: E402
|
||||||
|
TrainingReleaseError,
|
||||||
|
create_training_release_manifest,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/operator_samples_manifest.json")
|
DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/operator_samples_manifest.json")
|
||||||
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-dataset")
|
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-dataset")
|
||||||
@@ -24,6 +38,14 @@ Transformer: Any = None
|
|||||||
Image: Any = None
|
Image: Any = None
|
||||||
|
|
||||||
|
|
||||||
|
def file_sha256(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
def parse_args() -> argparse.Namespace:
|
def parse_args() -> argparse.Namespace:
|
||||||
parser = argparse.ArgumentParser(
|
parser = argparse.ArgumentParser(
|
||||||
description="Export operator real-data samples to a YOLO detection dataset.",
|
description="Export operator real-data samples to a YOLO detection dataset.",
|
||||||
@@ -45,6 +67,12 @@ def parse_args() -> argparse.Namespace:
|
|||||||
default=os.environ.get("OPERATOR_YOLO_VAL_SAMPLES", "turnhout"),
|
default=os.environ.get("OPERATOR_YOLO_VAL_SAMPLES", "turnhout"),
|
||||||
help="Comma/space separated sample slugs assigned to validation. Defaults to turnhout.",
|
help="Comma/space separated sample slugs assigned to validation. Defaults to turnhout.",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--review-audit",
|
||||||
|
type=Path,
|
||||||
|
required=True,
|
||||||
|
help="Passed corpus audit with accepted human-review evidence for this frozen source manifest.",
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--force",
|
"--force",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
@@ -218,12 +246,16 @@ def ensure_yolo_directories(output_dir: Path) -> None:
|
|||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
args = parse_args()
|
args = parse_args()
|
||||||
|
manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
|
||||||
|
try:
|
||||||
|
assert_frozen_manifest_training_eligible(args.manifest_path, verify_live=True)
|
||||||
|
except TrainingEligibilityError as exc:
|
||||||
|
raise SystemExit(str(exc)) from exc
|
||||||
ensure_dependencies()
|
ensure_dependencies()
|
||||||
if args.force and args.output_dir.exists():
|
if args.force and args.output_dir.exists():
|
||||||
shutil.rmtree(args.output_dir)
|
shutil.rmtree(args.output_dir)
|
||||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
ensure_yolo_directories(args.output_dir)
|
ensure_yolo_directories(args.output_dir)
|
||||||
manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
|
|
||||||
samples = manifest.get("samples") or []
|
samples = manifest.get("samples") or []
|
||||||
if not samples:
|
if not samples:
|
||||||
raise SystemExit("Operator sample manifest contains no samples")
|
raise SystemExit("Operator sample manifest contains no samples")
|
||||||
@@ -234,12 +266,26 @@ def main() -> int:
|
|||||||
if not any(item["split"] == "val" for item in exported):
|
if not any(item["split"] == "val" for item in exported):
|
||||||
raise SystemExit("YOLO dataset export produced no validation samples")
|
raise SystemExit("YOLO dataset export produced no validation samples")
|
||||||
dataset_yaml = write_dataset_yaml(args.output_dir)
|
dataset_yaml = write_dataset_yaml(args.output_dir)
|
||||||
|
try:
|
||||||
|
release_paths = create_training_release_manifest(
|
||||||
|
train_yaml=dataset_yaml,
|
||||||
|
corpus_manifest=args.manifest_path,
|
||||||
|
review_audit_path=args.review_audit,
|
||||||
|
)
|
||||||
|
except TrainingReleaseError as exc:
|
||||||
|
raise SystemExit(str(exc)) from exc
|
||||||
summary = {
|
summary = {
|
||||||
"status": "ok",
|
"status": "ok",
|
||||||
"dataset_yaml": str(dataset_yaml),
|
"dataset_yaml": str(dataset_yaml),
|
||||||
|
"training_release_manifest": str(release_paths["release_manifest"]),
|
||||||
|
"training_release_manifest_sha256": file_sha256(release_paths["release_manifest"]),
|
||||||
|
"training_release_freeze": str(release_paths["release_freeze"]),
|
||||||
|
"training_asset_manifest": str(release_paths["asset_manifest"]),
|
||||||
"output_dir": str(args.output_dir),
|
"output_dir": str(args.output_dir),
|
||||||
"class_names": ["building"],
|
"class_names": ["building"],
|
||||||
"sample_count": len(exported),
|
"sample_count": len(exported),
|
||||||
|
"source_manifest": str(args.manifest_path),
|
||||||
|
"source_manifest_sha256": file_sha256(args.manifest_path),
|
||||||
"train_sample_count": sum(1 for item in exported if item["split"] == "train"),
|
"train_sample_count": sum(1 for item in exported if item["split"] == "train"),
|
||||||
"val_sample_count": sum(1 for item in exported if item["split"] == "val"),
|
"val_sample_count": sum(1 for item in exported if item["split"] == "val"),
|
||||||
"label_count": sum(item["label_count"] for item in exported),
|
"label_count": sum(item["label_count"] for item in exported),
|
||||||
|
|||||||
@@ -18,6 +18,19 @@ import sys
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Iterable
|
from typing import Any, Iterable
|
||||||
|
|
||||||
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||||
|
if str(SCRIPT_DIR) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SCRIPT_DIR))
|
||||||
|
|
||||||
|
from training_dataset_eligibility import ( # noqa: E402
|
||||||
|
TrainingEligibilityError,
|
||||||
|
assert_frozen_manifest_training_eligible,
|
||||||
|
)
|
||||||
|
from training_release_manifest import ( # noqa: E402
|
||||||
|
TrainingReleaseError,
|
||||||
|
create_training_release_manifest,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/operator_samples_manifest.json")
|
DEFAULT_MANIFEST_PATH = Path("/app/storage/operator-data/operator_samples_manifest.json")
|
||||||
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-tile-dataset")
|
DEFAULT_OUTPUT_DIR = Path("/app/storage/operator-data/yolo-building-tile-dataset")
|
||||||
@@ -45,6 +58,14 @@ Transformer: Any = None
|
|||||||
Image: Any = None
|
Image: Any = None
|
||||||
|
|
||||||
|
|
||||||
|
def file_sha256(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
class TileWindow:
|
class TileWindow:
|
||||||
row_off: int
|
row_off: int
|
||||||
@@ -87,6 +108,12 @@ def parse_args() -> argparse.Namespace:
|
|||||||
default=os.environ.get("OPERATOR_YOLO_REFERENCE_SOURCE", DEFAULT_REFERENCE_SOURCE),
|
default=os.environ.get("OPERATOR_YOLO_REFERENCE_SOURCE", DEFAULT_REFERENCE_SOURCE),
|
||||||
help="Required source_name in reference GeoJSON features.",
|
help="Required source_name in reference GeoJSON features.",
|
||||||
)
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--review-audit",
|
||||||
|
type=Path,
|
||||||
|
required=True,
|
||||||
|
help="Passed corpus audit with accepted human-review evidence for this frozen source manifest.",
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--reference-layer",
|
"--reference-layer",
|
||||||
default=os.environ.get("OPERATOR_YOLO_REFERENCE_LAYER", DEFAULT_REFERENCE_LAYER),
|
default=os.environ.get("OPERATOR_YOLO_REFERENCE_LAYER", DEFAULT_REFERENCE_LAYER),
|
||||||
@@ -626,13 +653,17 @@ def main() -> int:
|
|||||||
):
|
):
|
||||||
if not value or any(character not in "abcdefghijklmnopqrstuvwxyz0123456789_-" for character in value):
|
if not value or any(character not in "abcdefghijklmnopqrstuvwxyz0123456789_-" for character in value):
|
||||||
raise SystemExit(f"YOLO {label} must be a non-empty canonical slug")
|
raise SystemExit(f"YOLO {label} must be a non-empty canonical slug")
|
||||||
|
manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
|
||||||
|
try:
|
||||||
|
assert_frozen_manifest_training_eligible(args.manifest_path, verify_live=True)
|
||||||
|
except TrainingEligibilityError as exc:
|
||||||
|
raise SystemExit(str(exc)) from exc
|
||||||
ensure_dependencies()
|
ensure_dependencies()
|
||||||
if args.force and args.output_dir.exists():
|
if args.force and args.output_dir.exists():
|
||||||
shutil.rmtree(args.output_dir)
|
shutil.rmtree(args.output_dir)
|
||||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
ensure_yolo_directories(args.output_dir)
|
ensure_yolo_directories(args.output_dir)
|
||||||
|
|
||||||
manifest = json.loads(args.manifest_path.read_text(encoding="utf-8-sig"))
|
|
||||||
manifest_samples = manifest.get("samples") or []
|
manifest_samples = manifest.get("samples") or []
|
||||||
if not manifest_samples:
|
if not manifest_samples:
|
||||||
raise SystemExit("Operator sample manifest contains no samples")
|
raise SystemExit("Operator sample manifest contains no samples")
|
||||||
@@ -672,6 +703,14 @@ def main() -> int:
|
|||||||
if not args.allow_empty_validation and not any(tile["split"] == "val" for tile in kept_tiles):
|
if not args.allow_empty_validation and not any(tile["split"] == "val" for tile in kept_tiles):
|
||||||
raise SystemExit("YOLO tile dataset export produced no validation tiles")
|
raise SystemExit("YOLO tile dataset export produced no validation tiles")
|
||||||
dataset_yaml = write_dataset_yaml(args.output_dir, class_name)
|
dataset_yaml = write_dataset_yaml(args.output_dir, class_name)
|
||||||
|
try:
|
||||||
|
release_paths = create_training_release_manifest(
|
||||||
|
train_yaml=dataset_yaml,
|
||||||
|
corpus_manifest=args.manifest_path,
|
||||||
|
review_audit_path=args.review_audit,
|
||||||
|
)
|
||||||
|
except TrainingReleaseError as exc:
|
||||||
|
raise SystemExit(str(exc)) from exc
|
||||||
positive_tiles = [tile for tile in kept_tiles if not tile["is_negative"]]
|
positive_tiles = [tile for tile in kept_tiles if not tile["is_negative"]]
|
||||||
negative_tiles = [tile for tile in kept_tiles if tile["is_negative"]]
|
negative_tiles = [tile for tile in kept_tiles if tile["is_negative"]]
|
||||||
skipped_negative_tiles = [tile for tile in exported_tiles if not tile["kept"] and tile["is_negative"]]
|
skipped_negative_tiles = [tile for tile in exported_tiles if not tile["kept"] and tile["is_negative"]]
|
||||||
@@ -684,6 +723,10 @@ def main() -> int:
|
|||||||
summary = {
|
summary = {
|
||||||
"status": "ok",
|
"status": "ok",
|
||||||
"dataset_yaml": str(dataset_yaml),
|
"dataset_yaml": str(dataset_yaml),
|
||||||
|
"training_release_manifest": str(release_paths["release_manifest"]),
|
||||||
|
"training_release_manifest_sha256": file_sha256(release_paths["release_manifest"]),
|
||||||
|
"training_release_freeze": str(release_paths["release_freeze"]),
|
||||||
|
"training_asset_manifest": str(release_paths["asset_manifest"]),
|
||||||
"output_dir": str(args.output_dir),
|
"output_dir": str(args.output_dir),
|
||||||
"class_names": [class_name],
|
"class_names": [class_name],
|
||||||
"reference_source": reference_source,
|
"reference_source": reference_source,
|
||||||
@@ -697,6 +740,8 @@ def main() -> int:
|
|||||||
"drop_low_variance_negatives": args.drop_low_variance_negatives,
|
"drop_low_variance_negatives": args.drop_low_variance_negatives,
|
||||||
"blank_range_threshold": args.blank_range_threshold,
|
"blank_range_threshold": args.blank_range_threshold,
|
||||||
"source_manifest_sample_count": len(manifest_samples),
|
"source_manifest_sample_count": len(manifest_samples),
|
||||||
|
"source_manifest": str(args.manifest_path),
|
||||||
|
"source_manifest_sha256": file_sha256(args.manifest_path),
|
||||||
"source_sample_count": len(samples),
|
"source_sample_count": len(samples),
|
||||||
"selected_sample_slugs": sorted(
|
"selected_sample_slugs": sorted(
|
||||||
str(sample.get("sample_slug") or "").strip().lower() for sample in samples
|
str(sample.get("sample_slug") or "").strip().lower() for sample in samples
|
||||||
|
|||||||
@@ -16,6 +16,12 @@ from dataclasses import dataclass, replace
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||||
|
if str(SCRIPT_DIR) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SCRIPT_DIR))
|
||||||
|
|
||||||
|
from training_dataset_eligibility import TRAINING_ELIGIBILITY_POLICY_VERSION # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
WMS_URL = "https://geo.api.vlaanderen.be/omwrgbmrvl/wms"
|
WMS_URL = "https://geo.api.vlaanderen.be/omwrgbmrvl/wms"
|
||||||
GRB_GBG_URL = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items"
|
GRB_GBG_URL = "https://geo.api.vlaanderen.be/GRB/ogc/features/v1/collections/GBG/items"
|
||||||
@@ -803,6 +809,14 @@ def main() -> int:
|
|||||||
manifest = {
|
manifest = {
|
||||||
"schema_version": 2,
|
"schema_version": 2,
|
||||||
"description": "GeoIntel operator real-data samples for configured-YOLO QA validation.",
|
"description": "GeoIntel operator real-data samples for configured-YOLO QA validation.",
|
||||||
|
"training_eligibility": {
|
||||||
|
"policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||||
|
"status": "not_eligible",
|
||||||
|
"reason": (
|
||||||
|
"Direct provider downloads are QA-only until re-ingested through the governed "
|
||||||
|
"dataset source registry, contract validator and provenance snapshot flow."
|
||||||
|
),
|
||||||
|
},
|
||||||
"output_dir": str(output_dir),
|
"output_dir": str(output_dir),
|
||||||
"sample_width": args.width,
|
"sample_width": args.width,
|
||||||
"sample_height": args.height,
|
"sample_height": args.height,
|
||||||
|
|||||||
@@ -626,7 +626,7 @@ def provision_dataset(
|
|||||||
from app.services.dataset_service import DatasetService
|
from app.services.dataset_service import DatasetService
|
||||||
|
|
||||||
partition_checksums = {
|
partition_checksums = {
|
||||||
summary["nis_code"]: summary["sha256"]
|
summary["filename"]: summary["sha256"]
|
||||||
for summary in manifest["partitions"]
|
for summary in manifest["partitions"]
|
||||||
}
|
}
|
||||||
metadata_json = {
|
metadata_json = {
|
||||||
|
|||||||
@@ -43,7 +43,6 @@ from provision_regional_grb_buildings import (
|
|||||||
list_paginated_items,
|
list_paginated_items,
|
||||||
next_page_url,
|
next_page_url,
|
||||||
observed_at,
|
observed_at,
|
||||||
response_data,
|
|
||||||
reusable_manifest,
|
reusable_manifest,
|
||||||
safe_slug,
|
safe_slug,
|
||||||
sha256_file,
|
sha256_file,
|
||||||
@@ -758,7 +757,7 @@ def provision_dataset(
|
|||||||
"source_urls": manifest["grb_source_urls"],
|
"source_urls": manifest["grb_source_urls"],
|
||||||
"artifact_sha256": manifest["artifact_sha256"],
|
"artifact_sha256": manifest["artifact_sha256"],
|
||||||
"artifact_size_bytes": manifest["artifact_size_bytes"],
|
"artifact_size_bytes": manifest["artifact_size_bytes"],
|
||||||
"partition_checksums": {summary["nis_code"]: summary["sha256"] for summary in manifest["partitions"]},
|
"partition_checksums": {summary["filename"]: summary["sha256"] for summary in manifest["partitions"]},
|
||||||
"partition_assignment_rule": manifest["partition_assignment_rule"],
|
"partition_assignment_rule": manifest["partition_assignment_rule"],
|
||||||
"reference_truncated": False,
|
"reference_truncated": False,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,8 +10,20 @@ import json
|
|||||||
import math
|
import math
|
||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
|
||||||
|
|
||||||
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||||
|
if str(SCRIPT_DIR) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SCRIPT_DIR))
|
||||||
|
|
||||||
|
from training_release_manifest import ( # noqa: E402
|
||||||
|
TrainingReleaseError,
|
||||||
|
assert_yolo_summary_bound_to_training_release,
|
||||||
|
file_sha256,
|
||||||
|
training_release_paths,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def sha256(path: Path) -> str:
|
def sha256(path: Path) -> str:
|
||||||
@@ -97,6 +109,8 @@ def link_or_copy(source: Path, target: Path) -> None:
|
|||||||
def main() -> int:
|
def main() -> int:
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
parser.add_argument("--summary", type=Path, required=True)
|
parser.add_argument("--summary", type=Path, required=True)
|
||||||
|
parser.add_argument("--train-yaml", type=Path, required=True)
|
||||||
|
parser.add_argument("--corpus-manifest", type=Path, required=True)
|
||||||
parser.add_argument("--model", type=Path, required=True)
|
parser.add_argument("--model", type=Path, required=True)
|
||||||
parser.add_argument("--output-dir", type=Path, required=True)
|
parser.add_argument("--output-dir", type=Path, required=True)
|
||||||
parser.add_argument("--device", default="cuda:0")
|
parser.add_argument("--device", default="cuda:0")
|
||||||
@@ -110,7 +124,21 @@ def main() -> int:
|
|||||||
parser.add_argument("--max-prompts-per-pass", type=int, default=96)
|
parser.add_argument("--max-prompts-per-pass", type=int, default=96)
|
||||||
parser.add_argument("--fallback-policy", choices=("retain", "drop"), default="retain")
|
parser.add_argument("--fallback-policy", choices=("retain", "drop"), default="retain")
|
||||||
parser.add_argument("--force", action="store_true")
|
parser.add_argument("--force", action="store_true")
|
||||||
|
parser.add_argument(
|
||||||
|
"--fixture-mode",
|
||||||
|
action="store_true",
|
||||||
|
help="Accept only an explicitly fixture-only source release; refined labels remain non-trainable.",
|
||||||
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
try:
|
||||||
|
source_release = assert_yolo_summary_bound_to_training_release(
|
||||||
|
summary_path=args.summary,
|
||||||
|
train_yaml=args.train_yaml,
|
||||||
|
corpus_manifest=args.corpus_manifest,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
|
except TrainingReleaseError as exc:
|
||||||
|
raise SystemExit(str(exc)) from exc
|
||||||
if args.output_dir.exists():
|
if args.output_dir.exists():
|
||||||
if not args.force:
|
if not args.force:
|
||||||
raise SystemExit(f"Output exists: {args.output_dir}")
|
raise SystemExit(f"Output exists: {args.output_dir}")
|
||||||
@@ -194,6 +222,16 @@ def main() -> int:
|
|||||||
print(f"{index}/{len(summary['tiles'])} {tile['sample_slug']}: {len(source_boxes)}", flush=True)
|
print(f"{index}/{len(summary['tiles'])} {tile['sample_slug']}: {len(source_boxes)}", flush=True)
|
||||||
|
|
||||||
output_summary = dict(summary)
|
output_summary = dict(summary)
|
||||||
|
# SAM changes label bytes and semantics. It therefore cannot inherit the
|
||||||
|
# source release; keep the parent evidence separately and require a new
|
||||||
|
# governed corpus/review/release before any training consumer can use it.
|
||||||
|
for field_name in (
|
||||||
|
"training_release_manifest",
|
||||||
|
"training_release_manifest_sha256",
|
||||||
|
"training_release_freeze",
|
||||||
|
"training_asset_manifest",
|
||||||
|
):
|
||||||
|
output_summary.pop(field_name, None)
|
||||||
output_summary.update(
|
output_summary.update(
|
||||||
{
|
{
|
||||||
"output_dir": str(args.output_dir),
|
"output_dir": str(args.output_dir),
|
||||||
@@ -207,6 +245,10 @@ def main() -> int:
|
|||||||
"label_count": refined_count if args.fallback_policy == "drop" else summary["label_count"],
|
"label_count": refined_count if args.fallback_policy == "drop" else summary["label_count"],
|
||||||
"positive_tile_count": sum(not tile["is_negative"] for tile in output_tiles),
|
"positive_tile_count": sum(not tile["is_negative"] for tile in output_tiles),
|
||||||
"negative_tile_count": sum(tile["is_negative"] for tile in output_tiles),
|
"negative_tile_count": sum(tile["is_negative"] for tile in output_tiles),
|
||||||
|
"training_eligible": False,
|
||||||
|
"training_eligibility_reason": (
|
||||||
|
"SAM-refined labels require a new governed corpus, validation report, human review and immutable training release."
|
||||||
|
),
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
summary_path = args.output_dir / "yolo_tile_dataset_summary.json"
|
summary_path = args.output_dir / "yolo_tile_dataset_summary.json"
|
||||||
@@ -220,6 +262,11 @@ def main() -> int:
|
|||||||
"status": "ok",
|
"status": "ok",
|
||||||
"source_summary": str(args.summary),
|
"source_summary": str(args.summary),
|
||||||
"source_summary_sha256": sha256(args.summary),
|
"source_summary_sha256": sha256(args.summary),
|
||||||
|
"source_training_release": str(training_release_paths(args.train_yaml)["release_manifest"]),
|
||||||
|
"source_training_release_sha256": file_sha256(
|
||||||
|
training_release_paths(args.train_yaml)["release_manifest"]
|
||||||
|
),
|
||||||
|
"source_corpus_manifest_sha256": source_release["corpus"]["manifest_sha256"],
|
||||||
"sam_model": str(args.model),
|
"sam_model": str(args.model),
|
||||||
"sam_model_sha256": sha256(args.model),
|
"sam_model_sha256": sha256(args.model),
|
||||||
"device": args.device,
|
"device": args.device,
|
||||||
@@ -236,6 +283,8 @@ def main() -> int:
|
|||||||
"fallback_label_count": fallback_count,
|
"fallback_label_count": fallback_count,
|
||||||
"dropped_fallback_label_count": fallback_count if args.fallback_policy == "drop" else 0,
|
"dropped_fallback_label_count": fallback_count if args.fallback_policy == "drop" else 0,
|
||||||
"fallback_reason_counts": reason_counts,
|
"fallback_reason_counts": reason_counts,
|
||||||
|
"training_eligible": False,
|
||||||
|
"fixture_mode": bool(args.fixture_mode),
|
||||||
}
|
}
|
||||||
(args.output_dir / "sam-refinement.json").write_text(json.dumps(evidence, indent=2), encoding="utf-8")
|
(args.output_dir / "sam-refinement.json").write_text(json.dumps(evidence, indent=2), encoding="utf-8")
|
||||||
print(json.dumps(evidence, indent=2))
|
print(json.dumps(evidence, indent=2))
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import argparse
|
|||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
import shutil
|
import shutil
|
||||||
|
import sys
|
||||||
from collections import Counter
|
from collections import Counter
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
@@ -15,6 +16,19 @@ from pyproj import Transformer
|
|||||||
from shapely.geometry import box
|
from shapely.geometry import box
|
||||||
from shapely.ops import transform as shapely_transform
|
from shapely.ops import transform as shapely_transform
|
||||||
|
|
||||||
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||||
|
if str(SCRIPT_DIR) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SCRIPT_DIR))
|
||||||
|
|
||||||
|
from training_dataset_eligibility import ( # noqa: E402
|
||||||
|
TrainingEligibilityError,
|
||||||
|
assert_frozen_manifest_training_eligible,
|
||||||
|
)
|
||||||
|
from training_release_manifest import ( # noqa: E402
|
||||||
|
TrainingReleaseError,
|
||||||
|
assert_yolo_summary_bound_to_embedded_training_release,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def sha256(path: Path) -> str:
|
def sha256(path: Path) -> str:
|
||||||
digest = hashlib.sha256()
|
digest = hashlib.sha256()
|
||||||
@@ -73,6 +87,11 @@ def main() -> int:
|
|||||||
parser.add_argument("--internal-val-samples", required=True)
|
parser.add_argument("--internal-val-samples", required=True)
|
||||||
parser.add_argument("--version", required=True)
|
parser.add_argument("--version", required=True)
|
||||||
parser.add_argument("--output-dir", type=Path, required=True)
|
parser.add_argument("--output-dir", type=Path, required=True)
|
||||||
|
parser.add_argument(
|
||||||
|
"--fixture-mode",
|
||||||
|
action="store_true",
|
||||||
|
help="Accept only an explicitly fixture-only source corpus; never use for operational holdout rotation.",
|
||||||
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
role_slugs = {
|
role_slugs = {
|
||||||
@@ -86,6 +105,20 @@ def main() -> int:
|
|||||||
raise SystemExit("rotated holdout lists overlap")
|
raise SystemExit("rotated holdout lists overlap")
|
||||||
|
|
||||||
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
|
manifest = json.loads(args.corpus_manifest.read_text(encoding="utf-8"))
|
||||||
|
try:
|
||||||
|
assert_frozen_manifest_training_eligible(
|
||||||
|
args.corpus_manifest,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
verify_live=True,
|
||||||
|
)
|
||||||
|
for path in args.source_summary:
|
||||||
|
assert_yolo_summary_bound_to_embedded_training_release(
|
||||||
|
summary_path=path,
|
||||||
|
corpus_manifest=args.corpus_manifest,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
|
except (TrainingEligibilityError, TrainingReleaseError) as exc:
|
||||||
|
raise SystemExit(str(exc)) from exc
|
||||||
source_samples = {item["sample_slug"]: item for item in manifest["samples"]}
|
source_samples = {item["sample_slug"]: item for item in manifest["samples"]}
|
||||||
missing = sorted(all_holdouts - source_samples.keys())
|
missing = sorted(all_holdouts - source_samples.keys())
|
||||||
if missing:
|
if missing:
|
||||||
@@ -126,6 +159,18 @@ def main() -> int:
|
|||||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
manifest_path = args.output_dir / "operator_samples_manifest.json"
|
manifest_path = args.output_dir / "operator_samples_manifest.json"
|
||||||
write_json(manifest_path, rotated_manifest)
|
write_json(manifest_path, rotated_manifest)
|
||||||
|
write_json(
|
||||||
|
args.output_dir / "corpus-freeze.json",
|
||||||
|
{
|
||||||
|
"schema_version": 2,
|
||||||
|
"dataset_version": rotated_manifest["dataset_version"],
|
||||||
|
"manifest_sha256": sha256(manifest_path),
|
||||||
|
"sample_count": len(rotated_manifest["samples"]),
|
||||||
|
"immutable": True,
|
||||||
|
"training_eligibility_policy": rotated_manifest["training_eligibility"]["policy_version"],
|
||||||
|
"fixture_mode": bool(args.fixture_mode),
|
||||||
|
},
|
||||||
|
)
|
||||||
leakage = audit_spatial_leakage(rotated_manifest["samples"])
|
leakage = audit_spatial_leakage(rotated_manifest["samples"])
|
||||||
write_json(args.output_dir / "spatial-leakage-audit.json", leakage)
|
write_json(args.output_dir / "spatial-leakage-audit.json", leakage)
|
||||||
if leakage["status"] != "ok":
|
if leakage["status"] != "ok":
|
||||||
@@ -193,6 +238,11 @@ def main() -> int:
|
|||||||
"fit_samples_in_internal_validation": sorted(
|
"fit_samples_in_internal_validation": sorted(
|
||||||
{tile["sample_slug"] for tile in train_tiles} & {tile["sample_slug"] for tile in internal_val_tiles}
|
{tile["sample_slug"] for tile in train_tiles} & {tile["sample_slug"] for tile in internal_val_tiles}
|
||||||
),
|
),
|
||||||
|
"fixture_mode": bool(args.fixture_mode),
|
||||||
|
"training_eligible": False,
|
||||||
|
"training_eligibility_reason": (
|
||||||
|
"A rotated split needs a new human review, label contract validation and immutable training release."
|
||||||
|
),
|
||||||
}
|
}
|
||||||
if evidence["protected_samples_in_training"]:
|
if evidence["protected_samples_in_training"]:
|
||||||
raise SystemExit("protected samples leaked into rotated training lists")
|
raise SystemExit("protected samples leaked into rotated training lists")
|
||||||
|
|||||||
@@ -0,0 +1,133 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Emit reproducible local evidence for the Phase-2 data foundation.
|
||||||
|
|
||||||
|
The collector does not connect to PostgreSQL, source providers, a GPU or any
|
||||||
|
training corpus. It inventories the versioned source-policy and contract
|
||||||
|
definitions that are present in this checkout. Runtime migration/application
|
||||||
|
evidence is recorded separately because it needs an explicitly chosen target
|
||||||
|
database and must never be inferred from this static report.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
BACKEND = ROOT / "backend"
|
||||||
|
if str(BACKEND) not in sys.path:
|
||||||
|
sys.path.insert(0, str(BACKEND))
|
||||||
|
|
||||||
|
from app.services.data_contract_validation import build_default_data_contract_registry # noqa: E402
|
||||||
|
from app.services.source_registry_service import SERVER_OWNED_SOURCE_DEFINITIONS # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
|
def _git_value(*args: str) -> str | None:
|
||||||
|
try:
|
||||||
|
return subprocess.check_output(
|
||||||
|
["git", *args],
|
||||||
|
cwd=ROOT,
|
||||||
|
text=True,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
).strip() or None
|
||||||
|
except (OSError, subprocess.CalledProcessError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def collect() -> dict[str, Any]:
|
||||||
|
definitions = list(SERVER_OWNED_SOURCE_DEFINITIONS.values())
|
||||||
|
contracts = build_default_data_contract_registry().registered_contracts()
|
||||||
|
classification_counts = Counter(item.classification for item in definitions)
|
||||||
|
source_items = [
|
||||||
|
{
|
||||||
|
"source_key": item.source_key,
|
||||||
|
"classification": item.classification,
|
||||||
|
"authority_name": item.authority_name,
|
||||||
|
"authority_scope": item.authority_scope,
|
||||||
|
"provider_adapter_key": item.provider_adapter_key,
|
||||||
|
"default_crs": item.default_crs,
|
||||||
|
"default_units": item.default_units,
|
||||||
|
"freshness_status": item.freshness_status,
|
||||||
|
"ingest_status": item.ingest_status,
|
||||||
|
"ground_truth_allowed": bool((item.usage_policy or {}).get("ground_truth_allowed")),
|
||||||
|
"training_allowed": bool((item.usage_policy or {}).get("training_allowed")),
|
||||||
|
}
|
||||||
|
for item in sorted(definitions, key=lambda definition: definition.source_key)
|
||||||
|
]
|
||||||
|
contract_items = [
|
||||||
|
{
|
||||||
|
"key": item.key,
|
||||||
|
"version": item.version,
|
||||||
|
"kind": item.kind.value,
|
||||||
|
"fingerprint_sha256": item.fingerprint(),
|
||||||
|
"canonical_storage_crs": item.canonical_storage_crs,
|
||||||
|
"accepted_source_crs": sorted(item.accepted_source_crs),
|
||||||
|
"required_metadata_fields": list(item.required_metadata_fields),
|
||||||
|
"requires_source_registry": item.lineage_rules.require_source_registry,
|
||||||
|
"requires_source_snapshot": item.lineage_rules.require_source_snapshot,
|
||||||
|
"requires_upstream_assets": item.lineage_rules.require_upstream_assets,
|
||||||
|
}
|
||||||
|
for item in sorted(contracts, key=lambda contract: (contract.kind.value, contract.key, contract.version))
|
||||||
|
]
|
||||||
|
return {
|
||||||
|
"schema_version": 1,
|
||||||
|
"program": "GeoIntel Accuracy Improvement Program",
|
||||||
|
"phase": "P2",
|
||||||
|
"collected_at": datetime.now(UTC).isoformat(),
|
||||||
|
"repository": {
|
||||||
|
"branch": _git_value("branch", "--show-current"),
|
||||||
|
"head": _git_value("rev-parse", "HEAD"),
|
||||||
|
"dirty": bool(_git_value("status", "--porcelain")),
|
||||||
|
},
|
||||||
|
"scope": "Belgium and the Belgian North Sea",
|
||||||
|
"migration_revision": "202608010001",
|
||||||
|
"source_registry": {
|
||||||
|
"definition_count": len(source_items),
|
||||||
|
"classification_counts": dict(sorted(classification_counts.items())),
|
||||||
|
"required_building_policy": {
|
||||||
|
"grb_primary_building_validation": SERVER_OWNED_SOURCE_DEFINITIONS["grb"].usage_policy[
|
||||||
|
"validation_authority"
|
||||||
|
].get("building_validation"),
|
||||||
|
"buildings_register_classification": SERVER_OWNED_SOURCE_DEFINITIONS[
|
||||||
|
"digitaal_vlaanderen_buildings_addresses_register"
|
||||||
|
].classification,
|
||||||
|
"sentinel_2_classification": SERVER_OWNED_SOURCE_DEFINITIONS["sentinel_2"].classification,
|
||||||
|
"dhmv_classification": SERVER_OWNED_SOURCE_DEFINITIONS["digitaal_vlaanderen_dhmv"].classification,
|
||||||
|
"osm_ground_truth_allowed": SERVER_OWNED_SOURCE_DEFINITIONS["osm"].usage_policy[
|
||||||
|
"ground_truth_allowed"
|
||||||
|
],
|
||||||
|
},
|
||||||
|
"definitions": source_items,
|
||||||
|
},
|
||||||
|
"data_contracts": contract_items,
|
||||||
|
"claim_boundary": (
|
||||||
|
"Static policy/contract inventory only. It does not attest that a database migration ran, "
|
||||||
|
"that legacy rows are complete, or that a model/corpus is release-ready."
|
||||||
|
),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument(
|
||||||
|
"--output",
|
||||||
|
type=Path,
|
||||||
|
default=ROOT / "artifacts" / "evidence" / "accuracy" / "P2" / "source-contract-inventory.json",
|
||||||
|
)
|
||||||
|
args = parser.parse_args()
|
||||||
|
result = collect()
|
||||||
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
args.output.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||||
|
print(json.dumps({"output": str(args.output), "source_count": result["source_registry"]["definition_count"]}))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -13,6 +13,21 @@ from datetime import UTC, datetime
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||||
|
if str(SCRIPT_DIR) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SCRIPT_DIR))
|
||||||
|
|
||||||
|
from training_dataset_eligibility import ( # noqa: E402
|
||||||
|
TrainingEligibilityError,
|
||||||
|
assert_frozen_manifest_training_eligible,
|
||||||
|
)
|
||||||
|
from training_release_manifest import ( # noqa: E402
|
||||||
|
TrainingReleaseError,
|
||||||
|
assert_training_release_eligible,
|
||||||
|
human_review_audit_failures,
|
||||||
|
training_release_paths,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def sha256(path: Path) -> str:
|
def sha256(path: Path) -> str:
|
||||||
digest = hashlib.sha256()
|
digest = hashlib.sha256()
|
||||||
@@ -32,11 +47,14 @@ def write_json(path: Path, value: dict[str, Any]) -> None:
|
|||||||
def dataset_audit_failures(
|
def dataset_audit_failures(
|
||||||
audit: dict[str, Any],
|
audit: dict[str, Any],
|
||||||
train_quality_audit: dict[str, Any],
|
train_quality_audit: dict[str, Any],
|
||||||
|
*,
|
||||||
|
fixture_mode: bool = False,
|
||||||
) -> list[str]:
|
) -> list[str]:
|
||||||
"""Return automated corpus blockers while leaving final human review deferred."""
|
"""Return fail-closed corpus blockers, including human review in normal mode."""
|
||||||
failures = [str(item) for item in audit.get("failures") or []]
|
failures = [str(item) for item in audit.get("failures") or []]
|
||||||
status = audit.get("status")
|
status = audit.get("status")
|
||||||
if status not in {"ok", "needs_human_review"}:
|
permitted_statuses = {"ok", "needs_human_review"} if fixture_mode else {"ok"}
|
||||||
|
if status not in permitted_statuses:
|
||||||
failures.append(f"unsupported audit status: {status}")
|
failures.append(f"unsupported audit status: {status}")
|
||||||
if audit.get("manifest_immutable") is not True:
|
if audit.get("manifest_immutable") is not True:
|
||||||
failures.append("corpus manifest is not immutable")
|
failures.append("corpus manifest is not immutable")
|
||||||
@@ -50,9 +68,56 @@ def dataset_audit_failures(
|
|||||||
failures.append("train tile quality audit contains missing label files")
|
failures.append("train tile quality audit contains missing label files")
|
||||||
if int(train_quality_audit.get("low_variance_positive_tile_count", -1)) != 0:
|
if int(train_quality_audit.get("low_variance_positive_tile_count", -1)) != 0:
|
||||||
failures.append("dataset contains blank/low-variance positive tiles")
|
failures.append("dataset contains blank/low-variance positive tiles")
|
||||||
|
if not fixture_mode:
|
||||||
|
failures.extend(human_review_audit_failures(audit))
|
||||||
return failures
|
return failures
|
||||||
|
|
||||||
|
|
||||||
|
def verify_training_inputs(
|
||||||
|
*,
|
||||||
|
train_yaml: Path,
|
||||||
|
corpus_manifest: Path,
|
||||||
|
fixture_mode: bool,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Re-check every immutable input before initial, retry or resume training."""
|
||||||
|
|
||||||
|
try:
|
||||||
|
assert_frozen_manifest_training_eligible(
|
||||||
|
corpus_manifest,
|
||||||
|
fixture_mode=fixture_mode,
|
||||||
|
verify_live=True,
|
||||||
|
)
|
||||||
|
return assert_training_release_eligible(
|
||||||
|
train_yaml=train_yaml,
|
||||||
|
corpus_manifest=corpus_manifest,
|
||||||
|
fixture_mode=fixture_mode,
|
||||||
|
)
|
||||||
|
except (TrainingEligibilityError, TrainingReleaseError) as exc:
|
||||||
|
raise TrainingReleaseError(str(exc)) from exc
|
||||||
|
|
||||||
|
|
||||||
|
def assert_dataset_audit_bound_to_release(
|
||||||
|
*,
|
||||||
|
release: dict[str, Any],
|
||||||
|
dataset_audit: Path,
|
||||||
|
fixture_mode: bool,
|
||||||
|
) -> None:
|
||||||
|
"""Do not let a caller swap the reviewed corpus audit after release sealing."""
|
||||||
|
|
||||||
|
if fixture_mode:
|
||||||
|
return
|
||||||
|
review = release.get("human_review")
|
||||||
|
if not isinstance(review, dict):
|
||||||
|
raise TrainingReleaseError("Training release has no human-review audit binding")
|
||||||
|
review_audit_path = review.get("audit_path")
|
||||||
|
if not isinstance(review_audit_path, str) or not review_audit_path:
|
||||||
|
raise TrainingReleaseError("Training release human-review audit path is missing")
|
||||||
|
if Path(review_audit_path).resolve(strict=False) != dataset_audit.resolve(strict=False):
|
||||||
|
raise TrainingReleaseError(
|
||||||
|
"--dataset-audit does not match the immutable training-release human-review audit"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def select_calibration_threshold(report: dict[str, Any]) -> dict[str, Any]:
|
def select_calibration_threshold(report: dict[str, Any]) -> dict[str, Any]:
|
||||||
"""Choose a threshold without consulting test or background evidence."""
|
"""Choose a threshold without consulting test or background evidence."""
|
||||||
eligible = [item for item in report["sweeps"] if item["pure_empty_false_positives"] == 0]
|
eligible = [item for item in report["sweeps"] if item["pure_empty_false_positives"] == 0]
|
||||||
@@ -111,6 +176,16 @@ def rejected_candidate_score(assessment: dict[str, Any]) -> tuple[float, ...]:
|
|||||||
return tuple(normalized)
|
return tuple(normalized)
|
||||||
|
|
||||||
|
|
||||||
|
def protected_feedback_roles(assessment: dict[str, Any]) -> list[str]:
|
||||||
|
"""Return protected evidence roles that make iterative retraining illegal."""
|
||||||
|
|
||||||
|
return [
|
||||||
|
role
|
||||||
|
for role in ("test", "background")
|
||||||
|
if assessment.get(role) is not None
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
def training_command(
|
def training_command(
|
||||||
yolo: str,
|
yolo: str,
|
||||||
*,
|
*,
|
||||||
@@ -183,17 +258,23 @@ def failure_sampling_command(
|
|||||||
corpus_manifest: Path,
|
corpus_manifest: Path,
|
||||||
assessment: Path,
|
assessment: Path,
|
||||||
output_dir: Path,
|
output_dir: Path,
|
||||||
|
review_audit: Path,
|
||||||
sampling_round: int = 0,
|
sampling_round: int = 0,
|
||||||
|
fixture_mode: bool = False,
|
||||||
) -> list[str]:
|
) -> list[str]:
|
||||||
return [
|
command = [
|
||||||
sys.executable,
|
sys.executable,
|
||||||
str(scripts_dir / "build_failure_driven_yolo_sampling.py"),
|
str(scripts_dir / "build_failure_driven_yolo_sampling.py"),
|
||||||
"--summary", str(train_summary),
|
"--summary", str(train_summary),
|
||||||
"--corpus-manifest", str(corpus_manifest),
|
"--corpus-manifest", str(corpus_manifest),
|
||||||
"--assessment", str(assessment),
|
"--assessment", str(assessment),
|
||||||
"--output-dir", str(output_dir),
|
"--output-dir", str(output_dir),
|
||||||
|
"--review-audit", str(review_audit),
|
||||||
"--sampling-round", str(sampling_round),
|
"--sampling-round", str(sampling_round),
|
||||||
]
|
]
|
||||||
|
if fixture_mode:
|
||||||
|
command.append("--fixture-mode")
|
||||||
|
return command
|
||||||
|
|
||||||
|
|
||||||
def resumable_training_command(yolo: str, checkpoint: Path) -> list[str]:
|
def resumable_training_command(yolo: str, checkpoint: Path) -> list[str]:
|
||||||
@@ -259,6 +340,14 @@ def main() -> int:
|
|||||||
parser.add_argument("--min-region-recall", type=float, default=0.4)
|
parser.add_argument("--min-region-recall", type=float, default=0.4)
|
||||||
parser.add_argument("--max-pure-empty-fp", type=int, default=0)
|
parser.add_argument("--max-pure-empty-fp", type=int, default=0)
|
||||||
parser.add_argument("--dry-run", action="store_true")
|
parser.add_argument("--dry-run", action="store_true")
|
||||||
|
parser.add_argument(
|
||||||
|
"--fixture-mode",
|
||||||
|
action="store_true",
|
||||||
|
help=(
|
||||||
|
"Accept an explicitly fixture-only corpus manifest. "
|
||||||
|
"This mode is prohibited for operational training."
|
||||||
|
),
|
||||||
|
)
|
||||||
parser.add_argument(
|
parser.add_argument(
|
||||||
"--evaluate-initial-model",
|
"--evaluate-initial-model",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
@@ -267,9 +356,29 @@ def main() -> int:
|
|||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
if args.iterations < 1:
|
if args.iterations < 1:
|
||||||
raise SystemExit("--iterations must be positive")
|
raise SystemExit("--iterations must be positive")
|
||||||
|
try:
|
||||||
|
initial_release = verify_training_inputs(
|
||||||
|
train_yaml=args.train_yaml,
|
||||||
|
corpus_manifest=args.corpus_manifest,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
|
except TrainingReleaseError as exc:
|
||||||
|
raise SystemExit(str(exc)) from exc
|
||||||
|
try:
|
||||||
|
assert_dataset_audit_bound_to_release(
|
||||||
|
release=initial_release,
|
||||||
|
dataset_audit=args.dataset_audit,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
|
except TrainingReleaseError as exc:
|
||||||
|
raise SystemExit(str(exc)) from exc
|
||||||
dataset_audit = json.loads(args.dataset_audit.read_text(encoding="utf-8"))
|
dataset_audit = json.loads(args.dataset_audit.read_text(encoding="utf-8"))
|
||||||
train_quality_audit = json.loads(args.train_quality_audit.read_text(encoding="utf-8"))
|
train_quality_audit = json.loads(args.train_quality_audit.read_text(encoding="utf-8"))
|
||||||
audit_failures = dataset_audit_failures(dataset_audit, train_quality_audit)
|
audit_failures = dataset_audit_failures(
|
||||||
|
dataset_audit,
|
||||||
|
train_quality_audit,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
if audit_failures:
|
if audit_failures:
|
||||||
raise SystemExit(f"Dataset audit is not eligible for training: {audit_failures}")
|
raise SystemExit(f"Dataset audit is not eligible for training: {audit_failures}")
|
||||||
|
|
||||||
@@ -285,6 +394,13 @@ def main() -> int:
|
|||||||
"train_quality_audit": str(args.train_quality_audit),
|
"train_quality_audit": str(args.train_quality_audit),
|
||||||
"train_quality_audit_sha256": sha256(args.train_quality_audit),
|
"train_quality_audit_sha256": sha256(args.train_quality_audit),
|
||||||
"corpus_manifest": str(args.corpus_manifest),
|
"corpus_manifest": str(args.corpus_manifest),
|
||||||
|
"corpus_manifest_sha256": sha256(args.corpus_manifest),
|
||||||
|
"initial_training_release": str(training_release_paths(args.train_yaml)["release_manifest"]),
|
||||||
|
"initial_training_release_sha256": sha256(
|
||||||
|
training_release_paths(args.train_yaml)["release_manifest"]
|
||||||
|
),
|
||||||
|
"initial_training_release_contract": initial_release["contract_version"],
|
||||||
|
"fixture_mode": bool(args.fixture_mode),
|
||||||
"iterations": [],
|
"iterations": [],
|
||||||
}
|
}
|
||||||
if state_path.is_file():
|
if state_path.is_file():
|
||||||
@@ -304,6 +420,26 @@ def main() -> int:
|
|||||||
evaluate_existing = args.evaluate_initial_model and offset == 0 and not state["iterations"]
|
evaluate_existing = args.evaluate_initial_model and offset == 0 and not state["iterations"]
|
||||||
partial_checkpoint = train_run / "weights" / "last.pt"
|
partial_checkpoint = train_run / "weights" / "last.pt"
|
||||||
resume_partial = not evaluate_existing and partial_checkpoint.is_file()
|
resume_partial = not evaluate_existing and partial_checkpoint.is_file()
|
||||||
|
try:
|
||||||
|
release = verify_training_inputs(
|
||||||
|
train_yaml=train_yaml,
|
||||||
|
corpus_manifest=args.corpus_manifest,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
|
except TrainingReleaseError as exc:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Training inputs changed before {name}; refusing initial/retry/resume execution: {exc}"
|
||||||
|
) from exc
|
||||||
|
try:
|
||||||
|
assert_dataset_audit_bound_to_release(
|
||||||
|
release=release,
|
||||||
|
dataset_audit=args.dataset_audit,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
|
except TrainingReleaseError as exc:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Training audit changed before {name}; refusing initial/retry/resume execution: {exc}"
|
||||||
|
) from exc
|
||||||
command = None if evaluate_existing else (
|
command = None if evaluate_existing else (
|
||||||
resumable_training_command(args.yolo, partial_checkpoint)
|
resumable_training_command(args.yolo, partial_checkpoint)
|
||||||
if resume_partial else training_command(
|
if resume_partial else training_command(
|
||||||
@@ -454,11 +590,27 @@ def main() -> int:
|
|||||||
"candidate_sha256": sha256(candidate),
|
"candidate_sha256": sha256(candidate),
|
||||||
"training_skipped_for_existing_checkpoint": evaluate_existing,
|
"training_skipped_for_existing_checkpoint": evaluate_existing,
|
||||||
"training_resumed_from_partial_checkpoint": resume_partial,
|
"training_resumed_from_partial_checkpoint": resume_partial,
|
||||||
|
"training_release": str(training_release_paths(train_yaml)["release_manifest"]),
|
||||||
|
"training_release_sha256": sha256(training_release_paths(train_yaml)["release_manifest"]),
|
||||||
|
"training_release_asset_manifest_sha256": release["asset_manifest"]["sha256"],
|
||||||
"assessment": str(assessment),
|
"assessment": str(assessment),
|
||||||
"status": decision["status"],
|
"status": decision["status"],
|
||||||
"failures": decision["failures"],
|
"failures": decision["failures"],
|
||||||
}
|
}
|
||||||
state["iterations"].append(record)
|
state["iterations"].append(record)
|
||||||
|
protected_feedback = protected_feedback_roles(decision)
|
||||||
|
if decision["status"] != "training_complete" and protected_feedback:
|
||||||
|
record["protected_feedback_blocked"] = protected_feedback
|
||||||
|
record["retraining_prohibited"] = True
|
||||||
|
state["status"] = "protected_evaluation_rejected"
|
||||||
|
state["stopped_at"] = datetime.now(UTC).isoformat()
|
||||||
|
state["stop_reason"] = (
|
||||||
|
"Protected test/background evidence was opened for a rejected candidate; "
|
||||||
|
"its results cannot generate another training YAML."
|
||||||
|
)
|
||||||
|
write_json(state_path, state)
|
||||||
|
print(json.dumps(state, indent=2))
|
||||||
|
return 3
|
||||||
score = rejected_candidate_score(decision) if decision["status"] != "training_complete" else ()
|
score = rejected_candidate_score(decision) if decision["status"] != "training_complete" else ()
|
||||||
incumbent_score = tuple(state.get("incumbent_rejected_score", ()))
|
incumbent_score = tuple(state.get("incumbent_rejected_score", ()))
|
||||||
if not incumbent_score or score > incumbent_score:
|
if not incumbent_score or score > incumbent_score:
|
||||||
@@ -482,14 +634,29 @@ def main() -> int:
|
|||||||
corpus_manifest=args.corpus_manifest,
|
corpus_manifest=args.corpus_manifest,
|
||||||
assessment=assessment,
|
assessment=assessment,
|
||||||
output_dir=sampling_dir,
|
output_dir=sampling_dir,
|
||||||
|
review_audit=args.dataset_audit,
|
||||||
sampling_round=index,
|
sampling_round=index,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
),
|
),
|
||||||
iteration_dir / "failure-driven-sampling.log",
|
iteration_dir / "failure-driven-sampling.log",
|
||||||
)
|
)
|
||||||
sampling_evidence = sampling_dir / "failure-driven-sampling.json"
|
sampling_evidence = sampling_dir / "failure-driven-sampling.json"
|
||||||
next_train_yaml = sampling_dir / "dataset.yaml"
|
next_train_yaml = sampling_dir / "dataset.yaml"
|
||||||
if not sampling_evidence.is_file() or not next_train_yaml.is_file():
|
next_release_paths = training_release_paths(next_train_yaml)
|
||||||
|
if not sampling_evidence.is_file() or not next_train_yaml.is_file() or not all(
|
||||||
|
path.is_file() for path in next_release_paths.values()
|
||||||
|
):
|
||||||
raise RuntimeError("Failure-driven sampling produced incomplete evidence")
|
raise RuntimeError("Failure-driven sampling produced incomplete evidence")
|
||||||
|
try:
|
||||||
|
verify_training_inputs(
|
||||||
|
train_yaml=next_train_yaml,
|
||||||
|
corpus_manifest=args.corpus_manifest,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
|
except TrainingReleaseError as exc:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Failure-driven sampling produced an unbound training release: {exc}"
|
||||||
|
) from exc
|
||||||
record["failure_driven_sampling"] = str(sampling_evidence)
|
record["failure_driven_sampling"] = str(sampling_evidence)
|
||||||
record["failure_driven_sampling_sha256"] = sha256(sampling_evidence)
|
record["failure_driven_sampling_sha256"] = sha256(sampling_evidence)
|
||||||
record["next_train_yaml"] = str(next_train_yaml)
|
record["next_train_yaml"] = str(next_train_yaml)
|
||||||
|
|||||||
@@ -3,6 +3,8 @@ from __future__ import annotations
|
|||||||
import argparse
|
import argparse
|
||||||
import json
|
import json
|
||||||
import sys
|
import sys
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from hashlib import sha256
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from uuid import uuid4
|
from uuid import uuid4
|
||||||
|
|
||||||
@@ -12,9 +14,10 @@ BACKEND_ROOT = ROOT / "backend"
|
|||||||
if str(BACKEND_ROOT) not in sys.path:
|
if str(BACKEND_ROOT) not in sys.path:
|
||||||
sys.path.insert(0, str(BACKEND_ROOT))
|
sys.path.insert(0, str(BACKEND_ROOT))
|
||||||
|
|
||||||
from app.models import Area, Dataset, Metric, QualityCheck # noqa: E402
|
from app.models import Dataset, Metric, QualityCheck, SourceRegistry, SourceSnapshot # noqa: E402
|
||||||
from app.services.qa_service import QaService # noqa: E402
|
from app.services.qa_service import QaService # noqa: E402
|
||||||
from app.services.quality_service import QualityService # noqa: E402
|
from app.services.quality_service import QualityService # noqa: E402
|
||||||
|
from app.services.source_registry_service import SourceRegistryService # noqa: E402
|
||||||
|
|
||||||
|
|
||||||
class BenchmarkSession:
|
class BenchmarkSession:
|
||||||
@@ -55,18 +58,58 @@ def _load_manifest() -> dict:
|
|||||||
|
|
||||||
|
|
||||||
def _dataset(dataset_id, project_id, name: str, path: Path, *, role: str) -> Dataset:
|
def _dataset(dataset_id, project_id, name: str, path: Path, *, role: str) -> Dataset:
|
||||||
|
# The benchmark is still synthetic evidence, but it intentionally models
|
||||||
|
# the same complete Phase 2 provenance binding required at the QA service
|
||||||
|
# boundary. The candidate is a derived fixture; the reference simulates
|
||||||
|
# a governed GRB building snapshot. No legacy fixture exception is used.
|
||||||
|
source_key = "grb" if role == "reference" else "derived"
|
||||||
|
source = SourceRegistry(
|
||||||
|
id=uuid4(),
|
||||||
|
**SourceRegistryService.definition_for(source_key).as_model_values(),
|
||||||
|
)
|
||||||
|
checksum = sha256(path.read_bytes()).hexdigest()
|
||||||
|
snapshot = SourceSnapshot(
|
||||||
|
id=uuid4(),
|
||||||
|
source_registry_id=source.id,
|
||||||
|
snapshot_key=f"golden-qa:{source_key}:{checksum}",
|
||||||
|
checksum_sha256=checksum,
|
||||||
|
fetched_at=datetime.now(UTC),
|
||||||
|
crs="EPSG:4326",
|
||||||
|
units=source.default_units,
|
||||||
|
spatial_resolution_json={"status": "fixture"},
|
||||||
|
temporal_coverage_json={"status": "fixture"},
|
||||||
|
geographic_coverage_json={"scope": "golden-qa-fixture"},
|
||||||
|
observed_schema_json={"dataset_type": "vector", "fixture_mode": True},
|
||||||
|
freshness_status="current",
|
||||||
|
ingest_status="ingested",
|
||||||
|
known_limitations_json=["Synthetic golden benchmark fixture; not production evidence."],
|
||||||
|
snapshot_metadata_json={"fixture_mode": True, "benchmark": "golden-qa"},
|
||||||
|
)
|
||||||
return Dataset(
|
return Dataset(
|
||||||
id=dataset_id,
|
id=dataset_id,
|
||||||
project_id=project_id,
|
project_id=project_id,
|
||||||
name=name,
|
name=name,
|
||||||
dataset_type="vector",
|
dataset_type="vector",
|
||||||
source="golden_fixture",
|
source="governed golden benchmark fixture",
|
||||||
dataset_role=role,
|
dataset_role="reference" if role == "reference" else "derived",
|
||||||
source_name="fixture",
|
source_name=source.source_key,
|
||||||
reference_layer_name="buildings" if role == "reference" else None,
|
reference_layer_name="buildings" if role == "reference" else None,
|
||||||
storage_path=str(path),
|
storage_path=str(path),
|
||||||
crs="EPSG:4326",
|
crs="EPSG:4326",
|
||||||
metadata_json={"crs_assumed": False},
|
checksum_sha256=checksum,
|
||||||
|
source_registry_id=source.id,
|
||||||
|
source_snapshot_id=snapshot.id,
|
||||||
|
source_registry=source,
|
||||||
|
source_snapshot=snapshot,
|
||||||
|
data_contract_key="geointel.vector.geojson",
|
||||||
|
data_contract_version="1.0.0",
|
||||||
|
validation_status="passed",
|
||||||
|
provenance_status="complete",
|
||||||
|
lineage_status="not_applicable",
|
||||||
|
quarantine_status="not_quarantined",
|
||||||
|
metadata_json={"crs_assumed": False, "fixture_mode": True},
|
||||||
|
source_metadata={"fixture_mode": True, "source_registry_key": source.source_key},
|
||||||
|
provenance_metadata={"fixture_mode": True, "source_snapshot_id": str(snapshot.id)},
|
||||||
status="ready",
|
status="ready",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -109,6 +109,10 @@ ${PYTHON_BIN} -m py_compile scripts/audit_source_freshness.py
|
|||||||
${PYTHON_BIN} -m py_compile scripts/manage_grb_refresh.py
|
${PYTHON_BIN} -m py_compile scripts/manage_grb_refresh.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
|
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_dataset.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py
|
${PYTHON_BIN} -m py_compile scripts/export_operator_yolo_tile_dataset.py
|
||||||
|
${PYTHON_BIN} -m py_compile scripts/training_dataset_eligibility.py
|
||||||
|
${PYTHON_BIN} -m py_compile scripts/training_release_manifest.py
|
||||||
|
${PYTHON_BIN} -m py_compile scripts/run_belgium_building_training_loop.py
|
||||||
|
${PYTHON_BIN} -m py_compile scripts/build_failure_driven_yolo_sampling.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/audit_operator_yolo_dataset_quality.py
|
${PYTHON_BIN} -m py_compile scripts/audit_operator_yolo_dataset_quality.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/build_detection_model_promotion_report.py
|
${PYTHON_BIN} -m py_compile scripts/build_detection_model_promotion_report.py
|
||||||
${PYTHON_BIN} -m py_compile scripts/build_mol_operational_benchmark_report.py
|
${PYTHON_BIN} -m py_compile scripts/build_mol_operational_benchmark_report.py
|
||||||
|
|||||||
@@ -6,10 +6,20 @@ from __future__ import annotations
|
|||||||
import argparse
|
import argparse
|
||||||
import json
|
import json
|
||||||
import subprocess
|
import subprocess
|
||||||
|
import sys
|
||||||
import time
|
import time
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||||
|
if str(SCRIPT_DIR) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SCRIPT_DIR))
|
||||||
|
|
||||||
|
from training_release_manifest import ( # noqa: E402
|
||||||
|
TrainingReleaseError,
|
||||||
|
assert_training_release_eligible,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def container_running(container: str) -> bool:
|
def container_running(container: str) -> bool:
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
@@ -44,11 +54,33 @@ def load_completion_command(path: Path) -> list[str]:
|
|||||||
return command
|
return command
|
||||||
|
|
||||||
|
|
||||||
|
def verify_training_release(
|
||||||
|
*,
|
||||||
|
train_yaml: Path,
|
||||||
|
corpus_manifest: Path,
|
||||||
|
fixture_mode: bool,
|
||||||
|
) -> None:
|
||||||
|
"""Re-check the exact release before each detached resume command."""
|
||||||
|
|
||||||
|
assert_training_release_eligible(
|
||||||
|
train_yaml=train_yaml,
|
||||||
|
corpus_manifest=corpus_manifest,
|
||||||
|
fixture_mode=fixture_mode,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def main() -> int:
|
def main() -> int:
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
parser.add_argument("--container", required=True)
|
parser.add_argument("--container", required=True)
|
||||||
parser.add_argument("--host-run-dir", type=Path, required=True)
|
parser.add_argument("--host-run-dir", type=Path, required=True)
|
||||||
parser.add_argument("--container-checkpoint", required=True)
|
parser.add_argument("--container-checkpoint", required=True)
|
||||||
|
parser.add_argument("--train-yaml", type=Path, required=True)
|
||||||
|
parser.add_argument("--corpus-manifest", type=Path, required=True)
|
||||||
|
parser.add_argument(
|
||||||
|
"--fixture-mode",
|
||||||
|
action="store_true",
|
||||||
|
help="Only valid for an explicitly fixture-only frozen corpus release.",
|
||||||
|
)
|
||||||
parser.add_argument("--run-marker", required=True)
|
parser.add_argument("--run-marker", required=True)
|
||||||
parser.add_argument("--yolo", default="/opt/geointel/venv/bin/yolo")
|
parser.add_argument("--yolo", default="/opt/geointel/venv/bin/yolo")
|
||||||
parser.add_argument("--poll-seconds", type=int, default=30)
|
parser.add_argument("--poll-seconds", type=int, default=30)
|
||||||
@@ -101,6 +133,17 @@ def main() -> int:
|
|||||||
state["status"] = "resume_budget_exhausted"
|
state["status"] = "resume_budget_exhausted"
|
||||||
write_state(state_path, state)
|
write_state(state_path, state)
|
||||||
return 3
|
return 3
|
||||||
|
try:
|
||||||
|
verify_training_release(
|
||||||
|
train_yaml=args.train_yaml,
|
||||||
|
corpus_manifest=args.corpus_manifest,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
|
except TrainingReleaseError as exc:
|
||||||
|
state["status"] = "training_release_verification_failed"
|
||||||
|
state["training_release_error"] = str(exc)
|
||||||
|
write_state(state_path, state)
|
||||||
|
return 6
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
["docker", "exec", "-d", args.container, args.yolo, "train",
|
["docker", "exec", "-d", args.container, args.yolo, "train",
|
||||||
f"resume={args.container_checkpoint}", "device=0"],
|
f"resume={args.container_checkpoint}", "device=0"],
|
||||||
|
|||||||
@@ -4,8 +4,248 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import hashlib
|
||||||
import json
|
import json
|
||||||
from pathlib import Path
|
import sys
|
||||||
|
from collections import Counter
|
||||||
|
from pathlib import Path, PurePosixPath
|
||||||
|
from typing import Any, Mapping
|
||||||
|
|
||||||
|
|
||||||
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||||
|
if str(SCRIPT_DIR) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SCRIPT_DIR))
|
||||||
|
|
||||||
|
from training_dataset_eligibility import ( # noqa: E402
|
||||||
|
TrainingEligibilityError,
|
||||||
|
assert_frozen_manifest_training_eligible,
|
||||||
|
)
|
||||||
|
from training_release_manifest import ( # noqa: E402
|
||||||
|
TrainingReleaseError,
|
||||||
|
assert_yolo_summary_bound_to_embedded_training_release,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
PROPOSAL_DATASET_PROVENANCE_NAME = "proposal-dataset-provenance.json"
|
||||||
|
PROPOSAL_DATASET_SCHEMA_VERSION = 1
|
||||||
|
_CROP_IMAGE_SUFFIXES = {".bmp", ".jpeg", ".jpg", ".png", ".tif", ".tiff", ".webp"}
|
||||||
|
|
||||||
|
|
||||||
|
class ProposalDatasetProvenanceError(ValueError):
|
||||||
|
"""Raised when proposal crops no longer match their governed provenance."""
|
||||||
|
|
||||||
|
|
||||||
|
def sha256(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical_json_bytes(payload: Mapping[str, Any]) -> bytes:
|
||||||
|
return json.dumps(payload, ensure_ascii=False, separators=(",", ":"), sort_keys=True).encode("utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _payload_sha256(payload: Mapping[str, Any]) -> str:
|
||||||
|
normalized = dict(payload)
|
||||||
|
normalized.pop("manifest_sha256", None)
|
||||||
|
return hashlib.sha256(_canonical_json_bytes(normalized)).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _is_sha256(value: Any) -> bool:
|
||||||
|
return isinstance(value, str) and len(value) == 64 and all(character in "0123456789abcdef" for character in value)
|
||||||
|
|
||||||
|
|
||||||
|
def _require_mapping(value: Any, *, field: str) -> Mapping[str, Any]:
|
||||||
|
if not isinstance(value, Mapping):
|
||||||
|
raise ProposalDatasetProvenanceError(f"proposal dataset provenance {field} must be an object")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _require_sha256(value: Any, *, field: str) -> str:
|
||||||
|
if not _is_sha256(value):
|
||||||
|
raise ProposalDatasetProvenanceError(f"proposal dataset provenance {field} must be a lowercase SHA-256")
|
||||||
|
return str(value)
|
||||||
|
|
||||||
|
|
||||||
|
def _require_nonempty_path(value: Any, *, field: str) -> str:
|
||||||
|
if not isinstance(value, str) or not value.strip():
|
||||||
|
raise ProposalDatasetProvenanceError(f"proposal dataset provenance {field} must be a non-empty path")
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_relative_crop_path(value: Any, *, dataset_root: Path) -> tuple[str, Path]:
|
||||||
|
if not isinstance(value, str) or not value or "\\" in value:
|
||||||
|
raise ProposalDatasetProvenanceError("proposal crop relative_path must be a non-empty POSIX relative path")
|
||||||
|
relative = PurePosixPath(value)
|
||||||
|
if relative.is_absolute() or ".." in relative.parts or "." in relative.parts:
|
||||||
|
raise ProposalDatasetProvenanceError(f"proposal crop has an unsafe relative path: {value!r}")
|
||||||
|
path = (dataset_root / Path(*relative.parts)).resolve(strict=False)
|
||||||
|
try:
|
||||||
|
path.relative_to(dataset_root)
|
||||||
|
except ValueError as exc: # pragma: no cover - resolved-path defence
|
||||||
|
raise ProposalDatasetProvenanceError(f"proposal crop escapes dataset directory: {value!r}") from exc
|
||||||
|
return relative.as_posix(), path
|
||||||
|
|
||||||
|
|
||||||
|
def load_governed_corpus_manifest(corpus_manifest_path: Path, *, fixture_mode: bool) -> dict[str, Any]:
|
||||||
|
"""Assert frozen evidence and current live Dataset eligibility before PyTorch."""
|
||||||
|
|
||||||
|
try:
|
||||||
|
manifest = assert_frozen_manifest_training_eligible(
|
||||||
|
corpus_manifest_path,
|
||||||
|
fixture_mode=fixture_mode,
|
||||||
|
verify_live=True,
|
||||||
|
)
|
||||||
|
except TrainingEligibilityError as exc:
|
||||||
|
raise ProposalDatasetProvenanceError(str(exc)) from exc
|
||||||
|
if not isinstance(manifest, dict): # pragma: no cover - defensive contract boundary
|
||||||
|
raise ProposalDatasetProvenanceError("governed corpus manifest must be a JSON object")
|
||||||
|
return manifest
|
||||||
|
|
||||||
|
|
||||||
|
def validate_proposal_dataset_provenance(
|
||||||
|
dataset_dir: Path,
|
||||||
|
corpus_manifest_path: Path,
|
||||||
|
*,
|
||||||
|
fixture_mode: bool,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Fail closed unless every trainable crop still matches its source evidence.
|
||||||
|
|
||||||
|
ImageFolder discovers files from the directory tree. Merely verifying a
|
||||||
|
manifest is insufficient: any unbound image placed under train/ or val/
|
||||||
|
would otherwise enter PyTorch training. This validator compares the
|
||||||
|
complete tree with the immutable crop manifest and re-binds it to the
|
||||||
|
supplied frozen corpus and summary.
|
||||||
|
"""
|
||||||
|
|
||||||
|
root = dataset_dir.expanduser().resolve(strict=False)
|
||||||
|
if not root.is_dir():
|
||||||
|
raise ProposalDatasetProvenanceError(f"proposal dataset directory is unavailable: {root}")
|
||||||
|
manifest_path = root / PROPOSAL_DATASET_PROVENANCE_NAME
|
||||||
|
try:
|
||||||
|
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, json.JSONDecodeError) as exc:
|
||||||
|
raise ProposalDatasetProvenanceError(f"proposal dataset provenance is unreadable: {manifest_path}") from exc
|
||||||
|
if not isinstance(payload, dict):
|
||||||
|
raise ProposalDatasetProvenanceError("proposal dataset provenance must be a JSON object")
|
||||||
|
if payload.get("schema_version") != PROPOSAL_DATASET_SCHEMA_VERSION:
|
||||||
|
raise ProposalDatasetProvenanceError("proposal dataset provenance schema_version is unsupported")
|
||||||
|
if payload.get("status") != "ok" or payload.get("immutable") is not True:
|
||||||
|
raise ProposalDatasetProvenanceError("proposal dataset provenance is not an immutable successful release")
|
||||||
|
if payload.get("governed_corpus_live_recheck") is not True:
|
||||||
|
raise ProposalDatasetProvenanceError("proposal dataset provenance lacks the governed live-source recheck")
|
||||||
|
if bool(payload.get("fixture_mode")) != fixture_mode:
|
||||||
|
raise ProposalDatasetProvenanceError("proposal dataset provenance fixture-mode binding mismatches this invocation")
|
||||||
|
if payload.get("manifest_sha256") != _payload_sha256(payload):
|
||||||
|
raise ProposalDatasetProvenanceError("proposal dataset provenance manifest checksum mismatches its content")
|
||||||
|
|
||||||
|
expected_corpus_sha256 = sha256(corpus_manifest_path)
|
||||||
|
source = _require_mapping(payload.get("source"), field="source")
|
||||||
|
recorded_corpus = _require_mapping(source.get("corpus_manifest"), field="source.corpus_manifest")
|
||||||
|
if _require_sha256(recorded_corpus.get("sha256"), field="source.corpus_manifest.sha256") != expected_corpus_sha256:
|
||||||
|
raise ProposalDatasetProvenanceError("proposal crops are not bound to the supplied governed corpus manifest")
|
||||||
|
|
||||||
|
corpus_freeze_path = corpus_manifest_path.parent / "corpus-freeze.json"
|
||||||
|
recorded_freeze = _require_mapping(source.get("corpus_freeze"), field="source.corpus_freeze")
|
||||||
|
if not corpus_freeze_path.is_file() or _require_sha256(
|
||||||
|
recorded_freeze.get("sha256"), field="source.corpus_freeze.sha256"
|
||||||
|
) != sha256(corpus_freeze_path):
|
||||||
|
raise ProposalDatasetProvenanceError("proposal crops are not bound to the current frozen corpus sidecar")
|
||||||
|
|
||||||
|
summary = _require_mapping(source.get("summary"), field="source.summary")
|
||||||
|
summary_path_value = _require_nonempty_path(summary.get("path"), field="source.summary.path")
|
||||||
|
summary_path = Path(summary_path_value).expanduser().resolve(strict=False)
|
||||||
|
if not summary_path.is_file():
|
||||||
|
raise ProposalDatasetProvenanceError(f"proposal source summary is unavailable: {summary_path}")
|
||||||
|
if _require_sha256(summary.get("sha256"), field="source.summary.sha256") != sha256(summary_path):
|
||||||
|
raise ProposalDatasetProvenanceError("proposal source summary checksum no longer matches the crop provenance")
|
||||||
|
try:
|
||||||
|
summary_payload = json.loads(summary_path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, json.JSONDecodeError) as exc:
|
||||||
|
raise ProposalDatasetProvenanceError(f"proposal source summary is unreadable: {summary_path}") from exc
|
||||||
|
if not isinstance(summary_payload, Mapping) or summary_payload.get("source_manifest_sha256") != expected_corpus_sha256:
|
||||||
|
raise ProposalDatasetProvenanceError("proposal source summary is not bound to the supplied governed corpus manifest")
|
||||||
|
if summary.get("source_manifest_sha256") != expected_corpus_sha256:
|
||||||
|
raise ProposalDatasetProvenanceError("proposal crop provenance summary binding is invalid")
|
||||||
|
try:
|
||||||
|
source_release = assert_yolo_summary_bound_to_embedded_training_release(
|
||||||
|
summary_path=summary_path,
|
||||||
|
corpus_manifest=corpus_manifest_path,
|
||||||
|
fixture_mode=fixture_mode,
|
||||||
|
)
|
||||||
|
except TrainingReleaseError as exc:
|
||||||
|
raise ProposalDatasetProvenanceError(
|
||||||
|
"proposal source summary has no eligible immutable training release"
|
||||||
|
) from exc
|
||||||
|
recorded_release = _require_mapping(source.get("training_release"), field="source.training_release")
|
||||||
|
if (
|
||||||
|
recorded_release.get("dataset_yaml_path") != source_release["dataset_yaml"]["path"]
|
||||||
|
or recorded_release.get("dataset_yaml_sha256") != source_release["dataset_yaml"]["sha256"]
|
||||||
|
or recorded_release.get("corpus_manifest_sha256") != source_release["corpus"]["manifest_sha256"]
|
||||||
|
):
|
||||||
|
raise ProposalDatasetProvenanceError("proposal crop provenance release binding is invalid")
|
||||||
|
|
||||||
|
proposal_model = _require_mapping(source.get("proposal_model"), field="source.proposal_model")
|
||||||
|
_require_nonempty_path(proposal_model.get("path"), field="source.proposal_model.path")
|
||||||
|
_require_sha256(proposal_model.get("sha256"), field="source.proposal_model.sha256")
|
||||||
|
entries = payload.get("crops")
|
||||||
|
if not isinstance(entries, list) or not entries:
|
||||||
|
raise ProposalDatasetProvenanceError("proposal dataset provenance has no crop entries")
|
||||||
|
|
||||||
|
expected_paths: set[str] = set()
|
||||||
|
calculated_counts: Counter[str] = Counter()
|
||||||
|
normalized_entries: list[dict[str, Any]] = []
|
||||||
|
for entry in entries:
|
||||||
|
item = _require_mapping(entry, field="crops[]")
|
||||||
|
relative_path, crop_path = _safe_relative_crop_path(item.get("relative_path"), dataset_root=root)
|
||||||
|
if relative_path in expected_paths:
|
||||||
|
raise ProposalDatasetProvenanceError(f"proposal crop provenance has duplicate path: {relative_path}")
|
||||||
|
expected_paths.add(relative_path)
|
||||||
|
split = item.get("split")
|
||||||
|
label = item.get("label")
|
||||||
|
if split not in {"train", "val"} or label not in {"negative", "positive"}:
|
||||||
|
raise ProposalDatasetProvenanceError(f"proposal crop has invalid split/class: {relative_path}")
|
||||||
|
if not isinstance(item.get("sample_slug"), str) or not item["sample_slug"].strip():
|
||||||
|
raise ProposalDatasetProvenanceError(f"proposal crop has no sample identity: {relative_path}")
|
||||||
|
if not crop_path.is_file():
|
||||||
|
raise ProposalDatasetProvenanceError(f"proposal crop is missing: {relative_path}")
|
||||||
|
if _require_sha256(item.get("sha256"), field=f"crops[{relative_path}].sha256") != sha256(crop_path):
|
||||||
|
raise ProposalDatasetProvenanceError(f"proposal crop checksum mismatches provenance: {relative_path}")
|
||||||
|
if item.get("size_bytes") != crop_path.stat().st_size:
|
||||||
|
raise ProposalDatasetProvenanceError(f"proposal crop byte size mismatches provenance: {relative_path}")
|
||||||
|
_require_nonempty_path(item.get("source_image_path"), field=f"crops[{relative_path}].source_image_path")
|
||||||
|
_require_sha256(item.get("source_image_sha256"), field=f"crops[{relative_path}].source_image_sha256")
|
||||||
|
_require_nonempty_path(item.get("source_label_path"), field=f"crops[{relative_path}].source_label_path")
|
||||||
|
_require_sha256(item.get("source_label_sha256"), field=f"crops[{relative_path}].source_label_sha256")
|
||||||
|
calculated_counts[f"{split}/{label}"] += 1
|
||||||
|
normalized_entries.append(dict(item))
|
||||||
|
|
||||||
|
canonical_entries = sorted(normalized_entries, key=lambda item: str(item["relative_path"]))
|
||||||
|
expected_crops_sha256 = hashlib.sha256(_canonical_json_bytes({"crops": canonical_entries})).hexdigest()
|
||||||
|
if payload.get("crops_sha256") != expected_crops_sha256:
|
||||||
|
raise ProposalDatasetProvenanceError("proposal crop collection checksum mismatches provenance")
|
||||||
|
if payload.get("crop_count") != len(entries):
|
||||||
|
raise ProposalDatasetProvenanceError("proposal crop count mismatches provenance")
|
||||||
|
if payload.get("counts") != dict(sorted(calculated_counts.items())):
|
||||||
|
raise ProposalDatasetProvenanceError("proposal crop class counts mismatch provenance")
|
||||||
|
if any(calculated_counts[f"{split}/{label}"] < 1 for split in ("train", "val") for label in ("negative", "positive")):
|
||||||
|
raise ProposalDatasetProvenanceError("proposal dataset has an empty train/validation class")
|
||||||
|
|
||||||
|
actual_paths = {
|
||||||
|
file_path.relative_to(root).as_posix()
|
||||||
|
for file_path in root.rglob("*")
|
||||||
|
if file_path.is_file() and file_path.suffix.lower() in _CROP_IMAGE_SUFFIXES
|
||||||
|
}
|
||||||
|
if actual_paths != expected_paths:
|
||||||
|
unexpected = sorted(actual_paths - expected_paths)
|
||||||
|
missing = sorted(expected_paths - actual_paths)
|
||||||
|
raise ProposalDatasetProvenanceError(
|
||||||
|
"proposal dataset files are not exactly the immutable crop manifest "
|
||||||
|
f"(unexpected={unexpected}, missing={missing})"
|
||||||
|
)
|
||||||
|
return payload
|
||||||
|
|
||||||
|
|
||||||
def binary_metrics(scores: list[float], labels: list[int], threshold: float = 0.5) -> dict[str, float | int]:
|
def binary_metrics(scores: list[float], labels: list[int], threshold: float = 0.5) -> dict[str, float | int]:
|
||||||
@@ -21,16 +261,44 @@ def binary_metrics(scores: list[float], labels: list[int], threshold: float = 0.
|
|||||||
def main() -> int:
|
def main() -> int:
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
parser.add_argument("--dataset-dir", type=Path, required=True)
|
parser.add_argument("--dataset-dir", type=Path, required=True)
|
||||||
|
parser.add_argument(
|
||||||
|
"--corpus-manifest",
|
||||||
|
type=Path,
|
||||||
|
required=True,
|
||||||
|
help="The exact frozen governed corpus manifest that produced the proposal crops.",
|
||||||
|
)
|
||||||
parser.add_argument("--output-dir", type=Path, required=True)
|
parser.add_argument("--output-dir", type=Path, required=True)
|
||||||
parser.add_argument("--epochs", type=int, default=12)
|
parser.add_argument("--epochs", type=int, default=12)
|
||||||
parser.add_argument("--batch", type=int, default=64)
|
parser.add_argument("--batch", type=int, default=64)
|
||||||
parser.add_argument("--lr", type=float, default=1e-4)
|
parser.add_argument("--lr", type=float, default=1e-4)
|
||||||
parser.add_argument("--device", default="cuda:0")
|
parser.add_argument("--device", default="cuda:0")
|
||||||
parser.add_argument("--export-existing-best", action="store_true")
|
parser.add_argument("--export-existing-best", action="store_true")
|
||||||
|
parser.add_argument(
|
||||||
|
"--fixture-mode",
|
||||||
|
action="store_true",
|
||||||
|
help="Permit only an explicitly fixture-only corpus; operational runs always resolve live dataset evidence.",
|
||||||
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
if args.output_dir.exists() and not args.export_existing_best:
|
if args.output_dir.exists() and not args.export_existing_best:
|
||||||
parser.error(f"output already exists: {args.output_dir}")
|
parser.error(f"output already exists: {args.output_dir}")
|
||||||
|
|
||||||
|
# All governed-source checks run before importing torch/torchvision or
|
||||||
|
# touching CUDA. A source revocation therefore cannot start PyTorch work.
|
||||||
|
try:
|
||||||
|
governed_corpus = load_governed_corpus_manifest(
|
||||||
|
args.corpus_manifest,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
|
proposal_provenance = validate_proposal_dataset_provenance(
|
||||||
|
args.dataset_dir,
|
||||||
|
args.corpus_manifest,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
|
except ProposalDatasetProvenanceError as exc:
|
||||||
|
raise SystemExit(str(exc)) from exc
|
||||||
|
proposal_provenance_path = args.dataset_dir / PROPOSAL_DATASET_PROVENANCE_NAME
|
||||||
|
proposal_provenance_sha256 = sha256(proposal_provenance_path)
|
||||||
|
|
||||||
import torch
|
import torch
|
||||||
from torch import nn
|
from torch import nn
|
||||||
from torch.utils.data import DataLoader
|
from torch.utils.data import DataLoader
|
||||||
@@ -54,7 +322,18 @@ def main() -> int:
|
|||||||
model.load_state_dict(torch.load(state_path, map_location=device))
|
model.load_state_dict(torch.load(state_path, map_location=device))
|
||||||
model.eval()
|
model.eval()
|
||||||
torch.jit.script(model).save(str(args.output_dir / "proposal-classifier.torchscript.pt"))
|
torch.jit.script(model).save(str(args.output_dir / "proposal-classifier.torchscript.pt"))
|
||||||
print(json.dumps({"status": "exported_existing_best", "model": str(state_path)}))
|
print(
|
||||||
|
json.dumps(
|
||||||
|
{
|
||||||
|
"status": "exported_existing_best",
|
||||||
|
"model": str(state_path),
|
||||||
|
"proposal_dataset_provenance": str(proposal_provenance_path),
|
||||||
|
"proposal_dataset_provenance_sha256": proposal_provenance_sha256,
|
||||||
|
"governed_corpus_manifest_sha256": sha256(args.corpus_manifest),
|
||||||
|
"governed_corpus_live_recheck": True,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
)
|
||||||
return 0
|
return 0
|
||||||
train_loader = DataLoader(train_ds, batch_size=args.batch, shuffle=True, num_workers=0)
|
train_loader = DataLoader(train_ds, batch_size=args.batch, shuffle=True, num_workers=0)
|
||||||
val_loader = DataLoader(val_ds, batch_size=args.batch, shuffle=False, num_workers=0)
|
val_loader = DataLoader(val_ds, batch_size=args.batch, shuffle=False, num_workers=0)
|
||||||
@@ -93,11 +372,27 @@ def main() -> int:
|
|||||||
model.load_state_dict(torch.load(args.output_dir / "best-state.pt", map_location=device))
|
model.load_state_dict(torch.load(args.output_dir / "best-state.pt", map_location=device))
|
||||||
model.eval()
|
model.eval()
|
||||||
scripted = torch.jit.script(model)
|
scripted = torch.jit.script(model)
|
||||||
scripted.save(str(args.output_dir / "proposal-classifier.torchscript.pt"))
|
model_path = args.output_dir / "proposal-classifier.torchscript.pt"
|
||||||
report = {"schema_version": 1, "status": "ok", "classes": train_ds.class_to_idx,
|
scripted.save(str(model_path))
|
||||||
"train_count": len(train_ds), "validation_count": len(val_ds),
|
report = {
|
||||||
"best_validation_f1": best_f1, "history": history,
|
"schema_version": 1,
|
||||||
"model": str(args.output_dir / "proposal-classifier.torchscript.pt")}
|
"status": "ok",
|
||||||
|
"classes": train_ds.class_to_idx,
|
||||||
|
"train_count": len(train_ds),
|
||||||
|
"validation_count": len(val_ds),
|
||||||
|
"best_validation_f1": best_f1,
|
||||||
|
"history": history,
|
||||||
|
"model": str(model_path),
|
||||||
|
"model_sha256": sha256(model_path),
|
||||||
|
"proposal_dataset_provenance": str(proposal_provenance_path),
|
||||||
|
"proposal_dataset_provenance_sha256": proposal_provenance_sha256,
|
||||||
|
"proposal_dataset_manifest_sha256": proposal_provenance["manifest_sha256"],
|
||||||
|
"corpus_manifest": str(args.corpus_manifest),
|
||||||
|
"corpus_manifest_sha256": sha256(args.corpus_manifest),
|
||||||
|
"fixture_mode": bool(args.fixture_mode),
|
||||||
|
"governed_corpus_live_recheck": True,
|
||||||
|
"governed_corpus_sample_count": len(governed_corpus.get("samples", [])),
|
||||||
|
}
|
||||||
(args.output_dir / "training-report.json").write_text(json.dumps(report, indent=2), encoding="utf-8")
|
(args.output_dir / "training-report.json").write_text(json.dumps(report, indent=2), encoding="utf-8")
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|||||||
@@ -8,9 +8,23 @@ import hashlib
|
|||||||
import json
|
import json
|
||||||
import random
|
import random
|
||||||
import shutil
|
import shutil
|
||||||
|
import sys
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, TypeVar
|
from typing import Any, TypeVar
|
||||||
|
|
||||||
|
SCRIPT_DIR = Path(__file__).resolve().parent
|
||||||
|
if str(SCRIPT_DIR) not in sys.path:
|
||||||
|
sys.path.insert(0, str(SCRIPT_DIR))
|
||||||
|
|
||||||
|
from training_dataset_eligibility import ( # noqa: E402
|
||||||
|
TrainingEligibilityError,
|
||||||
|
assert_frozen_manifest_training_eligible,
|
||||||
|
)
|
||||||
|
from training_release_manifest import ( # noqa: E402
|
||||||
|
TrainingReleaseError,
|
||||||
|
assert_yolo_summary_bound_to_training_release,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
T = TypeVar("T")
|
T = TypeVar("T")
|
||||||
|
|
||||||
@@ -111,6 +125,18 @@ def choose_threshold(probabilities: list[float], labels: list[int]) -> tuple[flo
|
|||||||
def main() -> int:
|
def main() -> int:
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
parser.add_argument("--summary", type=Path, required=True)
|
parser.add_argument("--summary", type=Path, required=True)
|
||||||
|
parser.add_argument(
|
||||||
|
"--corpus-manifest",
|
||||||
|
type=Path,
|
||||||
|
required=True,
|
||||||
|
help="Immutable governed corpus manifest that produced the tile summary.",
|
||||||
|
)
|
||||||
|
parser.add_argument(
|
||||||
|
"--train-yaml",
|
||||||
|
type=Path,
|
||||||
|
required=True,
|
||||||
|
help="YOLO dataset YAML with an eligible immutable GeoIntel training release.",
|
||||||
|
)
|
||||||
parser.add_argument("--proposal-model", type=Path, required=True)
|
parser.add_argument("--proposal-model", type=Path, required=True)
|
||||||
parser.add_argument("--output-dir", type=Path, required=True)
|
parser.add_argument("--output-dir", type=Path, required=True)
|
||||||
parser.add_argument("--device", default="cuda:0")
|
parser.add_argument("--device", default="cuda:0")
|
||||||
@@ -124,7 +150,31 @@ def main() -> int:
|
|||||||
parser.add_argument("--proposal-chunk-size", type=int, default=16)
|
parser.add_argument("--proposal-chunk-size", type=int, default=16)
|
||||||
parser.add_argument("--seed", type=int, default=20260727)
|
parser.add_argument("--seed", type=int, default=20260727)
|
||||||
parser.add_argument("--force", action="store_true")
|
parser.add_argument("--force", action="store_true")
|
||||||
|
parser.add_argument(
|
||||||
|
"--fixture-mode",
|
||||||
|
action="store_true",
|
||||||
|
help="Accept only an explicitly fixture-only corpus manifest; never use for operational training.",
|
||||||
|
)
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
try:
|
||||||
|
assert_frozen_manifest_training_eligible(
|
||||||
|
args.corpus_manifest,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
verify_live=True,
|
||||||
|
)
|
||||||
|
assert_yolo_summary_bound_to_training_release(
|
||||||
|
summary_path=args.summary,
|
||||||
|
train_yaml=args.train_yaml,
|
||||||
|
corpus_manifest=args.corpus_manifest,
|
||||||
|
fixture_mode=args.fixture_mode,
|
||||||
|
)
|
||||||
|
except (TrainingEligibilityError, TrainingReleaseError) as exc:
|
||||||
|
raise SystemExit(str(exc)) from exc
|
||||||
|
summary = json.loads(args.summary.read_text(encoding="utf-8"))
|
||||||
|
if summary.get("source_manifest_sha256") != sha256(args.corpus_manifest):
|
||||||
|
raise SystemExit(
|
||||||
|
"Proposal-filter tile summary is not bound to the supplied governed corpus manifest"
|
||||||
|
)
|
||||||
if args.output_dir.exists():
|
if args.output_dir.exists():
|
||||||
if not args.force:
|
if not args.force:
|
||||||
raise SystemExit(f"Output exists: {args.output_dir}")
|
raise SystemExit(f"Output exists: {args.output_dir}")
|
||||||
@@ -133,12 +183,11 @@ def main() -> int:
|
|||||||
import torch
|
import torch
|
||||||
from torch import nn
|
from torch import nn
|
||||||
from torch.utils.data import DataLoader
|
from torch.utils.data import DataLoader
|
||||||
from torchvision import datasets, transforms
|
from torchvision import datasets
|
||||||
from torchvision.models import ResNet18_Weights, resnet18
|
from torchvision.models import ResNet18_Weights, resnet18
|
||||||
from ultralytics import YOLO
|
from ultralytics import YOLO
|
||||||
|
|
||||||
torch.manual_seed(args.seed)
|
torch.manual_seed(args.seed)
|
||||||
summary = json.loads(args.summary.read_text(encoding="utf-8"))
|
|
||||||
split_tiles = {
|
split_tiles = {
|
||||||
split: [tile for tile in summary["tiles"] if tile.get("kept", True) and tile["split"] == split]
|
split: [tile for tile in summary["tiles"] if tile.get("kept", True) and tile["split"] == split]
|
||||||
for split in ("train", "val")
|
for split in ("train", "val")
|
||||||
@@ -214,6 +263,9 @@ def main() -> int:
|
|||||||
evidence = {
|
evidence = {
|
||||||
"schema_version": 1, "status": "ok", "architecture": "resnet18_binary_proposal_filter",
|
"schema_version": 1, "status": "ok", "architecture": "resnet18_binary_proposal_filter",
|
||||||
"summary": str(args.summary), "summary_sha256": sha256(args.summary),
|
"summary": str(args.summary), "summary_sha256": sha256(args.summary),
|
||||||
|
"corpus_manifest": str(args.corpus_manifest),
|
||||||
|
"corpus_manifest_sha256": sha256(args.corpus_manifest),
|
||||||
|
"fixture_mode": bool(args.fixture_mode),
|
||||||
"proposal_model": str(args.proposal_model), "proposal_model_sha256": sha256(args.proposal_model),
|
"proposal_model": str(args.proposal_model), "proposal_model_sha256": sha256(args.proposal_model),
|
||||||
"device": args.device, "proposal_confidence": args.proposal_confidence, "crop_scale": args.crop_scale,
|
"device": args.device, "proposal_confidence": args.proposal_confidence, "crop_scale": args.crop_scale,
|
||||||
"proposal_chunk_size": args.proposal_chunk_size,
|
"proposal_chunk_size": args.proposal_chunk_size,
|
||||||
|
|||||||
@@ -53,6 +53,8 @@ if [[ -z "${PYTHON_BIN:-}" ]]; then
|
|||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
|
||||||
DATASET_YAML="${OPERATOR_YOLO_DATASET_DIR%/}/dataset.yaml"
|
DATASET_YAML="${OPERATOR_YOLO_DATASET_DIR%/}/dataset.yaml"
|
||||||
SUMMARY_PATH="${TRAIN_OUTPUT_DIR%/}/${TRAIN_RUN_NAME}/training_summary.json"
|
SUMMARY_PATH="${TRAIN_OUTPUT_DIR%/}/${TRAIN_RUN_NAME}/training_summary.json"
|
||||||
export DATASET_YAML
|
export DATASET_YAML
|
||||||
@@ -79,6 +81,11 @@ if [[ ! -f "${YOLO_BASE_MODEL_PATH}" ]]; then
|
|||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# A filename is not a training identity. Verify the immutable YAML/corpus/
|
||||||
|
# asset/review release before importing Ultralytics or allocating CUDA work.
|
||||||
|
"${PYTHON_BIN}" "${SCRIPT_DIR}/training_release_manifest.py" verify \
|
||||||
|
--train-yaml "${DATASET_YAML}"
|
||||||
|
|
||||||
mkdir -p "${TRAIN_OUTPUT_DIR}" "$(dirname "${TRAIN_MODEL_OUTPUT_PATH}")"
|
mkdir -p "${TRAIN_OUTPUT_DIR}" "$(dirname "${TRAIN_MODEL_OUTPUT_PATH}")"
|
||||||
|
|
||||||
"${PYTHON_BIN}" - <<'PY'
|
"${PYTHON_BIN}" - <<'PY'
|
||||||
|
|||||||
@@ -0,0 +1,545 @@
|
|||||||
|
"""Fail-closed provenance gate for Belgian building-training inputs.
|
||||||
|
|
||||||
|
The database data-contract validator decides whether a dataset can be stored.
|
||||||
|
This module is intentionally a second, independent gate at the point where
|
||||||
|
persisted datasets become irreversible training pairs. It has no database
|
||||||
|
side effects, making the decision reproducible in a corpus manifest and easy
|
||||||
|
to re-check before CUDA training starts.
|
||||||
|
|
||||||
|
Legacy data may only pass through this module in explicit fixture mode. That
|
||||||
|
mode is deliberately limited to records marked as fixtures and must never be
|
||||||
|
used for an operational corpus.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from collections.abc import Mapping
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Callable, Literal
|
||||||
|
from uuid import UUID
|
||||||
|
|
||||||
|
|
||||||
|
TRAINING_ELIGIBILITY_POLICY_VERSION = "geointel-training-source-eligibility/v1"
|
||||||
|
DatasetRole = Literal["raster", "reference"]
|
||||||
|
|
||||||
|
_SHA256_RE = re.compile(r"^[0-9a-f]{64}$", re.IGNORECASE)
|
||||||
|
_ALLOWED_SOURCE_CLASSIFICATIONS = {
|
||||||
|
"authoritative",
|
||||||
|
"corroborative",
|
||||||
|
"contextual",
|
||||||
|
"derived",
|
||||||
|
}
|
||||||
|
_FIXTURE_SOURCES = {"fixture", "test", "test_fixture", "test-fixture"}
|
||||||
|
|
||||||
|
|
||||||
|
class TrainingEligibilityError(ValueError):
|
||||||
|
"""Raised when persisted inputs cannot be used in an operational corpus."""
|
||||||
|
|
||||||
|
|
||||||
|
class TrainingEligibilityResult:
|
||||||
|
"""Stable, JSON-serializable decision for one persisted dataset."""
|
||||||
|
|
||||||
|
__slots__ = ("role", "eligible", "fixture_mode", "reasons", "evidence")
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
*,
|
||||||
|
role: DatasetRole,
|
||||||
|
eligible: bool,
|
||||||
|
fixture_mode: bool,
|
||||||
|
reasons: tuple[str, ...],
|
||||||
|
evidence: dict[str, Any],
|
||||||
|
) -> None:
|
||||||
|
self.role = role
|
||||||
|
self.eligible = eligible
|
||||||
|
self.fixture_mode = fixture_mode
|
||||||
|
self.reasons = reasons
|
||||||
|
self.evidence = evidence
|
||||||
|
|
||||||
|
def as_dict(self) -> dict[str, Any]:
|
||||||
|
return {
|
||||||
|
"policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||||
|
"role": self.role,
|
||||||
|
"eligible": self.eligible,
|
||||||
|
"fixture_mode": self.fixture_mode,
|
||||||
|
"reasons": list(self.reasons),
|
||||||
|
"evidence": self.evidence,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _value(record: Any, field: str, default: Any = None) -> Any:
|
||||||
|
if isinstance(record, Mapping):
|
||||||
|
return record.get(field, default)
|
||||||
|
return getattr(record, field, default)
|
||||||
|
|
||||||
|
|
||||||
|
def _normalise_status(value: Any) -> str:
|
||||||
|
return str(value or "").strip().lower()
|
||||||
|
|
||||||
|
|
||||||
|
def _normalise_mapping(value: Any) -> dict[str, Any]:
|
||||||
|
return dict(value) if isinstance(value, Mapping) else {}
|
||||||
|
|
||||||
|
|
||||||
|
def _source_identity(dataset: Any) -> tuple[str, str]:
|
||||||
|
return (
|
||||||
|
_normalise_status(_value(dataset, "source")),
|
||||||
|
_normalise_status(_value(dataset, "source_name")),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_explicit_fixture(dataset: Any, registry: Any) -> bool:
|
||||||
|
source, source_name = _source_identity(dataset)
|
||||||
|
metadata = _normalise_mapping(_value(dataset, "metadata_json"))
|
||||||
|
provenance = _normalise_mapping(_value(dataset, "provenance_metadata"))
|
||||||
|
usage_policy = _normalise_mapping(_value(registry, "usage_policy_json"))
|
||||||
|
return bool(
|
||||||
|
source in _FIXTURE_SOURCES
|
||||||
|
or source_name in _FIXTURE_SOURCES
|
||||||
|
or metadata.get("fixture") is True
|
||||||
|
or provenance.get("fixture") is True
|
||||||
|
or usage_policy.get("fixture_only") is True
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _dataset_evidence(dataset: Any, registry: Any, snapshot: Any) -> dict[str, Any]:
|
||||||
|
usage_policy = _normalise_mapping(_value(registry, "usage_policy_json"))
|
||||||
|
allowed_tasks = usage_policy.get("allowed_tasks")
|
||||||
|
validation_authority = _normalise_mapping(usage_policy.get("validation_authority"))
|
||||||
|
return {
|
||||||
|
"dataset_id": str(_value(dataset, "id") or ""),
|
||||||
|
"dataset_type": _normalise_status(_value(dataset, "dataset_type")),
|
||||||
|
"dataset_role": _normalise_status(_value(dataset, "dataset_role")),
|
||||||
|
"source": _normalise_status(_value(dataset, "source")),
|
||||||
|
"source_name": _normalise_status(_value(dataset, "source_name")),
|
||||||
|
"checksum_sha256": _value(dataset, "checksum_sha256"),
|
||||||
|
"data_contract_key": _value(dataset, "data_contract_key"),
|
||||||
|
"data_contract_version": _value(dataset, "data_contract_version"),
|
||||||
|
"validation_status": _normalise_status(_value(dataset, "validation_status")),
|
||||||
|
"provenance_status": _normalise_status(_value(dataset, "provenance_status")),
|
||||||
|
"lineage_status": _normalise_status(_value(dataset, "lineage_status")),
|
||||||
|
"quarantine_status": _normalise_status(_value(dataset, "quarantine_status")),
|
||||||
|
"dataset_status": _normalise_status(_value(dataset, "status")),
|
||||||
|
"dataset_source_registry_id": str(_value(dataset, "source_registry_id") or ""),
|
||||||
|
"dataset_source_snapshot_id": str(_value(dataset, "source_snapshot_id") or ""),
|
||||||
|
"source_registry_id": str(_value(registry, "id") or ""),
|
||||||
|
"source_key": _normalise_status(_value(registry, "source_key")),
|
||||||
|
"source_classification": _normalise_status(_value(registry, "classification")),
|
||||||
|
"source_freshness_status": _normalise_status(_value(registry, "freshness_status")),
|
||||||
|
"source_ingest_status": _normalise_status(_value(registry, "ingest_status")),
|
||||||
|
"source_training_allowed": usage_policy.get("training_allowed"),
|
||||||
|
"source_ground_truth_allowed": usage_policy.get("ground_truth_allowed"),
|
||||||
|
"source_allowed_tasks": list(allowed_tasks) if isinstance(allowed_tasks, list) else [],
|
||||||
|
"source_validation_authority": validation_authority,
|
||||||
|
"source_building_validation_authority": validation_authority.get("building_validation"),
|
||||||
|
"source_snapshot_id": str(_value(snapshot, "id") or _value(dataset, "source_snapshot_id") or ""),
|
||||||
|
"snapshot_source_registry_id": str(_value(snapshot, "source_registry_id") or ""),
|
||||||
|
"snapshot_key": _value(snapshot, "snapshot_key"),
|
||||||
|
"snapshot_checksum_sha256": _value(snapshot, "checksum_sha256"),
|
||||||
|
"snapshot_freshness_status": _normalise_status(_value(snapshot, "freshness_status")),
|
||||||
|
"snapshot_ingest_status": _normalise_status(_value(snapshot, "ingest_status")),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_dataset_training_eligibility(
|
||||||
|
dataset: Any,
|
||||||
|
*,
|
||||||
|
role: DatasetRole,
|
||||||
|
fixture_mode: bool = False,
|
||||||
|
) -> TrainingEligibilityResult:
|
||||||
|
"""Evaluate a source Dataset without mutating it.
|
||||||
|
|
||||||
|
Operational calls require server-attested source registry and snapshot
|
||||||
|
relationships. ``fixture_mode`` can relax only legacy provenance fields,
|
||||||
|
and only for an explicitly marked fixture. A failed validation or a
|
||||||
|
quarantine is never relaxed.
|
||||||
|
"""
|
||||||
|
|
||||||
|
registry = _value(dataset, "source_registry")
|
||||||
|
snapshot = _value(dataset, "source_snapshot")
|
||||||
|
evidence = _dataset_evidence(dataset, registry, snapshot)
|
||||||
|
reasons: list[str] = []
|
||||||
|
|
||||||
|
dataset_type = evidence["dataset_type"]
|
||||||
|
dataset_role = evidence["dataset_role"]
|
||||||
|
if role == "raster" and dataset_type != "raster":
|
||||||
|
reasons.append("dataset_type_not_raster")
|
||||||
|
if role == "reference":
|
||||||
|
if dataset_type != "vector":
|
||||||
|
reasons.append("dataset_type_not_vector")
|
||||||
|
if dataset_role != "reference":
|
||||||
|
reasons.append("dataset_role_not_reference")
|
||||||
|
|
||||||
|
if evidence["dataset_status"] != "ready":
|
||||||
|
reasons.append("dataset_not_ready")
|
||||||
|
if evidence["quarantine_status"] == "quarantined":
|
||||||
|
reasons.append("dataset_quarantined")
|
||||||
|
|
||||||
|
explicit_fixture = _is_explicit_fixture(dataset, registry)
|
||||||
|
if fixture_mode and not explicit_fixture:
|
||||||
|
reasons.append("fixture_mode_requires_explicit_fixture")
|
||||||
|
validation_status = evidence["validation_status"]
|
||||||
|
if validation_status == "failed":
|
||||||
|
reasons.append("validation_failed")
|
||||||
|
elif validation_status != "passed" and not (fixture_mode and explicit_fixture):
|
||||||
|
reasons.append("validation_not_passed")
|
||||||
|
|
||||||
|
if not fixture_mode:
|
||||||
|
if evidence["provenance_status"] != "complete":
|
||||||
|
reasons.append("provenance_not_complete")
|
||||||
|
if evidence["lineage_status"] not in {"complete", "not_applicable"}:
|
||||||
|
reasons.append("lineage_not_complete")
|
||||||
|
if not evidence["data_contract_key"] or not evidence["data_contract_version"]:
|
||||||
|
reasons.append("data_contract_not_versioned")
|
||||||
|
checksum = str(evidence["checksum_sha256"] or "")
|
||||||
|
if not _SHA256_RE.fullmatch(checksum):
|
||||||
|
reasons.append("dataset_checksum_invalid")
|
||||||
|
if registry is None:
|
||||||
|
reasons.append("source_registry_missing")
|
||||||
|
if snapshot is None:
|
||||||
|
reasons.append("source_snapshot_missing")
|
||||||
|
if registry is not None:
|
||||||
|
if (
|
||||||
|
evidence["dataset_source_registry_id"]
|
||||||
|
and evidence["dataset_source_registry_id"] != evidence["source_registry_id"]
|
||||||
|
):
|
||||||
|
reasons.append("dataset_source_registry_binding_mismatch")
|
||||||
|
classification = evidence["source_classification"]
|
||||||
|
if classification not in _ALLOWED_SOURCE_CLASSIFICATIONS:
|
||||||
|
reasons.append("source_classification_not_allowed")
|
||||||
|
if evidence["source_training_allowed"] is not True:
|
||||||
|
reasons.append("source_not_allowed_for_training")
|
||||||
|
if evidence["source_ingest_status"] == "quarantined":
|
||||||
|
reasons.append("source_registry_quarantined")
|
||||||
|
if snapshot is not None:
|
||||||
|
if (
|
||||||
|
evidence["dataset_source_snapshot_id"]
|
||||||
|
and evidence["dataset_source_snapshot_id"] != evidence["source_snapshot_id"]
|
||||||
|
):
|
||||||
|
reasons.append("dataset_source_snapshot_binding_mismatch")
|
||||||
|
if (
|
||||||
|
registry is not None
|
||||||
|
and evidence["snapshot_source_registry_id"]
|
||||||
|
and evidence["snapshot_source_registry_id"] != evidence["source_registry_id"]
|
||||||
|
):
|
||||||
|
reasons.append("source_snapshot_registry_mismatch")
|
||||||
|
if evidence["snapshot_ingest_status"] != "ingested":
|
||||||
|
reasons.append("source_snapshot_not_ingested")
|
||||||
|
if evidence["snapshot_freshness_status"] not in {"current", "not_applicable"}:
|
||||||
|
reasons.append("source_snapshot_freshness_not_approved")
|
||||||
|
snapshot_checksum = str(evidence["snapshot_checksum_sha256"] or "")
|
||||||
|
if not _SHA256_RE.fullmatch(snapshot_checksum):
|
||||||
|
reasons.append("source_snapshot_checksum_invalid")
|
||||||
|
elif snapshot_checksum.lower() != checksum.lower():
|
||||||
|
reasons.append("source_snapshot_checksum_mismatch")
|
||||||
|
if role == "reference":
|
||||||
|
if evidence["source_classification"] != "authoritative":
|
||||||
|
reasons.append("reference_source_not_authoritative")
|
||||||
|
if evidence["source_ground_truth_allowed"] is not True:
|
||||||
|
reasons.append("reference_source_not_ground_truth_allowed")
|
||||||
|
if "building_validation" not in evidence["source_allowed_tasks"]:
|
||||||
|
reasons.append("reference_source_not_approved_for_building_validation")
|
||||||
|
if evidence["source_building_validation_authority"] != "primary":
|
||||||
|
reasons.append("reference_building_validation_not_primary")
|
||||||
|
|
||||||
|
return TrainingEligibilityResult(
|
||||||
|
role=role,
|
||||||
|
eligible=not reasons,
|
||||||
|
fixture_mode=fixture_mode,
|
||||||
|
reasons=tuple(sorted(set(reasons))),
|
||||||
|
evidence=evidence,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def assert_dataset_training_eligible(
|
||||||
|
dataset: Any,
|
||||||
|
*,
|
||||||
|
role: DatasetRole,
|
||||||
|
fixture_mode: bool = False,
|
||||||
|
sample_slug: str | None = None,
|
||||||
|
) -> TrainingEligibilityResult:
|
||||||
|
result = evaluate_dataset_training_eligibility(dataset, role=role, fixture_mode=fixture_mode)
|
||||||
|
if result.eligible:
|
||||||
|
return result
|
||||||
|
label = f" for sample {sample_slug!r}" if sample_slug else ""
|
||||||
|
raise TrainingEligibilityError(
|
||||||
|
f"{role} dataset is not eligible for training{label}: {', '.join(result.reasons)}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def training_pair_evidence(
|
||||||
|
*,
|
||||||
|
raster: Any,
|
||||||
|
reference: Any,
|
||||||
|
fixture_mode: bool = False,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Return manifest-ready evidence for one raster/reference pair."""
|
||||||
|
|
||||||
|
raster_result = evaluate_dataset_training_eligibility(
|
||||||
|
raster,
|
||||||
|
role="raster",
|
||||||
|
fixture_mode=fixture_mode,
|
||||||
|
)
|
||||||
|
reference_result = evaluate_dataset_training_eligibility(
|
||||||
|
reference,
|
||||||
|
role="reference",
|
||||||
|
fixture_mode=fixture_mode,
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"policy_version": TRAINING_ELIGIBILITY_POLICY_VERSION,
|
||||||
|
"eligible": raster_result.eligible and reference_result.eligible,
|
||||||
|
"fixture_mode": fixture_mode,
|
||||||
|
"raster": raster_result.as_dict(),
|
||||||
|
"reference": reference_result.as_dict(),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def manifest_training_eligibility_failures(
|
||||||
|
manifest: Mapping[str, Any],
|
||||||
|
*,
|
||||||
|
fixture_mode: bool = False,
|
||||||
|
) -> list[str]:
|
||||||
|
"""Re-check immutable eligibility evidence before an actual training run."""
|
||||||
|
|
||||||
|
failures: list[str] = []
|
||||||
|
eligibility = manifest.get("training_eligibility")
|
||||||
|
if not isinstance(eligibility, Mapping):
|
||||||
|
return ["manifest_training_eligibility_missing"]
|
||||||
|
if eligibility.get("policy_version") != TRAINING_ELIGIBILITY_POLICY_VERSION:
|
||||||
|
failures.append("manifest_training_eligibility_policy_invalid")
|
||||||
|
if eligibility.get("status") != "eligible":
|
||||||
|
failures.append("manifest_training_eligibility_not_eligible")
|
||||||
|
manifest_fixture_mode = eligibility.get("fixture_mode") is True
|
||||||
|
if manifest_fixture_mode != fixture_mode:
|
||||||
|
failures.append("manifest_fixture_mode_mismatch")
|
||||||
|
|
||||||
|
samples = manifest.get("samples")
|
||||||
|
if not isinstance(samples, list) or not samples:
|
||||||
|
failures.append("manifest_samples_missing")
|
||||||
|
return failures
|
||||||
|
for sample in samples:
|
||||||
|
if not isinstance(sample, Mapping):
|
||||||
|
failures.append("manifest_sample_invalid")
|
||||||
|
continue
|
||||||
|
slug = str(sample.get("sample_slug") or "<unknown>")
|
||||||
|
pair = sample.get("training_eligibility")
|
||||||
|
if not isinstance(pair, Mapping):
|
||||||
|
failures.append(f"{slug}:training_eligibility_missing")
|
||||||
|
continue
|
||||||
|
if pair.get("policy_version") != TRAINING_ELIGIBILITY_POLICY_VERSION:
|
||||||
|
failures.append(f"{slug}:training_eligibility_policy_invalid")
|
||||||
|
if pair.get("fixture_mode") is not fixture_mode:
|
||||||
|
failures.append(f"{slug}:fixture_mode_mismatch")
|
||||||
|
if pair.get("eligible") is not True:
|
||||||
|
failures.append(f"{slug}:training_pair_not_eligible")
|
||||||
|
for role in ("raster", "reference"):
|
||||||
|
decision = pair.get(role)
|
||||||
|
if not isinstance(decision, Mapping):
|
||||||
|
failures.append(f"{slug}:{role}_eligibility_missing")
|
||||||
|
continue
|
||||||
|
if decision.get("eligible") is not True:
|
||||||
|
failures.append(f"{slug}:{role}_not_eligible")
|
||||||
|
reasons = decision.get("reasons")
|
||||||
|
if isinstance(reasons, list) and reasons:
|
||||||
|
failures.append(f"{slug}:{role}_has_rejection_reasons")
|
||||||
|
return sorted(set(failures))
|
||||||
|
|
||||||
|
|
||||||
|
def _default_live_dataset_access() -> tuple[Callable[[], Any], type[Any]]:
|
||||||
|
"""Load the database dependencies only for an operational live re-check.
|
||||||
|
|
||||||
|
This module is also imported by pure filesystem tooling and fixture tests.
|
||||||
|
Keeping the import lazy means those paths do not accidentally open a
|
||||||
|
database connection, while a normal release/verify invocation still fails
|
||||||
|
closed if the live registry cannot be checked.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
|
||||||
|
repo_root = Path(__file__).resolve().parents[1]
|
||||||
|
app_root = repo_root / "backend"
|
||||||
|
if str(app_root) not in sys.path:
|
||||||
|
sys.path.insert(0, str(app_root))
|
||||||
|
from app.db.session import SessionLocal
|
||||||
|
from app.models import Dataset
|
||||||
|
|
||||||
|
return SessionLocal, Dataset
|
||||||
|
|
||||||
|
|
||||||
|
def live_manifest_training_eligibility_failures(
|
||||||
|
manifest: Mapping[str, Any],
|
||||||
|
*,
|
||||||
|
fixture_mode: bool = False,
|
||||||
|
session_factory: Callable[[], Any] | None = None,
|
||||||
|
dataset_model: type[Any] | None = None,
|
||||||
|
) -> list[str]:
|
||||||
|
"""Re-evaluate every frozen corpus parent against the live database.
|
||||||
|
|
||||||
|
A frozen manifest proves what was eligible when it was created, not what
|
||||||
|
remains eligible now. Operational releases therefore resolve the exact
|
||||||
|
raster/reference Dataset ids again immediately before seal, retry, resume
|
||||||
|
and CUDA training. A later quarantine, failed contract, snapshot change
|
||||||
|
or missing record is a revocation and cannot be masked by the old manifest.
|
||||||
|
|
||||||
|
Fixture-only corpora deliberately have no operational authority and are
|
||||||
|
never used for a production release; their isolated tests may skip this
|
||||||
|
live database boundary.
|
||||||
|
"""
|
||||||
|
|
||||||
|
if fixture_mode:
|
||||||
|
return []
|
||||||
|
failures: list[str] = []
|
||||||
|
samples = manifest.get("samples")
|
||||||
|
if not isinstance(samples, list) or not samples:
|
||||||
|
return ["live_manifest_samples_missing"]
|
||||||
|
if session_factory is None or dataset_model is None:
|
||||||
|
try:
|
||||||
|
default_factory, default_model = _default_live_dataset_access()
|
||||||
|
except Exception:
|
||||||
|
return ["live_training_dataset_access_unavailable"]
|
||||||
|
session_factory = session_factory or default_factory
|
||||||
|
dataset_model = dataset_model or default_model
|
||||||
|
|
||||||
|
try:
|
||||||
|
db = session_factory()
|
||||||
|
except Exception:
|
||||||
|
return ["live_training_dataset_access_unavailable"]
|
||||||
|
try:
|
||||||
|
for sample in samples:
|
||||||
|
if not isinstance(sample, Mapping):
|
||||||
|
failures.append("live_manifest_sample_invalid")
|
||||||
|
continue
|
||||||
|
slug = str(sample.get("sample_slug") or "<unknown>")
|
||||||
|
recorded_pair = sample.get("training_eligibility")
|
||||||
|
for role, field_name in (("raster", "raster_dataset_id"), ("reference", "reference_dataset_id")):
|
||||||
|
raw_id = sample.get(field_name)
|
||||||
|
if not isinstance(raw_id, str) or not raw_id.strip():
|
||||||
|
failures.append(f"{slug}:{role}_dataset_id_missing_for_live_check")
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
dataset_id = UUID(raw_id)
|
||||||
|
except (TypeError, ValueError, AttributeError):
|
||||||
|
failures.append(f"{slug}:{role}_dataset_id_invalid_for_live_check")
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
dataset = db.get(dataset_model, dataset_id)
|
||||||
|
except Exception:
|
||||||
|
failures.append(f"{slug}:{role}_live_lookup_failed")
|
||||||
|
continue
|
||||||
|
if dataset is None:
|
||||||
|
failures.append(f"{slug}:{role}_live_dataset_missing")
|
||||||
|
continue
|
||||||
|
result = evaluate_dataset_training_eligibility(
|
||||||
|
dataset,
|
||||||
|
role=role, # type: ignore[arg-type]
|
||||||
|
fixture_mode=False,
|
||||||
|
)
|
||||||
|
if not result.eligible:
|
||||||
|
for reason in result.reasons:
|
||||||
|
failures.append(f"{slug}:{role}_live_revoked:{reason}")
|
||||||
|
if isinstance(recorded_pair, Mapping):
|
||||||
|
recorded_role = recorded_pair.get(role)
|
||||||
|
if isinstance(recorded_role, Mapping):
|
||||||
|
recorded_evidence = recorded_role.get("evidence")
|
||||||
|
if isinstance(recorded_evidence, Mapping) and str(recorded_evidence.get("dataset_id") or "") != raw_id:
|
||||||
|
failures.append(f"{slug}:{role}_manifest_dataset_binding_mismatch")
|
||||||
|
finally:
|
||||||
|
close = getattr(db, "close", None)
|
||||||
|
if callable(close):
|
||||||
|
close()
|
||||||
|
return sorted(set(failures))
|
||||||
|
|
||||||
|
|
||||||
|
def assert_manifest_training_eligible(
|
||||||
|
manifest: Mapping[str, Any],
|
||||||
|
*,
|
||||||
|
fixture_mode: bool = False,
|
||||||
|
) -> None:
|
||||||
|
failures = manifest_training_eligibility_failures(manifest, fixture_mode=fixture_mode)
|
||||||
|
if failures:
|
||||||
|
raise TrainingEligibilityError(
|
||||||
|
"Corpus manifest is not eligible for training: " + ", ".join(failures)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _file_sha256(path: Path) -> str:
|
||||||
|
digest = hashlib.sha256()
|
||||||
|
with path.open("rb") as stream:
|
||||||
|
for chunk in iter(lambda: stream.read(1024 * 1024), b""):
|
||||||
|
digest.update(chunk)
|
||||||
|
return digest.hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def frozen_manifest_training_eligibility_failures(
|
||||||
|
manifest_path: Path,
|
||||||
|
*,
|
||||||
|
fixture_mode: bool = False,
|
||||||
|
verify_live: bool = False,
|
||||||
|
session_factory: Callable[[], Any] | None = None,
|
||||||
|
dataset_model: type[Any] | None = None,
|
||||||
|
) -> list[str]:
|
||||||
|
"""Verify an immutable manifest and its source-eligibility decision together."""
|
||||||
|
|
||||||
|
failures: list[str] = []
|
||||||
|
try:
|
||||||
|
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, json.JSONDecodeError):
|
||||||
|
return ["corpus_manifest_unreadable"]
|
||||||
|
if not isinstance(manifest, Mapping):
|
||||||
|
return ["corpus_manifest_invalid"]
|
||||||
|
failures.extend(manifest_training_eligibility_failures(manifest, fixture_mode=fixture_mode))
|
||||||
|
|
||||||
|
freeze_path = manifest_path.parent / "corpus-freeze.json"
|
||||||
|
try:
|
||||||
|
freeze = json.loads(freeze_path.read_text(encoding="utf-8"))
|
||||||
|
except (OSError, json.JSONDecodeError):
|
||||||
|
return sorted(set(failures + ["corpus_freeze_missing_or_invalid"]))
|
||||||
|
if not isinstance(freeze, Mapping):
|
||||||
|
return sorted(set(failures + ["corpus_freeze_missing_or_invalid"]))
|
||||||
|
if freeze.get("immutable") is not True:
|
||||||
|
failures.append("corpus_freeze_not_immutable")
|
||||||
|
if freeze.get("manifest_sha256") != _file_sha256(manifest_path):
|
||||||
|
failures.append("corpus_manifest_checksum_mismatch")
|
||||||
|
if freeze.get("training_eligibility_policy") != TRAINING_ELIGIBILITY_POLICY_VERSION:
|
||||||
|
failures.append("corpus_freeze_policy_invalid")
|
||||||
|
if bool(freeze.get("fixture_mode")) != fixture_mode:
|
||||||
|
failures.append("corpus_freeze_fixture_mode_mismatch")
|
||||||
|
if verify_live:
|
||||||
|
failures.extend(
|
||||||
|
live_manifest_training_eligibility_failures(
|
||||||
|
manifest,
|
||||||
|
fixture_mode=fixture_mode,
|
||||||
|
session_factory=session_factory,
|
||||||
|
dataset_model=dataset_model,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
return sorted(set(failures))
|
||||||
|
|
||||||
|
|
||||||
|
def assert_frozen_manifest_training_eligible(
|
||||||
|
manifest_path: Path,
|
||||||
|
*,
|
||||||
|
fixture_mode: bool = False,
|
||||||
|
verify_live: bool = False,
|
||||||
|
session_factory: Callable[[], Any] | None = None,
|
||||||
|
dataset_model: type[Any] | None = None,
|
||||||
|
) -> dict[str, Any]:
|
||||||
|
"""Load only a frozen, policy-valid corpus manifest for a training entrypoint."""
|
||||||
|
|
||||||
|
failures = frozen_manifest_training_eligibility_failures(
|
||||||
|
manifest_path,
|
||||||
|
fixture_mode=fixture_mode,
|
||||||
|
verify_live=verify_live,
|
||||||
|
session_factory=session_factory,
|
||||||
|
dataset_model=dataset_model,
|
||||||
|
)
|
||||||
|
if failures:
|
||||||
|
raise TrainingEligibilityError(
|
||||||
|
"Corpus manifest is not eligible for training: " + ", ".join(failures)
|
||||||
|
)
|
||||||
|
payload = json.loads(manifest_path.read_text(encoding="utf-8"))
|
||||||
|
assert isinstance(payload, dict)
|
||||||
|
return payload
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,738 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Exercise Phase-2 PostgreSQL migration guards on a disposable database.
|
||||||
|
|
||||||
|
This is intentionally *not* a general migration runner. It refuses every
|
||||||
|
database whose name does not start with ``geointel_phase2_`` so it cannot be
|
||||||
|
pointed accidentally at a developer, staging or production database. The
|
||||||
|
script upgrades the complete Alembic chain, proves the new trigger guards with
|
||||||
|
real DML, and optionally downgrades again.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from typing import Any, Iterator
|
||||||
|
from uuid import UUID, uuid4
|
||||||
|
|
||||||
|
from sqlalchemy import create_engine, text
|
||||||
|
from sqlalchemy.engine import Connection, Engine, make_url
|
||||||
|
from sqlalchemy.exc import IntegrityError
|
||||||
|
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
BACKEND = ROOT / "backend"
|
||||||
|
SAFE_DATABASE_PREFIX = "geointel_phase2_"
|
||||||
|
|
||||||
|
|
||||||
|
def _json_dump(path: Path, value: dict[str, Any]) -> None:
|
||||||
|
path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
path.write_text(
|
||||||
|
json.dumps(value, indent=2, sort_keys=True) + "\n", encoding="utf-8"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _require_disposable_database(database_url: str) -> None:
|
||||||
|
parsed = make_url(database_url)
|
||||||
|
database_name = str(parsed.database or "")
|
||||||
|
if not database_name.startswith(SAFE_DATABASE_PREFIX):
|
||||||
|
raise SystemExit(
|
||||||
|
"Refusing migration guard test: database name must start with "
|
||||||
|
f"{SAFE_DATABASE_PREFIX!r}, received {database_name!r}."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _run_alembic(database_url: str, *arguments: str) -> str:
|
||||||
|
environment = dict(os.environ)
|
||||||
|
environment["DATABASE_URL"] = database_url
|
||||||
|
completed = subprocess.run(
|
||||||
|
[sys.executable, "-m", "alembic", *arguments],
|
||||||
|
cwd=BACKEND,
|
||||||
|
env=environment,
|
||||||
|
text=True,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.STDOUT,
|
||||||
|
check=False,
|
||||||
|
)
|
||||||
|
if completed.returncode != 0:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Alembic {' '.join(arguments)} failed with exit code {completed.returncode}:\n{completed.stdout}"
|
||||||
|
)
|
||||||
|
return completed.stdout
|
||||||
|
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def _transaction(engine: Engine) -> Iterator[Connection]:
|
||||||
|
with engine.begin() as connection:
|
||||||
|
yield connection
|
||||||
|
|
||||||
|
|
||||||
|
def _insert_project(connection: Connection) -> UUID:
|
||||||
|
project_id = uuid4()
|
||||||
|
connection.execute(
|
||||||
|
text("INSERT INTO projects (id, name) VALUES (:id, :name)"),
|
||||||
|
{"id": project_id, "name": "Phase 2 migration guard fixture"},
|
||||||
|
)
|
||||||
|
return project_id
|
||||||
|
|
||||||
|
|
||||||
|
def _source_id(connection: Connection, source_key: str) -> UUID:
|
||||||
|
value = connection.execute(
|
||||||
|
text("SELECT id FROM source_registry WHERE source_key = :source_key"),
|
||||||
|
{"source_key": source_key},
|
||||||
|
).scalar_one()
|
||||||
|
return UUID(str(value))
|
||||||
|
|
||||||
|
|
||||||
|
def _insert_snapshot(
|
||||||
|
connection: Connection, source_registry_id: UUID, *, key: str, checksum: str
|
||||||
|
) -> UUID:
|
||||||
|
snapshot_id = uuid4()
|
||||||
|
connection.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO source_snapshots (
|
||||||
|
id, source_registry_id, snapshot_key, checksum_sha256,
|
||||||
|
crs, units, freshness_status, ingest_status
|
||||||
|
) VALUES (
|
||||||
|
:id, :source_registry_id, :snapshot_key, :checksum_sha256,
|
||||||
|
'EPSG:4326', 'metres', 'current', 'ingested'
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"id": snapshot_id,
|
||||||
|
"source_registry_id": source_registry_id,
|
||||||
|
"snapshot_key": key,
|
||||||
|
"checksum_sha256": checksum,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return snapshot_id
|
||||||
|
|
||||||
|
|
||||||
|
def _accepted_report(
|
||||||
|
*, contract_key: str = "geointel.vector.geojson", contract_version: str = "1.0.0"
|
||||||
|
) -> str:
|
||||||
|
"""Produce a structurally complete persisted validation report fixture."""
|
||||||
|
|
||||||
|
return json.dumps(
|
||||||
|
{
|
||||||
|
"asset_id": "phase2-guard-fixture",
|
||||||
|
"data_contract_key": contract_key,
|
||||||
|
"data_contract_version": contract_version,
|
||||||
|
"contract_fingerprint_sha256": "d" * 64,
|
||||||
|
"validation_status": "passed",
|
||||||
|
"provenance_status": "complete",
|
||||||
|
"lineage_status": "complete",
|
||||||
|
"quarantine_status": "not_quarantined",
|
||||||
|
"validation_scope": ["contract"],
|
||||||
|
"report_sha256": "e" * 64,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _insert_dataset(
|
||||||
|
connection: Connection,
|
||||||
|
*,
|
||||||
|
project_id: UUID,
|
||||||
|
source_registry_id: UUID,
|
||||||
|
source_snapshot_id: UUID,
|
||||||
|
suffix: str,
|
||||||
|
checksum_sha256: str,
|
||||||
|
) -> UUID:
|
||||||
|
dataset_id = uuid4()
|
||||||
|
connection.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO datasets (
|
||||||
|
id, project_id, name, dataset_type, source, source_name,
|
||||||
|
status, source_registry_id, source_snapshot_id,
|
||||||
|
data_contract_key, data_contract_version, validation_report_json,
|
||||||
|
validation_status, provenance_status, lineage_status, quarantine_status,
|
||||||
|
checksum_sha256
|
||||||
|
) VALUES (
|
||||||
|
:id, :project_id, :name, 'vector', 'governed', 'grb',
|
||||||
|
'ready', :source_registry_id, :source_snapshot_id,
|
||||||
|
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
|
||||||
|
'passed', 'complete', 'complete', 'not_quarantined', :checksum_sha256
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"id": dataset_id,
|
||||||
|
"project_id": project_id,
|
||||||
|
"name": f"phase2-{suffix}.geojson",
|
||||||
|
"source_registry_id": source_registry_id,
|
||||||
|
"source_snapshot_id": source_snapshot_id,
|
||||||
|
"validation_report_json": _accepted_report(),
|
||||||
|
"checksum_sha256": checksum_sha256,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
return dataset_id
|
||||||
|
|
||||||
|
|
||||||
|
def _assert_constraint_rejection(
|
||||||
|
engine: Engine, statement: str, parameters: dict[str, Any], *, label: str
|
||||||
|
) -> None:
|
||||||
|
try:
|
||||||
|
with _transaction(engine) as connection:
|
||||||
|
connection.execute(text(statement), parameters)
|
||||||
|
except IntegrityError as exc:
|
||||||
|
sqlstate = getattr(getattr(exc, "orig", None), "sqlstate", None)
|
||||||
|
if sqlstate == "23514":
|
||||||
|
return
|
||||||
|
raise AssertionError(
|
||||||
|
f"{label} failed with unexpected SQLSTATE {sqlstate!r}"
|
||||||
|
) from exc
|
||||||
|
raise AssertionError(f"{label} was accepted unexpectedly")
|
||||||
|
|
||||||
|
|
||||||
|
def verify(database_url: str, *, verify_downgrade: bool) -> dict[str, Any]:
|
||||||
|
_require_disposable_database(database_url)
|
||||||
|
started_at = datetime.now(UTC).isoformat()
|
||||||
|
upgrade_output = _run_alembic(database_url, "upgrade", "head")
|
||||||
|
engine = create_engine(database_url)
|
||||||
|
try:
|
||||||
|
edge_a = uuid4()
|
||||||
|
edge_b = uuid4()
|
||||||
|
with _transaction(engine) as connection:
|
||||||
|
project_id = _insert_project(connection)
|
||||||
|
grb_id = _source_id(connection, "grb")
|
||||||
|
osm_id = _source_id(connection, "osm")
|
||||||
|
grb_snapshot_id = _insert_snapshot(
|
||||||
|
connection, grb_id, key="phase2-grb", checksum="a" * 64
|
||||||
|
)
|
||||||
|
osm_snapshot_id = _insert_snapshot(
|
||||||
|
connection, osm_id, key="phase2-osm", checksum="b" * 64
|
||||||
|
)
|
||||||
|
derived_b_snapshot_id = _insert_snapshot(
|
||||||
|
connection, grb_id, key="phase2-derived-b", checksum="c" * 64
|
||||||
|
)
|
||||||
|
derived_c_snapshot_id = _insert_snapshot(
|
||||||
|
connection, grb_id, key="phase2-derived-c", checksum="d" * 64
|
||||||
|
)
|
||||||
|
mutation_snapshot_id = _insert_snapshot(
|
||||||
|
connection, grb_id, key="phase2-mutation", checksum="e" * 64
|
||||||
|
)
|
||||||
|
dataset_a = _insert_dataset(
|
||||||
|
connection,
|
||||||
|
project_id=project_id,
|
||||||
|
source_registry_id=grb_id,
|
||||||
|
source_snapshot_id=grb_snapshot_id,
|
||||||
|
suffix="a",
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
)
|
||||||
|
dataset_b = _insert_dataset(
|
||||||
|
connection,
|
||||||
|
project_id=project_id,
|
||||||
|
source_registry_id=grb_id,
|
||||||
|
source_snapshot_id=derived_b_snapshot_id,
|
||||||
|
suffix="b",
|
||||||
|
checksum_sha256="c" * 64,
|
||||||
|
)
|
||||||
|
dataset_c = _insert_dataset(
|
||||||
|
connection,
|
||||||
|
project_id=project_id,
|
||||||
|
source_registry_id=grb_id,
|
||||||
|
source_snapshot_id=derived_c_snapshot_id,
|
||||||
|
suffix="c",
|
||||||
|
checksum_sha256="d" * 64,
|
||||||
|
)
|
||||||
|
shared_dataset = _insert_dataset(
|
||||||
|
connection,
|
||||||
|
project_id=project_id,
|
||||||
|
source_registry_id=grb_id,
|
||||||
|
source_snapshot_id=grb_snapshot_id,
|
||||||
|
suffix="shared",
|
||||||
|
checksum_sha256="a" * 64,
|
||||||
|
)
|
||||||
|
mutation_dataset = _insert_dataset(
|
||||||
|
connection,
|
||||||
|
project_id=project_id,
|
||||||
|
source_registry_id=grb_id,
|
||||||
|
source_snapshot_id=mutation_snapshot_id,
|
||||||
|
suffix="mutation",
|
||||||
|
checksum_sha256="e" * 64,
|
||||||
|
)
|
||||||
|
version_id = uuid4()
|
||||||
|
connection.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO dataset_versions (
|
||||||
|
id, dataset_id, version, source_registry_id, source_snapshot_id,
|
||||||
|
data_contract_key, data_contract_version, validation_report_json,
|
||||||
|
validation_status, provenance_status, lineage_status, checksum_sha256
|
||||||
|
) VALUES (
|
||||||
|
:id, :dataset_id, 1, :source_registry_id, :source_snapshot_id,
|
||||||
|
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
|
||||||
|
'passed', 'complete', 'complete', :checksum_sha256
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"id": version_id,
|
||||||
|
"dataset_id": dataset_a,
|
||||||
|
"source_registry_id": grb_id,
|
||||||
|
"source_snapshot_id": grb_snapshot_id,
|
||||||
|
"validation_report_json": _accepted_report(),
|
||||||
|
"checksum_sha256": "a" * 64,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
connection.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO dataset_lineage_edges (
|
||||||
|
id, parent_dataset_id, child_dataset_id, relation_type, transformation_name
|
||||||
|
) VALUES
|
||||||
|
(:edge_a, :dataset_a, :dataset_b, 'derived_from', 'clip'),
|
||||||
|
(:edge_b, :dataset_b, :dataset_c, 'derived_from', 'buffer')
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{
|
||||||
|
"edge_a": edge_a,
|
||||||
|
"edge_b": edge_b,
|
||||||
|
"dataset_a": dataset_a,
|
||||||
|
"dataset_b": dataset_b,
|
||||||
|
"dataset_c": dataset_c,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
_assert_constraint_rejection(
|
||||||
|
engine,
|
||||||
|
"""
|
||||||
|
INSERT INTO datasets (
|
||||||
|
id, project_id, name, dataset_type, source, source_name, status,
|
||||||
|
source_registry_id, source_snapshot_id, validation_status,
|
||||||
|
data_contract_key, data_contract_version, validation_report_json,
|
||||||
|
provenance_status, lineage_status, quarantine_status, checksum_sha256
|
||||||
|
) VALUES (
|
||||||
|
:id, :project_id, 'mismatched.geojson', 'vector', 'governed', 'grb', 'ready',
|
||||||
|
:source_registry_id, :source_snapshot_id, 'passed',
|
||||||
|
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
|
||||||
|
'complete', 'complete', 'not_quarantined', :checksum_sha256
|
||||||
|
)
|
||||||
|
""",
|
||||||
|
{
|
||||||
|
"id": uuid4(),
|
||||||
|
"project_id": project_id,
|
||||||
|
"source_registry_id": grb_id,
|
||||||
|
"source_snapshot_id": osm_snapshot_id,
|
||||||
|
"validation_report_json": _accepted_report(),
|
||||||
|
"checksum_sha256": "b" * 64,
|
||||||
|
},
|
||||||
|
label="snapshot-registry mismatch guard",
|
||||||
|
)
|
||||||
|
_assert_constraint_rejection(
|
||||||
|
engine,
|
||||||
|
"""
|
||||||
|
INSERT INTO datasets (
|
||||||
|
id, project_id, name, dataset_type, source, source_name, status,
|
||||||
|
source_registry_id, source_snapshot_id, validation_status,
|
||||||
|
provenance_status, lineage_status, quarantine_status, checksum_sha256
|
||||||
|
) VALUES (
|
||||||
|
:id, :project_id, 'unreported.geojson', 'vector', 'governed', 'grb', 'ready',
|
||||||
|
:source_registry_id, :source_snapshot_id, 'passed', 'complete', 'complete', 'not_quarantined', :checksum_sha256
|
||||||
|
)
|
||||||
|
""",
|
||||||
|
{
|
||||||
|
"id": uuid4(),
|
||||||
|
"project_id": project_id,
|
||||||
|
"source_registry_id": grb_id,
|
||||||
|
"source_snapshot_id": grb_snapshot_id,
|
||||||
|
"checksum_sha256": "a" * 64,
|
||||||
|
},
|
||||||
|
label="passed contract report guard",
|
||||||
|
)
|
||||||
|
_assert_constraint_rejection(
|
||||||
|
engine,
|
||||||
|
"""
|
||||||
|
INSERT INTO datasets (
|
||||||
|
id, project_id, name, dataset_type, source, source_name, status,
|
||||||
|
source_registry_id, source_snapshot_id, validation_status,
|
||||||
|
data_contract_key, data_contract_version, validation_report_json,
|
||||||
|
provenance_status, lineage_status, quarantine_status
|
||||||
|
) VALUES (
|
||||||
|
:id, :project_id, 'missing-checksum.geojson', 'vector', 'governed', 'grb', 'ready',
|
||||||
|
:source_registry_id, :source_snapshot_id, 'passed',
|
||||||
|
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
|
||||||
|
'complete', 'complete', 'not_quarantined'
|
||||||
|
)
|
||||||
|
""",
|
||||||
|
{
|
||||||
|
"id": uuid4(),
|
||||||
|
"project_id": project_id,
|
||||||
|
"source_registry_id": grb_id,
|
||||||
|
"source_snapshot_id": grb_snapshot_id,
|
||||||
|
"validation_report_json": _accepted_report(),
|
||||||
|
},
|
||||||
|
label="passed dataset checksum required guard",
|
||||||
|
)
|
||||||
|
_assert_constraint_rejection(
|
||||||
|
engine,
|
||||||
|
"""
|
||||||
|
INSERT INTO datasets (
|
||||||
|
id, project_id, name, dataset_type, source, source_name, status,
|
||||||
|
source_registry_id, source_snapshot_id, validation_status,
|
||||||
|
data_contract_key, data_contract_version, validation_report_json,
|
||||||
|
provenance_status, lineage_status, quarantine_status, checksum_sha256
|
||||||
|
) VALUES (
|
||||||
|
:id, :project_id, 'checksum-mismatch.geojson', 'vector', 'governed', 'grb', 'ready',
|
||||||
|
:source_registry_id, :source_snapshot_id, 'passed',
|
||||||
|
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
|
||||||
|
'complete', 'complete', 'not_quarantined', :checksum_sha256
|
||||||
|
)
|
||||||
|
""",
|
||||||
|
{
|
||||||
|
"id": uuid4(),
|
||||||
|
"project_id": project_id,
|
||||||
|
"source_registry_id": grb_id,
|
||||||
|
"source_snapshot_id": grb_snapshot_id,
|
||||||
|
"validation_report_json": _accepted_report(),
|
||||||
|
"checksum_sha256": "f" * 64,
|
||||||
|
},
|
||||||
|
label="passed dataset snapshot checksum binding guard",
|
||||||
|
)
|
||||||
|
_assert_constraint_rejection(
|
||||||
|
engine,
|
||||||
|
"""
|
||||||
|
INSERT INTO dataset_versions (
|
||||||
|
id, dataset_id, version, source_registry_id, source_snapshot_id,
|
||||||
|
data_contract_key, data_contract_version, validation_report_json,
|
||||||
|
validation_status, provenance_status, lineage_status
|
||||||
|
) VALUES (
|
||||||
|
:id, :dataset_id, 2, :source_registry_id, :source_snapshot_id,
|
||||||
|
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
|
||||||
|
'passed', 'complete', 'complete'
|
||||||
|
)
|
||||||
|
""",
|
||||||
|
{
|
||||||
|
"id": uuid4(),
|
||||||
|
"dataset_id": dataset_a,
|
||||||
|
"source_registry_id": grb_id,
|
||||||
|
"source_snapshot_id": grb_snapshot_id,
|
||||||
|
"validation_report_json": _accepted_report(),
|
||||||
|
},
|
||||||
|
label="passed dataset version checksum required guard",
|
||||||
|
)
|
||||||
|
_assert_constraint_rejection(
|
||||||
|
engine,
|
||||||
|
"""
|
||||||
|
INSERT INTO dataset_versions (
|
||||||
|
id, dataset_id, version, source_registry_id, source_snapshot_id,
|
||||||
|
data_contract_key, data_contract_version, validation_report_json,
|
||||||
|
validation_status, provenance_status, lineage_status, checksum_sha256
|
||||||
|
) VALUES (
|
||||||
|
:id, :dataset_id, 2, :source_registry_id, :source_snapshot_id,
|
||||||
|
'geointel.vector.geojson', '1.0.0', CAST(:validation_report_json AS json),
|
||||||
|
'passed', 'complete', 'complete', :checksum_sha256
|
||||||
|
)
|
||||||
|
""",
|
||||||
|
{
|
||||||
|
"id": uuid4(),
|
||||||
|
"dataset_id": dataset_a,
|
||||||
|
"source_registry_id": grb_id,
|
||||||
|
"source_snapshot_id": grb_snapshot_id,
|
||||||
|
"validation_report_json": _accepted_report(),
|
||||||
|
"checksum_sha256": "f" * 64,
|
||||||
|
},
|
||||||
|
label="passed dataset version snapshot checksum binding guard",
|
||||||
|
)
|
||||||
|
_assert_constraint_rejection(
|
||||||
|
engine,
|
||||||
|
"UPDATE datasets SET data_contract_version = '9.9.9' WHERE id = :dataset_id",
|
||||||
|
{"dataset_id": dataset_b},
|
||||||
|
label="accepted contract evidence immutability guard",
|
||||||
|
)
|
||||||
|
_assert_constraint_rejection(
|
||||||
|
engine,
|
||||||
|
"UPDATE dataset_versions SET data_contract_version = '9.9.9' WHERE id = :dataset_version_id",
|
||||||
|
{"dataset_version_id": version_id},
|
||||||
|
label="accepted dataset-version contract evidence immutability guard",
|
||||||
|
)
|
||||||
|
_assert_constraint_rejection(
|
||||||
|
engine,
|
||||||
|
"UPDATE datasets SET storage_path = '/tampered/asset.geojson' WHERE id = :dataset_id",
|
||||||
|
{"dataset_id": mutation_dataset},
|
||||||
|
label="accepted dataset artifact immutability guard",
|
||||||
|
)
|
||||||
|
_assert_constraint_rejection(
|
||||||
|
engine,
|
||||||
|
"UPDATE datasets SET observed_at = CURRENT_TIMESTAMP WHERE id = :dataset_id",
|
||||||
|
{"dataset_id": mutation_dataset},
|
||||||
|
label="accepted dataset temporal evidence immutability guard",
|
||||||
|
)
|
||||||
|
_assert_constraint_rejection(
|
||||||
|
engine,
|
||||||
|
"UPDATE dataset_versions SET storage_path = '/tampered/version.geojson' WHERE id = :dataset_version_id",
|
||||||
|
{"dataset_version_id": version_id},
|
||||||
|
label="accepted dataset-version artifact immutability guard",
|
||||||
|
)
|
||||||
|
_assert_constraint_rejection(
|
||||||
|
engine,
|
||||||
|
"""
|
||||||
|
UPDATE datasets
|
||||||
|
SET validation_status = 'failed', checksum_sha256 = 'f' || repeat('0', 63)
|
||||||
|
WHERE id = :dataset_id
|
||||||
|
""",
|
||||||
|
{"dataset_id": mutation_dataset},
|
||||||
|
label="accepted dataset requires invalidation before artifact replacement",
|
||||||
|
)
|
||||||
|
with _transaction(engine) as connection:
|
||||||
|
connection.execute(
|
||||||
|
text(
|
||||||
|
"UPDATE datasets SET validation_status = 'failed' WHERE id = :dataset_id"
|
||||||
|
),
|
||||||
|
{"dataset_id": mutation_dataset},
|
||||||
|
)
|
||||||
|
connection.execute(
|
||||||
|
text(
|
||||||
|
"UPDATE datasets SET storage_path = '/replacement/asset.geojson' WHERE id = :dataset_id"
|
||||||
|
),
|
||||||
|
{"dataset_id": mutation_dataset},
|
||||||
|
)
|
||||||
|
_assert_constraint_rejection(
|
||||||
|
engine,
|
||||||
|
"UPDATE source_snapshots SET source_registry_id = :registry_id WHERE id = :snapshot_id",
|
||||||
|
{"registry_id": osm_id, "snapshot_id": grb_snapshot_id},
|
||||||
|
label="immutable snapshot registry guard",
|
||||||
|
)
|
||||||
|
_assert_constraint_rejection(
|
||||||
|
engine,
|
||||||
|
"UPDATE source_registry SET display_name = 'tampered' WHERE id = :registry_id",
|
||||||
|
{"registry_id": grb_id},
|
||||||
|
label="server-owned source registry guard",
|
||||||
|
)
|
||||||
|
_assert_constraint_rejection(
|
||||||
|
engine,
|
||||||
|
"UPDATE source_snapshots SET checksum_sha256 = :checksum_sha256 WHERE id = :snapshot_id",
|
||||||
|
{"checksum_sha256": "c" * 64, "snapshot_id": grb_snapshot_id},
|
||||||
|
label="immutable snapshot evidence guard",
|
||||||
|
)
|
||||||
|
_assert_constraint_rejection(
|
||||||
|
engine,
|
||||||
|
"""
|
||||||
|
INSERT INTO dataset_lineage_edges (
|
||||||
|
id, parent_dataset_id, child_dataset_id, relation_type, transformation_name
|
||||||
|
) VALUES (:id, :parent_dataset_id, :child_dataset_id, 'derived_from', 'cycle')
|
||||||
|
""",
|
||||||
|
{
|
||||||
|
"id": uuid4(),
|
||||||
|
"parent_dataset_id": dataset_c,
|
||||||
|
"child_dataset_id": dataset_a,
|
||||||
|
},
|
||||||
|
label="lineage cycle guard",
|
||||||
|
)
|
||||||
|
_assert_constraint_rejection(
|
||||||
|
engine,
|
||||||
|
"UPDATE dataset_lineage_edges SET transformation_name = 'tampered' WHERE id = :id",
|
||||||
|
{"id": edge_a},
|
||||||
|
label="lineage edge immutability update guard",
|
||||||
|
)
|
||||||
|
_assert_constraint_rejection(
|
||||||
|
engine,
|
||||||
|
"DELETE FROM dataset_lineage_edges WHERE id = :id",
|
||||||
|
{"id": edge_b},
|
||||||
|
label="lineage edge immutability delete guard",
|
||||||
|
)
|
||||||
|
with _transaction(engine) as connection:
|
||||||
|
connection.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
INSERT INTO dataset_quarantines (
|
||||||
|
id, dataset_version_id, stage, reason_code, status
|
||||||
|
) VALUES (:id, :dataset_version_id, 'integration_test', 'CHECKSUM_MISMATCH', 'quarantined')
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{"id": uuid4(), "dataset_version_id": version_id},
|
||||||
|
)
|
||||||
|
dataset_state = (
|
||||||
|
connection.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
SELECT status, quarantine_status, validation_status, provenance_status, lineage_status
|
||||||
|
FROM datasets WHERE id = :dataset_id
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{"dataset_id": dataset_a},
|
||||||
|
)
|
||||||
|
.mappings()
|
||||||
|
.one()
|
||||||
|
)
|
||||||
|
version_state = (
|
||||||
|
connection.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
SELECT validation_status, provenance_status, lineage_status
|
||||||
|
FROM dataset_versions WHERE id = :dataset_version_id
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{"dataset_version_id": version_id},
|
||||||
|
)
|
||||||
|
.mappings()
|
||||||
|
.one()
|
||||||
|
)
|
||||||
|
snapshot_state = connection.execute(
|
||||||
|
text(
|
||||||
|
"SELECT ingest_status FROM source_snapshots WHERE id = :snapshot_id"
|
||||||
|
),
|
||||||
|
{"snapshot_id": grb_snapshot_id},
|
||||||
|
).scalar_one()
|
||||||
|
child_dataset_state = (
|
||||||
|
connection.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
SELECT status, quarantine_status, validation_status, provenance_status, lineage_status
|
||||||
|
FROM datasets WHERE id = :dataset_id
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{"dataset_id": dataset_b},
|
||||||
|
)
|
||||||
|
.mappings()
|
||||||
|
.one()
|
||||||
|
)
|
||||||
|
grandchild_dataset_state = (
|
||||||
|
connection.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
SELECT status, quarantine_status, validation_status, provenance_status, lineage_status
|
||||||
|
FROM datasets WHERE id = :dataset_id
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{"dataset_id": dataset_c},
|
||||||
|
)
|
||||||
|
.mappings()
|
||||||
|
.one()
|
||||||
|
)
|
||||||
|
shared_dataset_state = (
|
||||||
|
connection.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
SELECT status, quarantine_status, validation_status, provenance_status, lineage_status
|
||||||
|
FROM datasets WHERE id = :dataset_id
|
||||||
|
"""
|
||||||
|
),
|
||||||
|
{"dataset_id": shared_dataset},
|
||||||
|
)
|
||||||
|
.mappings()
|
||||||
|
.one()
|
||||||
|
)
|
||||||
|
expected_dataset_state = {
|
||||||
|
"status": "quarantined",
|
||||||
|
"quarantine_status": "quarantined",
|
||||||
|
"validation_status": "failed",
|
||||||
|
"provenance_status": "incomplete",
|
||||||
|
"lineage_status": "incomplete",
|
||||||
|
}
|
||||||
|
if dict(dataset_state) != expected_dataset_state:
|
||||||
|
raise AssertionError(
|
||||||
|
f"quarantine parent propagation mismatch: {dict(dataset_state)}"
|
||||||
|
)
|
||||||
|
expected_version_state = {
|
||||||
|
"validation_status": "failed",
|
||||||
|
"provenance_status": "incomplete",
|
||||||
|
"lineage_status": "incomplete",
|
||||||
|
}
|
||||||
|
if dict(version_state) != expected_version_state:
|
||||||
|
raise AssertionError(
|
||||||
|
f"quarantine version propagation mismatch: {dict(version_state)}"
|
||||||
|
)
|
||||||
|
if snapshot_state != "quarantined":
|
||||||
|
raise AssertionError(
|
||||||
|
f"quarantine snapshot propagation mismatch: {snapshot_state}"
|
||||||
|
)
|
||||||
|
if dict(child_dataset_state) != expected_dataset_state:
|
||||||
|
raise AssertionError(
|
||||||
|
"quarantine transitive-child propagation mismatch: "
|
||||||
|
f"{dict(child_dataset_state)}"
|
||||||
|
)
|
||||||
|
if dict(grandchild_dataset_state) != expected_dataset_state:
|
||||||
|
raise AssertionError(
|
||||||
|
"quarantine transitive-grandchild propagation mismatch: "
|
||||||
|
f"{dict(grandchild_dataset_state)}"
|
||||||
|
)
|
||||||
|
if dict(shared_dataset_state) != expected_dataset_state:
|
||||||
|
raise AssertionError(
|
||||||
|
"quarantine shared-snapshot propagation mismatch: "
|
||||||
|
f"{dict(shared_dataset_state)}"
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
engine.dispose()
|
||||||
|
|
||||||
|
downgrade_output = (
|
||||||
|
_run_alembic(database_url, "downgrade", "base")
|
||||||
|
if verify_downgrade
|
||||||
|
else "not requested"
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"schema_version": 1,
|
||||||
|
"phase": "P2",
|
||||||
|
"started_at": started_at,
|
||||||
|
"completed_at": datetime.now(UTC).isoformat(),
|
||||||
|
"migration_revision": "202608010001",
|
||||||
|
"database_name": str(make_url(database_url).database),
|
||||||
|
"result": "passed",
|
||||||
|
"guards": {
|
||||||
|
"snapshot_registry_pairing": "passed",
|
||||||
|
"snapshot_registry_immutable": "passed",
|
||||||
|
"source_registry_immutable": "passed",
|
||||||
|
"snapshot_evidence_immutable": "passed",
|
||||||
|
"lineage_cycle": "passed",
|
||||||
|
"lineage_edge_immutable": "passed",
|
||||||
|
"version_quarantine_propagation": "passed",
|
||||||
|
"snapshot_quarantine_fanout": "passed",
|
||||||
|
"transitive_lineage_quarantine": "passed",
|
||||||
|
"passed_contract_report_required": "passed",
|
||||||
|
"passed_dataset_checksum_required_and_snapshot_bound": "passed",
|
||||||
|
"passed_dataset_version_checksum_required_and_snapshot_bound": "passed",
|
||||||
|
"accepted_contract_evidence_immutable": "passed",
|
||||||
|
"accepted_dataset_version_contract_evidence_immutable": "passed",
|
||||||
|
"accepted_artifact_and_temporal_evidence_immutable": "passed",
|
||||||
|
},
|
||||||
|
"upgrade_output_tail": upgrade_output.splitlines()[-8:],
|
||||||
|
"downgrade_output_tail": downgrade_output.splitlines()[-8:],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
parser = argparse.ArgumentParser()
|
||||||
|
parser.add_argument("--database-url", required=True)
|
||||||
|
parser.add_argument(
|
||||||
|
"--output",
|
||||||
|
type=Path,
|
||||||
|
default=ROOT
|
||||||
|
/ "artifacts"
|
||||||
|
/ "evidence"
|
||||||
|
/ "accuracy"
|
||||||
|
/ "P2"
|
||||||
|
/ "postgres-migration-guards.json",
|
||||||
|
)
|
||||||
|
parser.add_argument("--skip-downgrade", action="store_true")
|
||||||
|
args = parser.parse_args()
|
||||||
|
try:
|
||||||
|
result = verify(args.database_url, verify_downgrade=not args.skip_downgrade)
|
||||||
|
except Exception as exc:
|
||||||
|
result = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"phase": "P2",
|
||||||
|
"completed_at": datetime.now(UTC).isoformat(),
|
||||||
|
"migration_revision": "202608010001",
|
||||||
|
"result": "failed",
|
||||||
|
"error_type": type(exc).__name__,
|
||||||
|
"error": str(exc),
|
||||||
|
}
|
||||||
|
_json_dump(args.output, result)
|
||||||
|
print(json.dumps(result, indent=2))
|
||||||
|
return 1
|
||||||
|
_json_dump(args.output, result)
|
||||||
|
print(json.dumps(result, indent=2))
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
raise SystemExit(main())
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"class_id": 2,
|
||||||
|
"x_center": 0.95,
|
||||||
|
"y_center": 0.5,
|
||||||
|
"width": 0.2,
|
||||||
|
"height": -0.1
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
[
|
||||||
|
{
|
||||||
|
"class_id": 0,
|
||||||
|
"x_center": 0.5,
|
||||||
|
"y_center": 0.5,
|
||||||
|
"width": 0.2,
|
||||||
|
"height": 0.3
|
||||||
|
}
|
||||||
|
]
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
{
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"properties": {
|
||||||
|
"native_id": "GRB-GBG-001",
|
||||||
|
"feature_status": "active"
|
||||||
|
},
|
||||||
|
"geometry": {
|
||||||
|
"type": "Polygon",
|
||||||
|
"coordinates": [
|
||||||
|
[
|
||||||
|
[4.8470, 51.1550],
|
||||||
|
[4.8480, 51.1550],
|
||||||
|
[4.8480, 51.1560],
|
||||||
|
[4.8470, 51.1560],
|
||||||
|
[4.8470, 51.1550]
|
||||||
|
]
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
{
|
||||||
|
"type": "FeatureCollection",
|
||||||
|
"features": [
|
||||||
|
{
|
||||||
|
"type": "Feature",
|
||||||
|
"properties": {
|
||||||
|
"native_id": "GRB-GBG-002",
|
||||||
|
"feature_status": "active"
|
||||||
|
},
|
||||||
|
"geometry": {
|
||||||
|
"type": "Polygon",
|
||||||
|
"coordinates": [
|
||||||
|
[
|
||||||
|
[193277.5, 205708.3],
|
||||||
|
[193377.5, 205708.3],
|
||||||
|
[193377.5, 205808.3],
|
||||||
|
[193277.5, 205808.3],
|
||||||
|
[193277.5, 205708.3]
|
||||||
|
]
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -1,9 +1,14 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
import importlib.util
|
import importlib.util
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import types
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
from PIL import Image
|
||||||
|
|
||||||
|
|
||||||
ROOT = Path(__file__).resolve().parents[1]
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
@@ -22,6 +27,87 @@ miner = load("build_building_proposal_classifier_dataset")
|
|||||||
trainer = load("train_building_proposal_classifier")
|
trainer = load("train_building_proposal_classifier")
|
||||||
|
|
||||||
|
|
||||||
|
def _sha256(path: Path) -> str:
|
||||||
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def _write_json(path: Path, payload: dict) -> None:
|
||||||
|
path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8")
|
||||||
|
|
||||||
|
|
||||||
|
def _governed_proposal_crop_fixture(tmp_path: Path) -> tuple[Path, Path]:
|
||||||
|
"""Create a complete tiny crop release without importing Torch or YOLO."""
|
||||||
|
|
||||||
|
corpus_manifest = tmp_path / "corpus-manifest.json"
|
||||||
|
_write_json(corpus_manifest, {"samples": []})
|
||||||
|
corpus_sha256 = _sha256(corpus_manifest)
|
||||||
|
corpus_freeze = tmp_path / "corpus-freeze.json"
|
||||||
|
_write_json(corpus_freeze, {"immutable": True})
|
||||||
|
summary = tmp_path / "source-summary.json"
|
||||||
|
_write_json(summary, {"source_manifest_sha256": corpus_sha256, "tiles": []})
|
||||||
|
|
||||||
|
dataset_dir = tmp_path / "proposal-crops"
|
||||||
|
entries: list[dict] = []
|
||||||
|
for split in ("train", "val"):
|
||||||
|
for label, colour in (("negative", (0, 0, 0)), ("positive", (255, 255, 255))):
|
||||||
|
crop_path = dataset_dir / split / label / f"{split}-{label}.jpg"
|
||||||
|
crop_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
Image.new("RGB", (2, 2), colour).save(crop_path)
|
||||||
|
relative_path = crop_path.relative_to(dataset_dir).as_posix()
|
||||||
|
entries.append(
|
||||||
|
{
|
||||||
|
"relative_path": relative_path,
|
||||||
|
"sha256": _sha256(crop_path),
|
||||||
|
"size_bytes": crop_path.stat().st_size,
|
||||||
|
"split": split,
|
||||||
|
"label": label,
|
||||||
|
"sample_slug": f"{split}-{label}",
|
||||||
|
"proposal_index": 0,
|
||||||
|
"proposal_score": 0.8,
|
||||||
|
"source_box_xyxy": [0.0, 0.0, 1.0, 1.0],
|
||||||
|
"source_image_path": str(tmp_path / "source-image.tif"),
|
||||||
|
"source_image_sha256": "a" * 64,
|
||||||
|
"source_label_path": str(tmp_path / "source-label.txt"),
|
||||||
|
"source_label_sha256": "b" * 64,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
entries.sort(key=lambda item: item["relative_path"])
|
||||||
|
counts = {f"{split}/{label}": 1 for split in ("train", "val") for label in ("negative", "positive")}
|
||||||
|
payload: dict = {
|
||||||
|
"schema_version": 1,
|
||||||
|
"status": "ok",
|
||||||
|
"immutable": True,
|
||||||
|
"dataset_kind": "building_proposal_classifier_crops",
|
||||||
|
"fixture_mode": False,
|
||||||
|
"governed_corpus_live_recheck": True,
|
||||||
|
"source": {
|
||||||
|
"corpus_manifest": {"path": str(corpus_manifest), "sha256": corpus_sha256},
|
||||||
|
"corpus_freeze": {"path": str(corpus_freeze), "sha256": _sha256(corpus_freeze)},
|
||||||
|
"summary": {
|
||||||
|
"path": str(summary),
|
||||||
|
"sha256": _sha256(summary),
|
||||||
|
"source_manifest_sha256": corpus_sha256,
|
||||||
|
},
|
||||||
|
"training_release": {
|
||||||
|
"dataset_yaml_path": "fixture-dataset.yaml",
|
||||||
|
"dataset_yaml_sha256": "d" * 64,
|
||||||
|
"corpus_manifest_sha256": corpus_sha256,
|
||||||
|
},
|
||||||
|
"proposal_model": {"path": str(tmp_path / "proposal.pt"), "sha256": "c" * 64},
|
||||||
|
},
|
||||||
|
"parameters": {"crop_scale": 1.4},
|
||||||
|
"counts": counts,
|
||||||
|
"sample_counts": {item["sample_slug"]: 1 for item in entries},
|
||||||
|
"tile_count": 2,
|
||||||
|
"crop_count": len(entries),
|
||||||
|
"crops_sha256": hashlib.sha256(trainer._canonical_json_bytes({"crops": entries})).hexdigest(),
|
||||||
|
"crops": entries,
|
||||||
|
}
|
||||||
|
payload["manifest_sha256"] = trainer._payload_sha256(payload)
|
||||||
|
_write_json(dataset_dir / trainer.PROPOSAL_DATASET_PROVENANCE_NAME, payload)
|
||||||
|
return dataset_dir, corpus_manifest
|
||||||
|
|
||||||
|
|
||||||
def test_classify_proposals_consumes_reference_once() -> None:
|
def test_classify_proposals_consumes_reference_once() -> None:
|
||||||
reference = [(0.0, 0.0, 10.0, 10.0)]
|
reference = [(0.0, 0.0, 10.0, 10.0)]
|
||||||
proposals = [((0.0, 0.0, 10.0, 10.0), 0.9), ((0.0, 0.0, 10.0, 10.0), 0.8)]
|
proposals = [((0.0, 0.0, 10.0, 10.0), 0.9), ((0.0, 0.0, 10.0, 10.0), 0.8)]
|
||||||
@@ -38,3 +124,166 @@ def test_eligible_tiles_rejects_protected_manifest_split() -> None:
|
|||||||
def test_binary_metrics() -> None:
|
def test_binary_metrics() -> None:
|
||||||
result = trainer.binary_metrics([0.9, 0.8, 0.2, 0.1], [1, 0, 1, 0])
|
result = trainer.binary_metrics([0.9, 0.8, 0.2, 0.1], [1, 0, 1, 0])
|
||||||
assert result == {"tp": 1, "fp": 1, "fn": 1, "precision": 0.5, "recall": 0.5, "f1": 0.5}
|
assert result == {"tp": 1, "fp": 1, "fn": 1, "precision": 0.5, "recall": 0.5, "f1": 0.5}
|
||||||
|
|
||||||
|
|
||||||
|
def test_builder_rechecks_frozen_corpus_against_live_governed_state(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
observed: dict[str, object] = {}
|
||||||
|
|
||||||
|
def fake_assert(path: Path, **kwargs: object) -> dict:
|
||||||
|
observed["path"] = path
|
||||||
|
observed.update(kwargs)
|
||||||
|
return {"samples": []}
|
||||||
|
|
||||||
|
monkeypatch.setattr(miner, "assert_frozen_manifest_training_eligible", fake_assert)
|
||||||
|
manifest_path = tmp_path / "corpus.json"
|
||||||
|
manifest_path.write_text("{}", encoding="utf-8")
|
||||||
|
assert miner.load_governed_corpus_manifest(manifest_path, fixture_mode=False) == {"samples": []}
|
||||||
|
assert observed == {"path": manifest_path, "fixture_mode": False, "verify_live": True}
|
||||||
|
|
||||||
|
|
||||||
|
def test_builder_rejects_summary_not_bound_to_exact_corpus_manifest(tmp_path: Path) -> None:
|
||||||
|
manifest_path = tmp_path / "corpus.json"
|
||||||
|
manifest_path.write_text('{"samples": []}', encoding="utf-8")
|
||||||
|
with pytest.raises(ValueError, match="not bound"):
|
||||||
|
miner.assert_summary_source_manifest_binding({"source_manifest_sha256": "0" * 64}, manifest_path)
|
||||||
|
|
||||||
|
|
||||||
|
def test_builder_emits_immutable_checksum_bound_crop_provenance(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
class FakeTensor:
|
||||||
|
def __init__(self, values: list[list[float]] | list[float]) -> None:
|
||||||
|
self._values = values
|
||||||
|
|
||||||
|
def cpu(self) -> "FakeTensor":
|
||||||
|
return self
|
||||||
|
|
||||||
|
def tolist(self) -> list[list[float]] | list[float]:
|
||||||
|
return self._values
|
||||||
|
|
||||||
|
class FakeBoxes:
|
||||||
|
xyxy = FakeTensor([[40.0, 40.0, 60.0, 60.0], [0.0, 0.0, 10.0, 10.0]])
|
||||||
|
conf = FakeTensor([0.9, 0.8])
|
||||||
|
|
||||||
|
class FakeResult:
|
||||||
|
boxes = FakeBoxes()
|
||||||
|
|
||||||
|
class FakeYOLO:
|
||||||
|
def __init__(self, model: str) -> None:
|
||||||
|
self.model = model
|
||||||
|
|
||||||
|
def predict(self, sources: list[str], **_kwargs: object) -> list[FakeResult]:
|
||||||
|
return [FakeResult() for _source in sources]
|
||||||
|
|
||||||
|
corpus_manifest = tmp_path / "corpus-manifest.json"
|
||||||
|
manifest = {
|
||||||
|
"samples": [
|
||||||
|
{"sample_slug": "train-a", "split": "train", "region": "flanders"},
|
||||||
|
{"sample_slug": "val-b", "split": "val", "region": "flanders"},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
_write_json(corpus_manifest, manifest)
|
||||||
|
_write_json(tmp_path / "corpus-freeze.json", {"immutable": True})
|
||||||
|
tiles: list[dict[str, object]] = []
|
||||||
|
for sample_slug, split in (("train-a", "train"), ("val-b", "val")):
|
||||||
|
image_path = tmp_path / f"{sample_slug}.png"
|
||||||
|
Image.new("RGB", (100, 100), (50, 100, 150)).save(image_path)
|
||||||
|
label_path = tmp_path / f"{sample_slug}.txt"
|
||||||
|
label_path.write_text("0 0.5 0.5 0.2 0.2\n", encoding="utf-8")
|
||||||
|
tiles.append(
|
||||||
|
{
|
||||||
|
"sample_slug": sample_slug,
|
||||||
|
"split": split,
|
||||||
|
"kept": True,
|
||||||
|
"image_path": str(image_path),
|
||||||
|
"label_path": str(label_path),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
summary_path = tmp_path / "source-summary.json"
|
||||||
|
_write_json(
|
||||||
|
summary_path,
|
||||||
|
{
|
||||||
|
"source_manifest_sha256": _sha256(corpus_manifest),
|
||||||
|
"training_release_manifest": "fixture-training-release.json",
|
||||||
|
"training_release_manifest_sha256": "a" * 64,
|
||||||
|
"training_asset_manifest": "fixture-training-assets.json",
|
||||||
|
"tiles": tiles,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
model_path = tmp_path / "proposal-model.pt"
|
||||||
|
model_path.write_bytes(b"proposal-model")
|
||||||
|
output_dir = tmp_path / "proposal-crops"
|
||||||
|
|
||||||
|
monkeypatch.setattr(miner, "load_governed_corpus_manifest", lambda *_args, **_kwargs: manifest)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
miner,
|
||||||
|
"assert_yolo_summary_bound_to_embedded_training_release",
|
||||||
|
lambda **_kwargs: {
|
||||||
|
"dataset_yaml": {"path": "fixture-dataset.yaml", "sha256": "d" * 64},
|
||||||
|
"corpus": {"manifest_sha256": _sha256(corpus_manifest)},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
monkeypatch.setitem(sys.modules, "ultralytics", types.SimpleNamespace(YOLO=FakeYOLO))
|
||||||
|
monkeypatch.setattr(
|
||||||
|
miner.sys,
|
||||||
|
"argv",
|
||||||
|
[
|
||||||
|
"build_building_proposal_classifier_dataset.py",
|
||||||
|
"--model",
|
||||||
|
str(model_path),
|
||||||
|
"--summary",
|
||||||
|
str(summary_path),
|
||||||
|
"--corpus-manifest",
|
||||||
|
str(corpus_manifest),
|
||||||
|
"--output-dir",
|
||||||
|
str(output_dir),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert miner.main() == 0
|
||||||
|
provenance_path = output_dir / miner.PROPOSAL_DATASET_PROVENANCE_NAME
|
||||||
|
provenance = json.loads(provenance_path.read_text(encoding="utf-8"))
|
||||||
|
assert provenance["immutable"] is True
|
||||||
|
assert provenance["crop_count"] == 4
|
||||||
|
assert provenance["manifest_sha256"] == miner._immutable_payload_sha256(provenance)
|
||||||
|
assert all(_sha256(output_dir / item["relative_path"]) == item["sha256"] for item in provenance["crops"])
|
||||||
|
with pytest.raises(RuntimeError, match="immutable provenance"):
|
||||||
|
miner._write_immutable_json(provenance_path, {"different": True})
|
||||||
|
|
||||||
|
|
||||||
|
def test_trainer_validates_every_crop_and_source_binding_before_torch(
|
||||||
|
monkeypatch: pytest.MonkeyPatch,
|
||||||
|
tmp_path: Path,
|
||||||
|
) -> None:
|
||||||
|
dataset_dir, corpus_manifest = _governed_proposal_crop_fixture(tmp_path)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
trainer,
|
||||||
|
"assert_yolo_summary_bound_to_embedded_training_release",
|
||||||
|
lambda **_kwargs: {
|
||||||
|
"dataset_yaml": {"path": "fixture-dataset.yaml", "sha256": "d" * 64},
|
||||||
|
"corpus": {"manifest_sha256": _sha256(corpus_manifest)},
|
||||||
|
},
|
||||||
|
)
|
||||||
|
payload = trainer.validate_proposal_dataset_provenance(dataset_dir, corpus_manifest, fixture_mode=False)
|
||||||
|
assert payload["crop_count"] == 4
|
||||||
|
|
||||||
|
crop = dataset_dir / "train" / "positive" / "train-positive.jpg"
|
||||||
|
crop.write_bytes(b"tampered")
|
||||||
|
with pytest.raises(trainer.ProposalDatasetProvenanceError, match="checksum"):
|
||||||
|
trainer.validate_proposal_dataset_provenance(dataset_dir, corpus_manifest, fixture_mode=False)
|
||||||
|
|
||||||
|
|
||||||
|
def test_trainer_rechecks_live_governed_corpus_before_pytorch(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||||
|
observed: dict[str, object] = {}
|
||||||
|
|
||||||
|
def fake_assert(path: Path, **kwargs: object) -> dict:
|
||||||
|
observed["path"] = path
|
||||||
|
observed.update(kwargs)
|
||||||
|
return {"samples": []}
|
||||||
|
|
||||||
|
monkeypatch.setattr(trainer, "assert_frozen_manifest_training_eligible", fake_assert)
|
||||||
|
manifest_path = tmp_path / "corpus.json"
|
||||||
|
manifest_path.write_text("{}", encoding="utf-8")
|
||||||
|
assert trainer.load_governed_corpus_manifest(manifest_path, fixture_mode=False) == {"samples": []}
|
||||||
|
assert observed == {"path": manifest_path, "fixture_mode": False, "verify_live": True}
|
||||||
|
|||||||
Reference in New Issue
Block a user