Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(provenance): govern source snapshots and data inputs

Browse Source
This commit is contained in:
Jens
2026-08-01 23:46:17 +02:00
parent cebeb5f3b4
commit 5b3c17b494
96 changed files with 20156 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_segmentation_configured_models.py
+214 -2
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
from hashlib import sha256
import json
from pathlib import Path
from uuid import uuid4
@@ -8,9 +9,10 @@ import pytest
from geoalchemy2.shape import to_shape
from app.core.config import Settings
from app.models import Dataset, Project, Segmentation
from app.models import Dataset, Project, Segmentation, SourceRegistry, SourceSnapshot
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
from app.services.model_registry_service import ModelRegistryService
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
from app.services.segmentation_service import SegmentationService
ROOT = Path(__file__).resolve().parents[2]
@@ -80,18 +82,58 @@ class MissingDependencySegAdapter(AvailableSegAdapter):
return False
class NeverLoadSegAdapter(AvailableSegAdapter):
load_calls = 0
def load_model(self, model_path: Path):
type(self).load_calls += 1
raise AssertionError("unmanifested weights must not reach adapter.load_model")
def _project_and_dataset(dataset_type: str = "raster"):
project_id = uuid4()
dataset_id = uuid4()
source_id = uuid4()
snapshot_id = uuid4()
checksum = "a" * 64
project = Project(id=project_id, name="Mol")
source = SourceRegistry(
id=source_id,
source_key="digitaal_vlaanderen_orthophoto",
display_name="Governed orthophoto test source",
classification="contextual",
authority_name="Digitaal Vlaanderen",
authority_scope_json={"zone": "Flanders", "role": "imagery"},
)
snapshot = SourceSnapshot(
id=snapshot_id,
source_registry_id=source_id,
snapshot_key="configured-segmentation-orthophoto",
checksum_sha256=checksum,
ingest_status="ingested",
freshness_status="current",
)
dataset = Dataset(
id=dataset_id,
project_id=project_id,
name="ortho.tif",
dataset_type=dataset_type,
source="user_upload",
source="digitaal_vlaanderen_orthophoto",
source_name="digitaal_vlaanderen_orthophoto",
storage_path="storage/uploads/ortho.tif",
checksum_sha256=checksum,
source_registry_id=source_id,
source_snapshot_id=snapshot_id,
data_contract_key="geointel.raster.geotiff",
data_contract_version="1.0.0",
validation_status="passed",
provenance_status="complete",
lineage_status="not_applicable",
quarantine_status="not_quarantined",
status="ready",
)
dataset.source_registry = source
dataset.source_snapshot = snapshot
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
return db, project_id, dataset_id
@@ -108,6 +150,102 @@ def _settings(tmp_path: Path, **overrides) -> Settings:
return Settings(**values)
def _write_model_sidecar(
model_path: Path,
*,
model_id: str,
framework: str,
source_version: str | None,
db: FakeSession | None = None,
) -> None:
"""Create explicit local test evidence; no production code creates sidecars."""
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
source_registry_id = uuid4()
source_snapshot_id = uuid4()
resolved_source_version = source_version or "test-v1"
if db is not None:
source_registry = SourceRegistry(
id=source_registry_id,
source_key="model",
display_name="Governed test model artifact",
classification="experimental",
authority_name="GeoIntel test fixture",
freshness_status="current",
ingest_status="configured",
)
source_snapshot = SourceSnapshot(
id=source_snapshot_id,
source_registry_id=source_registry_id,
snapshot_key=f"model-{model_id}-{resolved_source_version}",
source_version=resolved_source_version,
checksum_sha256=model_sha256,
freshness_status="current",
ingest_status="ingested",
)
db.objects[(SourceRegistry, source_registry_id)] = source_registry
db.objects[(SourceSnapshot, source_snapshot_id)] = source_snapshot
payload = {
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
"data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"},
"model": {
"model_id": model_id,
"task_type": "segmentation",
"sha256": model_sha256,
"model_format": "pytorch",
"framework": framework,
"class_mapping": {"0": "segment"},
"source_version": resolved_source_version,
},
"source": {
"source_registry_id": str(source_registry_id),
"source_snapshot_id": str(source_snapshot_id),
"source_registry_key": "model",
"source_snapshot_checksum_sha256": model_sha256,
},
"lineage": {
"upstream_asset_ids": ["test-training-corpus"],
"upstream_checksums_sha256": ["a" * 64],
"transformations": [
{"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64}
],
},
"metadata": {"training_manifest_sha256": "c" * 64},
"imported_at": "2026-08-01T10:00:00+00:00",
}
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text(
json.dumps(payload, sort_keys=True),
encoding="utf-8",
)
def _write_configured_model_sidecars(
tmp_path: Path,
settings: Settings,
*,
include_yolo: bool = True,
include_sam: bool = True,
db: FakeSession | None = None,
) -> None:
if include_yolo:
_write_model_sidecar(
tmp_path / "seg.pt",
model_id=settings.yolo_seg_model_id,
framework="ultralytics/pytorch",
source_version=settings.yolo_seg_model_version,
db=db,
)
if include_sam:
_write_model_sidecar(
tmp_path / "sam.pt",
model_id=settings.sam_model_id,
framework="ultralytics/sam",
source_version=settings.sam_model_version,
db=db,
)
def _manifest(tmp_path: Path, tile_count: int = 1) -> Path:
tiles = []
for index in range(tile_count):
@@ -172,10 +310,31 @@ def test_segmentation_models_report_dependency_unavailable(tmp_path: Path) -> No
assert models["sam-configured"].status == "dependency_unavailable"
def test_segmentation_models_require_runtime_provenance_sidecars(tmp_path: Path) -> None:
(tmp_path / "seg.pt").write_bytes(b"unmanifested yolo segmentation weights")
(tmp_path / "sam.pt").write_bytes(b"unmanifested sam weights")
settings = _settings(tmp_path)
models = {
model.model_id: model
for model in ModelRegistryService.list_segmentation_model_capabilities(
settings=settings,
yolo_seg_adapter_class=AvailableSegAdapter,
sam_adapter_class=ClassAgnosticSamAdapter,
)
}
assert models["yolo-seg-configured"].configured is False
assert models["yolo-seg-configured"].status == "contract_incomplete"
assert models["sam-configured"].configured is False
assert models["sam-configured"].status == "contract_incomplete"
def test_segmentation_models_report_configured_with_local_weights(tmp_path: Path) -> None:
(tmp_path / "seg.pt").write_bytes(b"weights")
(tmp_path / "sam.pt").write_bytes(b"weights")
settings = _settings(tmp_path)
_write_configured_model_sidecars(tmp_path, settings)
models = {
model.model_id: model
@@ -204,6 +363,7 @@ def test_configured_segmentation_requires_tile_manifest(tmp_path: Path) -> None:
(tmp_path / "seg.pt").write_bytes(b"weights")
db, project_id, dataset_id = _project_and_dataset()
settings = _settings(tmp_path)
_write_configured_model_sidecars(tmp_path, settings, include_sam=False, db=db)
with pytest.raises(Exception) as exc_info:
SegmentationService.run_segmentation(
@@ -220,10 +380,60 @@ def test_configured_segmentation_requires_tile_manifest(tmp_path: Path) -> None:
assert getattr(exc_info.value, "code", None) == "SEGMENTATION_TILE_MANIFEST_REQUIRED"
def test_configured_segmentation_fails_before_adapter_load_without_sidecar(tmp_path: Path) -> None:
(tmp_path / "seg.pt").write_bytes(b"unmanifested weights")
db, project_id, dataset_id = _project_and_dataset()
settings = _settings(tmp_path)
NeverLoadSegAdapter.load_calls = 0
response = SegmentationService.run_segmentation(
db=db,
project_id=project_id,
dataset_id=dataset_id,
model_id="yolo-seg-configured",
confidence_threshold=0.5,
tile_manifest_path=str(_manifest(tmp_path)),
settings=settings,
yolo_seg_adapter_class=NeverLoadSegAdapter,
sam_adapter_class=ClassAgnosticSamAdapter,
)
assert response.status == "failed"
assert response.error_code == "SEGMENTATION_MODEL_UNAVAILABLE"
assert NeverLoadSegAdapter.load_calls == 0
def test_configured_segmentation_rejects_unbound_model_snapshot_before_adapter_load(tmp_path: Path) -> None:
(tmp_path / "seg.pt").write_bytes(b"structurally valid but unbound weights")
db, project_id, dataset_id = _project_and_dataset()
settings = _settings(tmp_path)
# The sidecar passes catalog validation but its source registry/snapshot
# was never registered in this production-session fixture.
_write_configured_model_sidecars(tmp_path, settings, include_sam=False)
NeverLoadSegAdapter.load_calls = 0
response = SegmentationService.run_segmentation(
db=db,
project_id=project_id,
dataset_id=dataset_id,
model_id="yolo-seg-configured",
confidence_threshold=0.5,
tile_manifest_path=str(_manifest(tmp_path)),
settings=settings,
yolo_seg_adapter_class=NeverLoadSegAdapter,
sam_adapter_class=ClassAgnosticSamAdapter,
)
assert response.status == "failed"
assert response.error_code == "MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND"
assert NeverLoadSegAdapter.load_calls == 0
def test_configured_yolo_seg_run_persists_georeferenced_masks(tmp_path: Path) -> None:
(tmp_path / "seg.pt").write_bytes(b"weights")
db, project_id, dataset_id = _project_and_dataset()
settings = _settings(tmp_path)
_write_configured_model_sidecars(tmp_path, settings, include_sam=False, db=db)
manifest_path = _manifest(tmp_path)
response = SegmentationService.run_segmentation(
@@ -255,12 +465,14 @@ def test_configured_yolo_seg_run_persists_georeferenced_masks(tmp_path: Path) ->
assert segmentation.area_m2 is not None and segmentation.area_m2 > 0
assert segmentation.provenance_json["inference"] == "local"
assert segmentation.provenance_json["model_id"] == "yolo-seg-configured"
assert segmentation.provenance_json["runtime_model_provenance"]["data_contract_key"] == "geointel.model.pytorch"
def test_configured_sam_run_is_class_agnostic(tmp_path: Path) -> None:
(tmp_path / "sam.pt").write_bytes(b"weights")
db, project_id, dataset_id = _project_and_dataset()
settings = _settings(tmp_path)
_write_configured_model_sidecars(tmp_path, settings, include_yolo=False, db=db)
manifest_path = _manifest(tmp_path)
response = SegmentationService.run_segmentation(