Jens/geointel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(provenance): govern source snapshots and data inputs

Browse Source
This commit is contained in:
Jens
2026-08-01 23:46:17 +02:00
parent cebeb5f3b4
commit 5b3c17b494
96 changed files with 20156 additions and 351 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/tests/test_sprint8b_yolo_foundation.py
+191 -3
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
from hashlib import sha256
import json
from pathlib import Path
import sys
@@ -10,10 +11,11 @@ import pytest
from app.core.config import Settings
from app.core.errors import AppError
from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project
from app.models import AnalysisRun, Area, Dataset, Detection, Job, Project, SourceRegistry, SourceSnapshot
from app.services.detection_georeferencing import pixel_bbox_to_epsg4326_polygon
from app.services.detection_service import DetectionService
from app.services.model_registry_service import ModelRegistryService
from app.services.runtime_model_provenance_service import RuntimeModelProvenanceService
from app.services.yolo_adapter import YoloDetectionAdapter
ROOT = Path(__file__).resolve().parents[2]
@@ -79,6 +81,14 @@ class MockYoloAdapter:
]
class NeverLoadUnboundModelAdapter(MockYoloAdapter):
load_calls = 0
def load_model(self, model_path: Path):
type(self).load_calls += 1
raise AssertionError("unbound model provenance must be rejected before adapter.load_model")
class MixedCaseYoloAdapter(MockYoloAdapter):
def predict_tile(self, model, tile_path: Path, confidence_threshold: float) -> list[dict]:
return [
@@ -141,15 +151,47 @@ class ExplodingPredictModel:
def _project_and_dataset(dataset_type: str = "raster"):
project_id = uuid4()
dataset_id = uuid4()
source_registry_id = uuid4()
source_snapshot_id = uuid4()
checksum = "a" * 64
project = Project(id=project_id, name="Geel")
source_registry = SourceRegistry(
id=source_registry_id,
source_key="test-derived-raster",
display_name="Governed test-derived raster",
classification="derived",
authority_name="GeoIntel test fixture",
usage_policy_json={"ground_truth_allowed": False},
)
source_snapshot = SourceSnapshot(
id=source_snapshot_id,
source_registry_id=source_registry_id,
snapshot_key="test-derived-raster-v1",
checksum_sha256=checksum,
freshness_status="current",
ingest_status="ingested",
)
dataset = Dataset(
id=dataset_id,
project_id=project_id,
name="source.tif",
dataset_type=dataset_type,
source="user_upload",
source="test-derived-raster",
source_name="test-derived-raster",
storage_path="storage/uploads/source.tif",
checksum_sha256=checksum,
source_registry_id=source_registry_id,
source_snapshot_id=source_snapshot_id,
data_contract_key="geointel.raster.geotiff",
data_contract_version="1.0.0",
validation_status="passed",
provenance_status="complete",
lineage_status="not_applicable",
quarantine_status="not_quarantined",
status="ready",
)
dataset.source_registry = source_registry
dataset.source_snapshot = source_snapshot
db = FakeSession(objects={(Project, project_id): project, (Dataset, dataset_id): dataset})
return db, project_id, dataset_id
@@ -165,6 +207,74 @@ def _settings(tmp_path: Path, **overrides) -> Settings:
return Settings(**values)
def _write_model_sidecar(
model_path: Path,
settings: Settings,
*,
db: FakeSession | None = None,
) -> None:
"""Create explicit test-only evidence; production never self-generates it."""
model_sha256 = sha256(model_path.read_bytes()).hexdigest()
source_registry_id = uuid4()
source_snapshot_id = uuid4()
source_version = settings.yolo_model_version or "test-v1"
if db is not None:
source_registry = SourceRegistry(
id=source_registry_id,
source_key="model",
display_name="Governed test model artifact",
classification="experimental",
authority_name="GeoIntel test fixture",
freshness_status="current",
ingest_status="configured",
)
source_snapshot = SourceSnapshot(
id=source_snapshot_id,
source_registry_id=source_registry_id,
snapshot_key=f"model-{source_version}",
source_version=source_version,
checksum_sha256=model_sha256,
freshness_status="current",
ingest_status="ingested",
)
db.objects[(SourceRegistry, source_registry_id)] = source_registry
db.objects[(SourceSnapshot, source_snapshot_id)] = source_snapshot
payload = {
"schema_version": RuntimeModelProvenanceService.MANIFEST_SCHEMA_VERSION,
"data_contract": {"key": "geointel.model.pytorch", "version": "1.0.0"},
"model": {
"model_id": settings.yolo_model_id,
"task_type": "object_detection",
"sha256": model_sha256,
"model_format": "pytorch",
"framework": "ultralytics/pytorch",
"class_mapping": {"0": "building"},
"source_version": source_version,
},
"source": {
"source_registry_id": str(source_registry_id),
"source_snapshot_id": str(source_snapshot_id),
"source_registry_key": "model",
"source_snapshot_checksum_sha256": model_sha256,
},
"lineage": {
"upstream_asset_ids": ["test-training-corpus"],
"upstream_checksums_sha256": ["a" * 64],
"transformations": [
{"name": "test-training", "version": "1.0.0", "checksum_sha256": "b" * 64}
],
},
"metadata": {"training_manifest_sha256": "c" * 64},
"imported_at": "2026-08-01T10:00:00+00:00",
}
payload["metadata"]["runtime_manifest_sha256"] = RuntimeModelProvenanceService.manifest_self_checksum(payload)
RuntimeModelProvenanceService.manifest_path_for_model(model_path).write_text(
json.dumps(payload, sort_keys=True),
encoding="utf-8",
)
def _manifest(tmp_path: Path, tile_count: int = 1) -> Path:
tiles = []
for index in range(tile_count):
@@ -223,10 +333,28 @@ def test_yolo_configured_model_reports_dependency_unavailable(tmp_path: Path) ->
assert model.status == "dependency_unavailable"
def test_yolo_configured_model_requires_a_runtime_provenance_sidecar(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"unmanifested local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path))
model = ModelRegistryService.get_model_capability(
"yolo-configured",
settings=settings,
yolo_adapter_class=AvailableAdapter,
)
assert model is not None
assert model.configured is False
assert model.status == "contract_incomplete"
assert "sidecar" in model.limitation_message
def test_yolo_configured_model_reports_configured_with_local_model_and_dependencies(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path))
_write_model_sidecar(model_path, settings)
model = ModelRegistryService.get_model_capability("yolo-configured", settings=settings, yolo_adapter_class=AvailableAdapter)
@@ -306,11 +434,37 @@ def test_yolo_run_requires_tile_manifest_path(tmp_path: Path) -> None:
assert getattr(exc_info.value, "code", None) == "DETECTION_TILE_MANIFEST_REQUIRED"
def test_yolo_run_fails_closed_before_adapter_load_without_sidecar(tmp_path: Path) -> None:
db, project_id, dataset_id = _project_and_dataset()
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"unmanifested local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path))
# AvailableAdapter intentionally has no load_model method. If runtime
# provenance were checked after adapter loading, this would raise instead
# of returning the explicit unavailable capability state.
result = DetectionService.run_detection(
db=db,
project_id=project_id,
dataset_id=dataset_id,
model_id="yolo-configured",
confidence_threshold=0.5,
tile_manifest_path=str(_manifest(tmp_path)),
settings=settings,
yolo_adapter_class=AvailableAdapter,
)
assert result.status == "failed"
assert result.error_code == "DETECTION_MODEL_UNAVAILABLE"
assert "sidecar" in result.message
def test_yolo_run_rejects_manifest_over_tile_limit(tmp_path: Path) -> None:
db, project_id, dataset_id = _project_and_dataset()
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_max_tiles=1)
_write_model_sidecar(model_path, settings, db=db)
manifest_path = _manifest(tmp_path, tile_count=2)
result = DetectionService.run_detection(
@@ -333,6 +487,7 @@ def test_yolo_run_rejects_missing_tile_manifest_file(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path))
_write_model_sidecar(model_path, settings, db=db)
result = DetectionService.run_detection(
db=db,
@@ -354,6 +509,7 @@ def test_yolo_run_rejects_invalid_tile_manifest_json(tmp_path: Path) -> None:
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path))
_write_model_sidecar(model_path, settings, db=db)
manifest_path = tmp_path / "manifest.json"
manifest_path.write_text("{not-json", encoding="utf-8")
@@ -372,6 +528,32 @@ def test_yolo_run_rejects_invalid_tile_manifest_json(tmp_path: Path) -> None:
assert result.error_code == "DETECTION_TILE_MANIFEST_INVALID"
def test_yolo_run_rejects_unbound_model_snapshot_before_adapter_load(tmp_path: Path) -> None:
db, project_id, dataset_id = _project_and_dataset()
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"structurally valid but unbound model")
settings = _settings(tmp_path, yolo_model_path=str(model_path))
# A catalog/preflight sidecar alone is deliberately insufficient for a
# production call. Do not register the declared source IDs in ``db``.
_write_model_sidecar(model_path, settings)
NeverLoadUnboundModelAdapter.load_calls = 0
result = DetectionService.run_detection(
db=db,
project_id=project_id,
dataset_id=dataset_id,
model_id="yolo-configured",
confidence_threshold=0.5,
tile_manifest_path=str(_manifest(tmp_path)),
settings=settings,
yolo_adapter_class=NeverLoadUnboundModelAdapter,
)
assert result.status == "failed"
assert result.error_code == "MODEL_PROVENANCE_SOURCE_REGISTRY_NOT_FOUND"
assert NeverLoadUnboundModelAdapter.load_calls == 0
def test_pixel_bbox_to_epsg4326_polygon_from_gdal_transform() -> None:
polygon = pixel_bbox_to_epsg4326_polygon(
bbox=[10, 20, 30, 40],
@@ -390,6 +572,7 @@ def test_yolo_run_persists_mocked_georeferenced_detections(tmp_path: Path) -> No
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_model_version="local-test")
_write_model_sidecar(model_path, settings, db=db)
manifest_path = _manifest(tmp_path, tile_count=1)
result = DetectionService.run_detection(
@@ -416,7 +599,10 @@ def test_yolo_run_persists_mocked_georeferenced_detections(tmp_path: Path) -> No
assert detections[0].confidence == 0.91
assert detections[0].source_tile_path.endswith("tile_0000.tif")
assert detections[0].bbox_json == {"x_min": 10.0, "y_min": 20.0, "x_max": 30.0, "y_max": 40.0}
assert detections[0].properties_json == {"adapter": "mock", "tile_index": 0}
assert detections[0].properties_json["adapter"] == "mock"
assert detections[0].properties_json["tile_index"] == 0
assert detections[0].properties_json["runtime_model_provenance"]["model_sha256"] == sha256(model_path.read_bytes()).hexdigest()
assert runs[0].parameters_json["runtime_model_provenance"]["data_contract_key"] == "geointel.model.pytorch"
assert runs[0].status == "success"
assert jobs[0].status == "success"
@@ -426,6 +612,7 @@ def test_yolo_class_filter_is_case_insensitive_and_persists_canonical_class(tmp_
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path))
_write_model_sidecar(model_path, settings, db=db)
manifest_path = _manifest(tmp_path, tile_count=1)
result = DetectionService.run_detection(
@@ -453,6 +640,7 @@ def test_yolo_run_suppresses_cross_tile_duplicate_detections(tmp_path: Path) ->
model_path = tmp_path / "model.pt"
model_path.write_bytes(b"local weights")
settings = _settings(tmp_path, yolo_model_path=str(model_path), yolo_duplicate_iou_threshold=0.5)
_write_model_sidecar(model_path, settings, db=db)
manifest_path = _manifest(tmp_path, tile_count=2)
result = DetectionService.run_detection(