feat(provenance): govern source snapshots and data inputs
This commit is contained in:
@@ -21,9 +21,11 @@ from app.schemas.segmentation import (
|
||||
)
|
||||
from app.services.detection_georeferencing import pixel_points_to_epsg4326_polygon
|
||||
from app.services.detection_service import DetectionService
|
||||
from app.services.dataset_consumption_gate_service import DatasetConsumptionGate
|
||||
from app.services.model_registry_service import ModelRegistryService
|
||||
from app.services.qa_service import QaService
|
||||
from app.services.quality_service import QualityService
|
||||
from app.services.runtime_model_provenance_service import RuntimeModelProvenance, RuntimeModelProvenanceService
|
||||
from app.services.segmentation_adapter import (
|
||||
FixtureSegmentationAdapter,
|
||||
SamSegmentationAdapter,
|
||||
@@ -89,6 +91,18 @@ class SegmentationService:
|
||||
status_code=400,
|
||||
)
|
||||
|
||||
# Production segmentation must consume only a passed, complete and
|
||||
# non-quarantined dataset. The fixture segmenter is QA/test-only and
|
||||
# cannot be classified as production inference.
|
||||
if model.model_id == "fixture-segmenter":
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
dataset,
|
||||
purpose="quality_assessment",
|
||||
fixture_mode=True,
|
||||
)
|
||||
elif model.model_id in configured_model_ids and model.configured:
|
||||
DatasetConsumptionGate.assert_eligible(dataset, purpose="production_inference")
|
||||
|
||||
run_parameters = {
|
||||
"model_id": model.model_id,
|
||||
"confidence_threshold": confidence_threshold,
|
||||
@@ -326,6 +340,27 @@ class SegmentationService:
|
||||
if reference_dataset.dataset_type not in {"vector", "geojson"}:
|
||||
raise AppError(code="INVALID_DATASET_TYPE", message="Reference dataset must be vector data", status_code=400)
|
||||
|
||||
candidate_dataset = db.get(Dataset, run.dataset_id)
|
||||
if not candidate_dataset:
|
||||
raise AppError(code="DATASET_NOT_FOUND", message="Segmentation source dataset not found", status_code=404)
|
||||
run_parameters = run.parameters_json if isinstance(run.parameters_json, dict) else {}
|
||||
fixture_parameters = run_parameters.get("parameters_json")
|
||||
fixture_mode = bool(
|
||||
run.model_name == "fixture-segmenter"
|
||||
and isinstance(fixture_parameters, dict)
|
||||
and fixture_parameters.get("fixture_mode") is True
|
||||
)
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
candidate_dataset,
|
||||
purpose="quality_assessment",
|
||||
fixture_mode=fixture_mode,
|
||||
)
|
||||
DatasetConsumptionGate.assert_eligible(
|
||||
reference_dataset,
|
||||
purpose="reference_validation",
|
||||
reference_task="building_validation",
|
||||
)
|
||||
|
||||
segmentations = SegmentationService._query_segmentation_rows(
|
||||
db,
|
||||
analysis_run_id=analysis_run_id,
|
||||
@@ -510,11 +545,26 @@ class SegmentationService:
|
||||
) -> tuple[list[Segmentation], dict[str, Any]]:
|
||||
manifest = DetectionService._load_tile_manifest(tile_manifest_path, settings.yolo_max_tiles)
|
||||
if model_name == settings.sam_model_id:
|
||||
adapter = sam_adapter_class(settings)
|
||||
model_path = Path(settings.sam_model_path or "").expanduser()
|
||||
allowed_frameworks = ("ultralytics/sam", "sam", "ultralytics", "pytorch")
|
||||
adapter = sam_adapter_class(settings)
|
||||
else:
|
||||
adapter = yolo_seg_adapter_class(settings)
|
||||
model_path = Path(settings.yolo_seg_model_path or "").expanduser()
|
||||
allowed_frameworks = ("ultralytics/pytorch", "ultralytics", "pytorch")
|
||||
adapter = yolo_seg_adapter_class(settings)
|
||||
runtime_model_provenance = RuntimeModelProvenanceService.validate_for_production_runtime(
|
||||
db=db,
|
||||
model_path=model_path,
|
||||
model_id=model_name,
|
||||
task_type="segmentation",
|
||||
expected_model_version=model_version,
|
||||
allowed_frameworks=allowed_frameworks,
|
||||
)
|
||||
SegmentationService._attach_runtime_model_provenance(
|
||||
analysis_run,
|
||||
job,
|
||||
runtime_model_provenance,
|
||||
)
|
||||
model = adapter.load_model(model_path)
|
||||
|
||||
allowed_classes = {DetectionService._canonical_class_name(value) for value in class_filter if DetectionService._canonical_class_name(value)}
|
||||
@@ -591,6 +641,7 @@ class SegmentationService:
|
||||
"tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()),
|
||||
"tile_index": candidate["tile_index"],
|
||||
"device": settings.yolo_device,
|
||||
"runtime_model_provenance": runtime_model_provenance.as_dict(),
|
||||
},
|
||||
)
|
||||
db.add(segmentation)
|
||||
@@ -603,8 +654,25 @@ class SegmentationService:
|
||||
"suppressed_segmentation_count": len(candidates) - len(filtered_candidates),
|
||||
"duplicate_iou_threshold": float(settings.segmentation_duplicate_iou_threshold),
|
||||
"tile_manifest_path": str(Path(tile_manifest_path or "").expanduser()),
|
||||
"runtime_model_provenance": runtime_model_provenance.as_dict(),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _attach_runtime_model_provenance(
|
||||
analysis_run: AnalysisRun,
|
||||
job: Job,
|
||||
provenance: RuntimeModelProvenance,
|
||||
) -> None:
|
||||
"""Record immutable model evidence with a configured segmentation run."""
|
||||
|
||||
evidence = provenance.as_dict()
|
||||
analysis_parameters = dict(analysis_run.parameters_json or {})
|
||||
analysis_parameters["runtime_model_provenance"] = evidence
|
||||
analysis_run.parameters_json = analysis_parameters
|
||||
job_parameters = dict(job.parameters_json or {})
|
||||
job_parameters["runtime_model_provenance"] = evidence
|
||||
job.parameters_json = job_parameters
|
||||
|
||||
@staticmethod
|
||||
def _geodesic_area_m2(geometry: MultiPolygon | Polygon) -> float | None:
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user